diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez19-img1.JPG b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez19-img1.JPG new file mode 100644 index 000000000..355da2c6d Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez19-img1.JPG differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.aux b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.aux new file mode 100644 index 000000000..c22559bd0 --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.aux @@ -0,0 +1,8 @@ +\relax +\@nameuse{bbl@beforestart} +\babel@aux{english}{} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}Lecture 19 - 18-05-2020}{1}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {1.1}Support Vector Machine (SVM)}{4}\protected@file@percent } +\@writefile{lof}{\contentsline {figure}{\numberline {1.1}{\ignorespaces Draw of SVG}}{4}\protected@file@percent } diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.log b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.log new file mode 100644 index 000000000..8b7c7a588 --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.log @@ -0,0 +1,426 @@ +This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 18 MAY 2020 10:18 +entering extended mode +**./lecture19.tex +(lecture19.tex +LaTeX2e <2020-02-02> patch level 5 +L3 programming layer <2020-03-06> +("C:\Program Files\MiKTeX 2.9\tex/latex/subfiles\subfiles.cls" +Document Class: subfiles 2020/02/14 v1.6 Multi-file projects (class) +Preamble taken from file `../main.tex' +("C:\Program Files\MiKTeX 2.9\tex/latex/tools\verbatim.sty" +Package: verbatim 2019/11/10 v1.5r LaTeX2e package for verbatim enhancements +\every@verbatim=\toks15 +\verbatim@line=\toks16 +\verbatim@in@stream=\read2 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/import\import.sty" +Package: import 2020/04/01 v 6.2 +) (../main.tex +("C:\Program Files\MiKTeX 2.9\tex/latex/base\report.cls" +Document Class: report 2019/12/20 v1.4l Standard LaTeX document class +("C:\Program Files\MiKTeX 2.9\tex/latex/base\size12.clo" +File: size12.clo 2019/12/20 v1.4l Standard LaTeX file (size option) +) +\c@part=\count167 +\c@chapter=\count168 +\c@section=\count169 +\c@subsection=\count170 +\c@subsubsection=\count171 +\c@paragraph=\count172 +\c@subparagraph=\count173 +\c@figure=\count174 +\c@table=\count175 +\abovecaptionskip=\skip47 +\belowcaptionskip=\skip48 +\bibindent=\dimen134 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsmath.sty" +Package: amsmath 2020/01/20 v2.17e AMS math features +\@mathmargin=\skip49 + +For additional information on amsmath, use the `?' option. +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amstext.sty" +Package: amstext 2000/06/29 v2.01 AMS text + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsgen.sty" +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks17 +\ex@=\dimen135 +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsbsy.sty" +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen136 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsopn.sty" +Package: amsopn 2016/03/08 v2.02 operator names +) +\inf@bad=\count176 +LaTeX Info: Redefining \frac on input line 227. +\uproot@=\count177 +\leftroot@=\count178 +LaTeX Info: Redefining \overline on input line 389. +\classnum@=\count179 +\DOTSCASE@=\count180 +LaTeX Info: Redefining \ldots on input line 486. +LaTeX Info: Redefining \dots on input line 489. +LaTeX Info: Redefining \cdots on input line 610. +\Mathstrutbox@=\box45 +\strutbox@=\box46 +\big@size=\dimen137 +LaTeX Font Info: Redeclaring font encoding OML on input line 733. +LaTeX Font Info: Redeclaring font encoding OMS on input line 734. +\macc@depth=\count181 +\c@MaxMatrixCols=\count182 +\dotsspace@=\muskip16 +\c@parentequation=\count183 +\dspbrk@lvl=\count184 +\tag@help=\toks18 +\row@=\count185 +\column@=\count186 +\maxfields@=\count187 +\andhelp@=\toks19 +\eqnshift@=\dimen138 +\alignsep@=\dimen139 +\tagshift@=\dimen140 +\tagwidth@=\dimen141 +\totwidth@=\dimen142 +\lineht@=\dimen143 +\@envbody=\toks20 +\multlinegap=\skip50 +\multlinetaggap=\skip51 +\mathdisplay@stack=\toks21 +LaTeX Info: Redefining \[ on input line 2859. +LaTeX Info: Redefining \] on input line 2860. +) +("C:\Program Files\MiKTeX 2.9\tex/latex/systeme\systeme.sty" +("C:\Program Files\MiKTeX 2.9\tex/latex/xstring\xstring.sty" +("C:\Program Files\MiKTeX 2.9\tex/generic/xstring\xstring.tex" +\integerpart=\count188 +\decimalpart=\count189 +) +Package: xstring 2019/02/06 v1.83 String manipulations (CT) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/systeme\systeme.tex" +\SYS_systemecode=\toks22 +\SYS_systempreamble=\toks23 +\SYSeqnum=\count190 +) +Package: systeme 2019/01/13 v0.32 Mise en forme de systemes d'equations (CT) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amssymb.sty" +Package: amssymb 2013/01/14 v3.01 AMS font symbols + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amsfonts.sty" +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/subfiles\subfiles.sty" +Package: subfiles 2020/02/14 v1.6 Multi-file projects (package) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.sty" +Package: babel 2020/02/28 3.41 The Babel package + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\switch.def" +File: switch.def 2020/02/28 3.41 Babel switching mechanism +) +************************************* +* Local config file bblopts.cfg used +* +("C:\Program Files\MiKTeX 2.9\tex/latex/arabi\bblopts.cfg" +File: bblopts.cfg 2005/09/08 v0.1 add Arabic and Farsi to "declared" options of + babel +) +("C:\Program Files\MiKTeX 2.9\tex/latex/babel-english\english.ldf" +Language: english 2017/06/06 v3.3r English support from the babel system + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.def" +File: babel.def 2020/02/28 3.41 Babel common definitions +\babel@savecnt=\count191 +\U@D=\dimen144 + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\txtbabel.def") +\bbl@readstream=\read3 +\bbl@dirlevel=\count192 +) +Package babel Info: \l@canadian = using hyphenrules for english +(babel) (\language0) on input line 102. +Package babel Info: \l@australian = using hyphenrules for ukenglish +(babel) (\language72) on input line 105. +Package babel Info: \l@newzealand = using hyphenrules for ukenglish +(babel) (\language72) on input line 108. +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/xcolor\xcolor.sty" +Package: xcolor 2016/05/11 v2.12 LaTeX color extensions (UK) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\color.cfg" +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package xcolor Info: Driver file: pdftex.def on input line 225. + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-def\pdftex.def" +File: pdftex.def 2018/01/08 v1.0l Graphics/color driver for pdftex +) +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1348. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1352. +Package xcolor Info: Model `RGB' extended on input line 1364. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1366. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1367. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1368. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1369. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1370. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1371. + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\dvipsnam.def" +File: dvipsnam.def 2016/06/17 v3.0m Driver-dependent file (DPC,SPQR) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphicx.sty" +Package: graphicx 2019/11/30 v1.2a Enhanced LaTeX Graphics (DPC,SPQR) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\keyval.sty" +Package: keyval 2014/10/28 v1.15 key=value parser (DPC) +\KV@toks@=\toks24 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphics.sty" +Package: graphics 2019/11/30 v1.4a Standard LaTeX Graphics (DPC,SPQR) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\trig.sty" +Package: trig 2016/01/03 v1.10 sin cos tan (DPC) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\graphics.cfg" +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: pdftex.def on input line 105. +) +\Gin@req@height=\dimen145 +\Gin@req@width=\dimen146 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/sectsty\sectsty.sty" +Package: sectsty 2002/02/25 v2.0.2 Commands to change all sectional heading sty +les + + +LaTeX Warning: Command \underbar has changed. + Check if current package is valid. + + +LaTeX Warning: Command \underline has changed. + Check if current package is valid. + +) ("C:\Program Files\MiKTeX 2.9\tex/latex/framed\framed.sty" +Package: framed 2011/10/22 v 0.96: framed or shaded text with page breaks +\OuterFrameSep=\skip52 +\fb@frw=\dimen147 +\fb@frh=\dimen148 +\FrameRule=\dimen149 +\FrameSep=\dimen150 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/titlesec\titlesec.sty" +Package: titlesec 2019/10/16 v2.13 Sectioning titles +\ttl@box=\box47 +\beforetitleunit=\skip53 +\aftertitleunit=\skip54 +\ttl@plus=\dimen151 +\ttl@minus=\dimen152 +\ttl@toksa=\toks25 +\titlewidth=\dimen153 +\titlewidthlast=\dimen154 +\titlewidthfirst=\dimen155 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/base\fontenc.sty" +Package: fontenc 2020/02/11 v2.0o Standard LaTeX package +))) +("C:\Program Files\MiKTeX 2.9\tex/latex/l3backend\l3backend-pdfmode.def" +File: l3backend-pdfmode.def 2020-03-12 L3 backend support: PDF mode +\l__kernel_color_stack_int=\count193 +\l__pdf_internal_box=\box48 +) +(lecture19.aux) +\openout1 = `lecture19.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. + +("C:\Program Files\MiKTeX 2.9\tex/context/base/mkii\supp-pdf.mkii" +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count194 +\scratchdimen=\dimen156 +\scratchbox=\box49 +\nofMPsegments=\count195 +\nofMParguments=\count196 +\everyMPshowfont=\toks26 +\MPscratchCnt=\count197 +\MPscratchDim=\dimen157 +\MPnumerator=\count198 +\makeMPintoPDFobject=\count199 +\everyMPtoPDFconversion=\toks27 +) ("C:\Program Files\MiKTeX 2.9\tex/latex/epstopdf-pkg\epstopdf-base.sty" +Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf + +("C:\Program Files\MiKTeX 2.9\tex/generic/infwarerr\infwarerr.sty" +Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/grfext\grfext.sty" +Package: grfext 2019/12/03 v1.3 Manage graphics extensions (HO) + +("C:\Program Files\MiKTeX 2.9\tex/generic/kvdefinekeys\kvdefinekeys.sty" +Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/kvoptions\kvoptions.sty" +Package: kvoptions 2019/11/29 v3.13 Key value format for package options (HO) + +("C:\Program Files\MiKTeX 2.9\tex/generic/ltxcmds\ltxcmds.sty" +Package: ltxcmds 2019/12/15 v1.24 LaTeX kernel commands for general use (HO) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/kvsetkeys\kvsetkeys.sty" +Package: kvsetkeys 2019/12/15 v1.18 Key value parser (HO) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/pdftexcmds\pdftexcmds.sty" +Package: pdftexcmds 2019/11/24 v0.31 Utility functions of pdfTeX for LuaTeX (HO +) + +("C:\Program Files\MiKTeX 2.9\tex/generic/iftex\iftex.sty" +Package: iftex 2020/03/06 v1.0d TeX engine tests +) +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode found. +) +Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 4 +85. +Package grfext Info: Graphics extension search list: +(grfext) [.pdf,.png,.jpg,.mps,.jpeg,.jbig2,.jb2,.PDF,.PNG,.JPG,.JPE +G,.JBIG2,.JB2,.eps] +(grfext) \AppendGraphicsExtensions on input line 504. +) +Chapter 1. +LaTeX Font Info: Trying to load font information for U+msa on input line 6. +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsa.fd" +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) +LaTeX Font Info: Trying to load font information for U+msb on input line 6. + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsb.fd" +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) +Underfull \hbox (badness 10000) in paragraph at lines 13--16 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 18--23 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 25--53 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 59--67 + + [] + +[1 + + +{C:/Users/AndreDany/AppData/Local/MiKTeX/2.9/pdftex/config/pdftex.map}] +Overfull \hbox (23.0944pt too wide) detected at line 72 +\OT1/cmr/m/n/12 = [] \OML/cmm/m/it/12 [] < k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 + x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 )\OML/cmm/m/it/12 ; k\OT1/cmr/m/n/12 (\ +OML/cmm/m/it/12 x; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 ) \OML/cmm/m/it/12 >[] \OT +1/cmr/m/n/12 = [] \OML/cmm/m/it/12 [] < \OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[ +]\OT1/cmr/m/n/12 )\OML/cmm/m/it/12 ; []\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x\OT1 +/cmr/m/n/12 ) \OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = [] \OML/cmm/m/it/12 []k\ +OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; x\OT1/cmr/m/n/12 ) = \OML/cmm/m/it/12 g\O +T1/cmr/m/n/12 (\OML/cmm/m/it/12 x\OT1/cmr/m/n/12 ) + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 76--79 + + [] + + +Overfull \hbox (32.28714pt too wide) detected at line 85 +\OML/cmm/m/it/12 < f; g >[]\OT1/cmr/m/n/12 =\OML/cmm/m/it/12 < [] [] k\OT1/cmr +/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 )\OML/cmm/m/it +/12 ; [] [] k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr +/m/n/12 ) \OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = [] [] \OML/cmm/m/it/12 [] [ +] < k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 ) +\OML/cmm/m/it/12 ; k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12  \ +OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 99--105 + + [] + +[2] +Underfull \hbox (badness 10000) in paragraph at lines 124--130 + + [] + +[3] +Underfull \hbox (badness 10000) in paragraph at lines 140--147 + + [] + +<../img/lez19-img1.JPG, id=24, 291.33844pt x 141.52875pt> +File: ../img/lez19-img1.JPG Graphic file (type jpg) + +Package pdftex.def Info: ../img/lez19-img1.JPG used on input line 157. +(pdftex.def) Requested size: 155.99762pt x 75.78299pt. + +Underfull \hbox (badness 10000) in paragraph at lines 164--172 + + [] + +[4 <../img/lez19-img1.JPG>] (lecture19.aux) ) +Here is how much of TeX's memory you used: + 5098 strings out of 480934 + 68805 string characters out of 2909670 + 334085 words of memory out of 3000000 + 20852 multiletter control sequences out of 15000+200000 + 544429 words of font info for 52 fonts, out of 3000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 42i,7n,50p,333b,234s stack positions out of 5000i,500n,10000p,200000b,50000s + +Output written on lecture19.pdf (4 pages, 164331 bytes). +PDF statistics: + 188 PDF objects out of 1000 (max. 8388607) + 0 named destinations out of 1000 (max. 500000) + 6 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.pdf b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.pdf new file mode 100644 index 000000000..851b635df Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.pdf differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.synctex.gz b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.synctex.gz new file mode 100644 index 000000000..b73ba6ef3 Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.synctex.gz differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.tex b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.tex new file mode 100644 index 000000000..b06a6fc64 --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture19.tex @@ -0,0 +1,172 @@ +\documentclass[../main.tex]{subfiles} +\begin{document} + +\chapter{Lecture 19 - 18-05-2020} + +$$ +k(x,x') = < \phi(x), \phi(x')> +\qquad \phi: X \rightarrow H +$$ +where $X \rightarrow \barra{R}^2$ and $H \rightarrow barra{R}^N $ +$$ +H_\delta = \{ \sum_{i =1}^N \alpha_i \, k_\delta (x_i, \cdot), x_1,..., x_N \in \barra{R}^d, \alpha_1, ... \alpha_N, N \in \barra{N } \} +$$ +Inner product measures "similarities" between data points. +\\ +$$ +x^T \, x' = \|x\| \, \|x'\| \, \cos \Theta \qquad x \in X \quad k(x,x') +$$ +$k$ sais how much similar are the structure (tree, documents etc). +\\ +I would like to learn a predictor based on the notion of similarity. +\\ +$$ +k(x,x') = < \phi(x), \phi(x')> +$$ +where $<>$ is the inner product. +\\ +So we have Data $\rightarrow$ Kernel $\rightarrow$ Kernel learning Algortithm +\\ +Kernels offer a uniform interface to data in such way they algoriithm can learn from data. +\\ +Given $K$ on $X$, I need to find $\exists H_k \quad \phi_k \ X \rightarrow H_k$ +\\ +$\exists <...>_k$ s.t $k(x,x') = <\phi_k(x), \phi_k(x')>_k $ +\\\\ +\bred{Theorem} +\\ +Given $K: X \times X \rightarrow \barra{R}$, symmetric +\\ +Then $K$ is a Kernel iif $\forall m \in \barra{N}$ $\forall x_1,...,x_m \in X$ +\\ +The $m \times m$ matrix $K$ \quad $K_{ij} = k (x_i,x_j)$ is positive semidefinite\\ +$ +\forall \alpha \in \barra{R}^m \qquad \alpha^T \, K \, \alpha \geq 0 +$ +\\ +In general, given a Kernel $K$ there is not unique representation for $\phi_k$ and $<...>_k$ (inner product). +\\ +However, there is a "canonical" representation: +$ +\phi_k(x) = K(x, \cdot) +$ +$$ +\phi_k : X \rightarrow H \qquad H_k = \{ \sum_{i=1}^N \alpha_i \, k (x_i, \cdot ), \alpha_1,..., \alpha_N \in \barra{R}, x_1,...,x_N \in X, N \in \barra{N} \} +$$ +We have to define an inner product like: +$$ +<\phi_k(x), \phi_k(x')>_k \ = \ k(x,x') +$$ +This is the canonical representation that helps mapping. +\\\\ +What happen to use this mechanism to perform predictions? +\\ +$ +x \in \barra{R}^d \ w \in \barra{R}^d \ w^T \, x \qquad \textit{\ where } g = \sum_{i=1}^N \alpha_i \, k (x_i, \cdot) +$ +$$ +\phi_k(x) \qquad g \in H_k \qquad _k \ = \ <\sum_i \alpha_i k(x_i, \cdot), \phi_k(x)> \ = $$ +We have to satisfy allinearity +$$ += \ \sum_i \alpha_i _k \ = \ \sum_i \alpha_i <\phi(x_i), \phi_k(x)>_k \ = \ \sum_i \alpha_i k(x_i, x) = g(x) +$$ +At the end we have: +$$ +_k \ = \ g(x) +$$ +\\\\ +Now, if i have two functions: +$$ +f = \sum_{i=1}^N \alpha_i \, k(x_i, \cdot) \qquad g = \sum_{j=1}^M \beta_j \, k (x'_j, \cdot) \qquad f,g \in H_k +$$ +$$ +_k = <\sum_i \alpha_i \, k(x_i,\cdot) , \sum_j \beta_j \, k(x'_j, \cdot) >_k \ = +\ \sum_i \sum_j \alpha_i \, \beta_j _k \ = +$$ +$$ += \ \sum_i \sum_j \alpha_i \, \beta_j \, k(x_i, x_j) +$$ +$$ +\|f\|^2 = _k = \sum_{ij} \alpha_i \, \alpha_j \, k(x_i, x_j) +$$ +Perceptron convergence theorem in kernel space: +$$ +M \leq \|U\|^2 ( \max_t \|x_t\| ^2) \qquad \forall u \in \barra{R}^d \quad y_t \, u^T \, x_t \geq 1 quad \forall g \in H_k \quad y_t \, g(x_t) \geq 1 +$$ +we know that: +$$ +\|x_t \|^2 \rightsquigarrow \| \phi_k(x_t)\|^2_k \ = \ <\phi_k(x_t), \phi_k(\alpha_t) >_k \ = \ k (x_t,x_t) +$$ +so +\\ +.... MANCA ULTIMA FORUMA +\\\\ +Ridge regression: +$$ +w = \left( \alpha \, I + S^T \, S \right)^{-1} \, S^T \, y +$$ +$S$ is $m \times d$ matrix whose rows are the training points $x_1,..., x_m \in \barra{R}^d$ +\\ +$y = (y_1,...,y_m) \quad y_t \in \barra{R}^d$ training labels $\alpha >0$ +$$ +\left( \alpha \, I + S^T \, S \right)^{-1} \, S^T \ = \ S^T \left( \alpha \, I_m + S\, S^T\right)^{-1} +$$ +where $d \times d$ and $d \times m $ = $ d \times +m$ and $m \times m$ +$$ +\left( S \, S^T\right)_{ij} = x_i^T x_j \qquad \rightsquigarrow \ <\phi(x_i),\phi(x_j)>_k = k(x_i, x_j) = K_{ij} +$$ +$$ +S^T = \left[ x_1,...,x_m \right] \ \rightsquigarrow\ \left[ \ \phi_k(x_i),..., \phi_k(x_m) \ \right] = \left[ \ k(x_1, \cdot), ..., k(k_m, \cdot) \ \right] \ = \ k(\cdot) +$$ +$$ +k (\cdot)^T \, \left( \alpha \, I_m + K \right)^{-1} \, y \ = \ g +$$ +where $1 \times +m$ and $m \times +m$ and $m \times +1$\\\\ +How to compute prediction? +$$ +g(x) = y^T \left( \alpha \, I_m + K \right)^{-1} \, k(x) +$$ +\qquad $1 \times m$ and $m \times m$ and $m \times 1$ +\\ +In fact, is the evaltuation of $g$ in any point $x$. +\\ +The drawback is that we pass from $d \times d$ matrix to a $m \times m$ matrix that can be huge. So it is not really efficient in this way, we need to use addictional "tricks" having a more compact representation of the last matrix prediction. +\newpage +\section{Support Vector Machine (SVM)} +It is a linear predictor and is a very popular one because has better performance than perceptron and we will see it for classification but there are also version for regression. +\\\\ +The idea here is that you want to come up with an hyperplane that is defined as a solution of an optimisation problem. +\\ +We have a classification dataset $(x_1,y_1)...(x_m,y_m) \qquad x_t \in \barra{R}^d \quad y_t \in \{ -1,1 \}$ and it is linearly separable. +\\ +Sum as the solution $w^*$ (optimisation problem) to this problem: +$$ +\min_{w \in \barra{R}^d} \frac{1}{2} \| w \|^2 \qquad s.t \quad y_t \, w^T \, x_t \geq 1 \quad t = 1,2,...,m +$$ +Geometrically $w^*$ corresponds to the maximum marging separating hyperplane like: +$$ +\gamma^* = \max_{u: \|u\|=1} y_t \, u^t \, x_t \qquad t=1,...,m +$$\ +\textbf{$u^*$ is achieving $\gamma^*$ is the maximal margin separator.}\\ +\begin{figure}[h] + \centering + \includegraphics[width=0.4\linewidth]{../img/lez19-img1.JPG} + \caption{Draw of SVG} + %\label{fig:} +\end{figure}\\ +So I want to maximise this distance. +$$ +\max_{\gamma > 0} \, \gamma^2 \qquad s.t \quad \| u \|^2 = 1 \qquad y_t \, u^t \, x_t \geq \gamma \quad t=1,...,m +$$ +So we can maximise instead of minimising. +\\ +What is the theorem? The equivalent between this two. +\\\\ +\bred{Theorem}:\\ +$\forall$ linear separator $(x_1,y_1)...(x_m,y_m)$ \\ +The max margin separator $u^*$ satisfies $u^* = \gamma^* \, w^*$ where $w^*$ is the SVM solution and $\gamma^*$ is the maximum margin. +\end{document} \ No newline at end of file diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture20.tex b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture20.tex new file mode 100644 index 000000000..8b59e1646 --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture20.tex @@ -0,0 +1,6 @@ +\documentclass[../main.tex]{subfiles} +\begin{document} + +\chapter{Lecture 20 - 19-05-2020} + +\end{document} \ No newline at end of file diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux index 8a90dad14..0cb55c009 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux @@ -196,5 +196,13 @@ \@writefile{toc}{\contentsline {section}{\numberline {18.2}Gaussian Kernel}{114}\protected@file@percent } \@writefile{lof}{\contentsline {figure}{\numberline {18.2}{\ignorespaces }}{115}\protected@file@percent } \@writefile{lof}{\contentsline {figure}{\numberline {18.3}{\ignorespaces }}{115}\protected@file@percent } +\@writefile{toc}{\contentsline {chapter}{\numberline {19}Lecture 19 - 18-05-2020}{117}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {19.1}Support Vector Machine (SVM)}{120}\protected@file@percent } +\@writefile{lof}{\contentsline {figure}{\numberline {19.1}{\ignorespaces Draw of SVG}}{120}\protected@file@percent } +\@writefile{toc}{\contentsline {chapter}{\numberline {20}Lecture 20 - 19-05-2020}{121}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} \bibstyle{abbrv} \bibdata{main} diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof index 1112b0f76..784815b20 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof @@ -92,3 +92,6 @@ \contentsline {figure}{\numberline {18.1}{\ignorespaces }}{112}% \contentsline {figure}{\numberline {18.2}{\ignorespaces }}{115}% \contentsline {figure}{\numberline {18.3}{\ignorespaces }}{115}% +\addvspace {10\p@ } +\contentsline {figure}{\numberline {19.1}{\ignorespaces Draw of SVG}}{120}% +\addvspace {10\p@ } diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log index adc720240..7d34a2af0 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 17 MAY 2020 17:34 +This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 18 MAY 2020 10:21 entering extended mode **./main.tex (main.tex @@ -2938,18 +2938,100 @@ Underfull \hbox (badness 10000) in paragraph at lines 171--145 [] -[116] (main.bbl +[116] (lectures/lecture19.tex +Chapter 19. + +Underfull \hbox (badness 10000) in paragraph at lines 13--16 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 18--23 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 25--53 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 59--67 + + [] + +[117 + +] +Overfull \hbox (23.0944pt too wide) detected at line 72 +\OT1/cmr/m/n/12 = [] \OML/cmm/m/it/12 [] < k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 + x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 )\OML/cmm/m/it/12 ; k\OT1/cmr/m/n/12 (\ +OML/cmm/m/it/12 x; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 ) \OML/cmm/m/it/12 >[] \OT +1/cmr/m/n/12 = [] \OML/cmm/m/it/12 [] < \OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[ +]\OT1/cmr/m/n/12 )\OML/cmm/m/it/12 ; []\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x\OT1 +/cmr/m/n/12 ) \OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = [] \OML/cmm/m/it/12 []k\ +OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; x\OT1/cmr/m/n/12 ) = \OML/cmm/m/it/12 g\O +T1/cmr/m/n/12 (\OML/cmm/m/it/12 x\OT1/cmr/m/n/12 ) + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 76--79 + + [] + + +Overfull \hbox (32.28714pt too wide) detected at line 85 +\OML/cmm/m/it/12 < f; g >[]\OT1/cmr/m/n/12 =\OML/cmm/m/it/12 < [] [] k\OT1/cmr +/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 )\OML/cmm/m/it +/12 ; [] [] k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr +/m/n/12 ) \OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = [] [] \OML/cmm/m/it/12 [] [ +] < k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12 \OT1/cmr/m/n/12 ) +\OML/cmm/m/it/12 ; k\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 x[]; \OMS/cmsy/m/n/12  \ +OML/cmm/m/it/12 >[] \OT1/cmr/m/n/12 = + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 99--105 + + [] + +[118] +Underfull \hbox (badness 10000) in paragraph at lines 124--130 + + [] + +[119] +Underfull \hbox (badness 10000) in paragraph at lines 140--147 + + [] + + +File: lectures/../img/lez19-img1.JPG Graphic file (type jpg) + +Package pdftex.def Info: lectures/../img/lez19-img1.JPG used on input line 157 +. +(pdftex.def) Requested size: 155.99762pt x 75.78299pt. +) +Underfull \hbox (badness 10000) in paragraph at lines 164--148 + + [] + +[120 <./lectures/../img/lez19-img1.JPG>] (lectures/lecture20.tex +Chapter 20. +) [121 + +] (main.bbl LaTeX Warning: Empty `thebibliography' environment on input line 3. -) [117 +) [122 ] (main.aux) ) Here is how much of TeX's memory you used: - 5682 strings out of 480934 - 86666 string characters out of 2909670 + 5696 strings out of 480934 + 87332 string characters out of 2909670 336249 words of memory out of 3000000 - 21279 multiletter control sequences out of 15000+200000 + 21286 multiletter control sequences out of 15000+200000 561784 words of font info for 96 fonts, out of 3000000 for 9000 1141 hyphenation exceptions out of 8191 34i,13n,42p,348b,361s stack positions out of 5000i,500n,10000p,200000b,50000s @@ -2981,9 +3063,9 @@ s/cm/cmr8.pfb> -Output written on main.pdf (118 pages, 2599444 bytes). +Output written on main.pdf (123 pages, 2630747 bytes). PDF statistics: - 988 PDF objects out of 1000 (max. 8388607) + 1008 PDF objects out of 1200 (max. 8388607) 0 named destinations out of 1000 (max. 500000) - 386 words of extra memory for PDF output out of 10000 (max. 10000000) + 391 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf index f7f2d1f50..ef787c70f 100644 Binary files a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz index 4a902c74f..983b6744a 100644 Binary files a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex index d0517ae4d..a014231f9 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex @@ -143,6 +143,11 @@ \newpage \subfile{lectures/lecture18} +\newpage +\subfile{lectures/lecture19} + +\newpage +\subfile{lectures/lecture20} \newpage \bibliographystyle{abbrv} \bibliography{main} diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc index b6f419c5a..4a602a888 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc @@ -83,3 +83,6 @@ \contentsline {subsection}{\numberline {18.1.1}Feature expansion}{112}% \contentsline {subsection}{\numberline {18.1.2}Kernels implements feature expansion (Efficiently}{113}% \contentsline {section}{\numberline {18.2}Gaussian Kernel}{114}% +\contentsline {chapter}{\numberline {19}Lecture 19 - 18-05-2020}{117}% +\contentsline {section}{\numberline {19.1}Support Vector Machine (SVM)}{120}% +\contentsline {chapter}{\numberline {20}Lecture 20 - 19-05-2020}{121}%