diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez21-img1.JPG b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez21-img1.JPG new file mode 100644 index 000000000..5e4f63398 Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/img/lez21-img1.JPG differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.aux b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.aux new file mode 100644 index 000000000..f1d0dea8f --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.aux @@ -0,0 +1,9 @@ +\relax +\@nameuse{bbl@beforestart} +\babel@aux{english}{} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}Lecture 21 - 25-05-2020}{1}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {1.1}Pegasos in Kernel space}{1}\protected@file@percent } +\@writefile{toc}{\contentsline {section}{\numberline {1.2}Stability}{1}\protected@file@percent } +\@writefile{lof}{\contentsline {figure}{\numberline {1.1}{\ignorespaces }}{5}\protected@file@percent } diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.bbl b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.bbl new file mode 100644 index 000000000..e69de29bb diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.blg b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.blg new file mode 100644 index 000000000..b55ee6240 --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.blg @@ -0,0 +1,5 @@ +This is BibTeX, Version 0.99dThe top-level auxiliary file: lecture21.aux +I found no \citation commands---while reading file lecture21.aux +I found no \bibdata command---while reading file lecture21.aux +I found no \bibstyle command---while reading file lecture21.aux +(There were 3 error messages) diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.log b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.log new file mode 100644 index 000000000..67a81870d --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.log @@ -0,0 +1,429 @@ +This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 25 MAY 2020 10:27 +entering extended mode +**./lecture21.tex +(lecture21.tex +LaTeX2e <2020-02-02> patch level 5 +L3 programming layer <2020-03-06> +("C:\Program Files\MiKTeX 2.9\tex/latex/subfiles\subfiles.cls" +Document Class: subfiles 2020/02/14 v1.6 Multi-file projects (class) +Preamble taken from file `../main.tex' +("C:\Program Files\MiKTeX 2.9\tex/latex/tools\verbatim.sty" +Package: verbatim 2019/11/10 v1.5r LaTeX2e package for verbatim enhancements +\every@verbatim=\toks15 +\verbatim@line=\toks16 +\verbatim@in@stream=\read2 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/import\import.sty" +Package: import 2020/04/01 v 6.2 +) (../main.tex +("C:\Program Files\MiKTeX 2.9\tex/latex/base\report.cls" +Document Class: report 2019/12/20 v1.4l Standard LaTeX document class +("C:\Program Files\MiKTeX 2.9\tex/latex/base\size12.clo" +File: size12.clo 2019/12/20 v1.4l Standard LaTeX file (size option) +) +\c@part=\count167 +\c@chapter=\count168 +\c@section=\count169 +\c@subsection=\count170 +\c@subsubsection=\count171 +\c@paragraph=\count172 +\c@subparagraph=\count173 +\c@figure=\count174 +\c@table=\count175 +\abovecaptionskip=\skip47 +\belowcaptionskip=\skip48 +\bibindent=\dimen134 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsmath.sty" +Package: amsmath 2020/01/20 v2.17e AMS math features +\@mathmargin=\skip49 + +For additional information on amsmath, use the `?' option. +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amstext.sty" +Package: amstext 2000/06/29 v2.01 AMS text + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsgen.sty" +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks17 +\ex@=\dimen135 +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsbsy.sty" +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen136 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsopn.sty" +Package: amsopn 2016/03/08 v2.02 operator names +) +\inf@bad=\count176 +LaTeX Info: Redefining \frac on input line 227. +\uproot@=\count177 +\leftroot@=\count178 +LaTeX Info: Redefining \overline on input line 389. +\classnum@=\count179 +\DOTSCASE@=\count180 +LaTeX Info: Redefining \ldots on input line 486. +LaTeX Info: Redefining \dots on input line 489. +LaTeX Info: Redefining \cdots on input line 610. +\Mathstrutbox@=\box45 +\strutbox@=\box46 +\big@size=\dimen137 +LaTeX Font Info: Redeclaring font encoding OML on input line 733. +LaTeX Font Info: Redeclaring font encoding OMS on input line 734. +\macc@depth=\count181 +\c@MaxMatrixCols=\count182 +\dotsspace@=\muskip16 +\c@parentequation=\count183 +\dspbrk@lvl=\count184 +\tag@help=\toks18 +\row@=\count185 +\column@=\count186 +\maxfields@=\count187 +\andhelp@=\toks19 +\eqnshift@=\dimen138 +\alignsep@=\dimen139 +\tagshift@=\dimen140 +\tagwidth@=\dimen141 +\totwidth@=\dimen142 +\lineht@=\dimen143 +\@envbody=\toks20 +\multlinegap=\skip50 +\multlinetaggap=\skip51 +\mathdisplay@stack=\toks21 +LaTeX Info: Redefining \[ on input line 2859. +LaTeX Info: Redefining \] on input line 2860. +) +("C:\Program Files\MiKTeX 2.9\tex/latex/systeme\systeme.sty" +("C:\Program Files\MiKTeX 2.9\tex/latex/xstring\xstring.sty" +("C:\Program Files\MiKTeX 2.9\tex/generic/xstring\xstring.tex" +\integerpart=\count188 +\decimalpart=\count189 +) +Package: xstring 2019/02/06 v1.83 String manipulations (CT) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/systeme\systeme.tex" +\SYS_systemecode=\toks22 +\SYS_systempreamble=\toks23 +\SYSeqnum=\count190 +) +Package: systeme 2019/01/13 v0.32 Mise en forme de systemes d'equations (CT) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amssymb.sty" +Package: amssymb 2013/01/14 v3.01 AMS font symbols + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amsfonts.sty" +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/subfiles\subfiles.sty" +Package: subfiles 2020/02/14 v1.6 Multi-file projects (package) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.sty" +Package: babel 2020/02/28 3.41 The Babel package + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\switch.def" +File: switch.def 2020/02/28 3.41 Babel switching mechanism +) +************************************* +* Local config file bblopts.cfg used +* +("C:\Program Files\MiKTeX 2.9\tex/latex/arabi\bblopts.cfg" +File: bblopts.cfg 2005/09/08 v0.1 add Arabic and Farsi to "declared" options of + babel +) +("C:\Program Files\MiKTeX 2.9\tex/latex/babel-english\english.ldf" +Language: english 2017/06/06 v3.3r English support from the babel system + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.def" +File: babel.def 2020/02/28 3.41 Babel common definitions +\babel@savecnt=\count191 +\U@D=\dimen144 + +("C:\Program Files\MiKTeX 2.9\tex/generic/babel\txtbabel.def") +\bbl@readstream=\read3 +\bbl@dirlevel=\count192 +) +Package babel Info: \l@canadian = using hyphenrules for english +(babel) (\language0) on input line 102. +Package babel Info: \l@australian = using hyphenrules for ukenglish +(babel) (\language72) on input line 105. +Package babel Info: \l@newzealand = using hyphenrules for ukenglish +(babel) (\language72) on input line 108. +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/xcolor\xcolor.sty" +Package: xcolor 2016/05/11 v2.12 LaTeX color extensions (UK) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\color.cfg" +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package xcolor Info: Driver file: pdftex.def on input line 225. + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-def\pdftex.def" +File: pdftex.def 2018/01/08 v1.0l Graphics/color driver for pdftex +) +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1348. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1352. +Package xcolor Info: Model `RGB' extended on input line 1364. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1366. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1367. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1368. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1369. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1370. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1371. + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\dvipsnam.def" +File: dvipsnam.def 2016/06/17 v3.0m Driver-dependent file (DPC,SPQR) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphicx.sty" +Package: graphicx 2019/11/30 v1.2a Enhanced LaTeX Graphics (DPC,SPQR) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\keyval.sty" +Package: keyval 2014/10/28 v1.15 key=value parser (DPC) +\KV@toks@=\toks24 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphics.sty" +Package: graphics 2019/11/30 v1.4a Standard LaTeX Graphics (DPC,SPQR) + +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\trig.sty" +Package: trig 2016/01/03 v1.10 sin cos tan (DPC) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\graphics.cfg" +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: pdftex.def on input line 105. +) +\Gin@req@height=\dimen145 +\Gin@req@width=\dimen146 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/sectsty\sectsty.sty" +Package: sectsty 2002/02/25 v2.0.2 Commands to change all sectional heading sty +les + + +LaTeX Warning: Command \underbar has changed. + Check if current package is valid. + + +LaTeX Warning: Command \underline has changed. + Check if current package is valid. + +) ("C:\Program Files\MiKTeX 2.9\tex/latex/framed\framed.sty" +Package: framed 2011/10/22 v 0.96: framed or shaded text with page breaks +\OuterFrameSep=\skip52 +\fb@frw=\dimen147 +\fb@frh=\dimen148 +\FrameRule=\dimen149 +\FrameSep=\dimen150 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/titlesec\titlesec.sty" +Package: titlesec 2019/10/16 v2.13 Sectioning titles +\ttl@box=\box47 +\beforetitleunit=\skip53 +\aftertitleunit=\skip54 +\ttl@plus=\dimen151 +\ttl@minus=\dimen152 +\ttl@toksa=\toks25 +\titlewidth=\dimen153 +\titlewidthlast=\dimen154 +\titlewidthfirst=\dimen155 +) +("C:\Program Files\MiKTeX 2.9\tex/latex/base\fontenc.sty" +Package: fontenc 2020/02/11 v2.0o Standard LaTeX package +))) +("C:\Program Files\MiKTeX 2.9\tex/latex/l3backend\l3backend-pdfmode.def" +File: l3backend-pdfmode.def 2020-03-12 L3 backend support: PDF mode +\l__kernel_color_stack_int=\count193 +\l__pdf_internal_box=\box48 +) +(lecture21.aux) +\openout1 = `lecture21.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 2. +LaTeX Font Info: ... okay on input line 2. + +("C:\Program Files\MiKTeX 2.9\tex/context/base/mkii\supp-pdf.mkii" +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count194 +\scratchdimen=\dimen156 +\scratchbox=\box49 +\nofMPsegments=\count195 +\nofMParguments=\count196 +\everyMPshowfont=\toks26 +\MPscratchCnt=\count197 +\MPscratchDim=\dimen157 +\MPnumerator=\count198 +\makeMPintoPDFobject=\count199 +\everyMPtoPDFconversion=\toks27 +) ("C:\Program Files\MiKTeX 2.9\tex/latex/epstopdf-pkg\epstopdf-base.sty" +Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf + +("C:\Program Files\MiKTeX 2.9\tex/generic/infwarerr\infwarerr.sty" +Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO) +) +("C:\Program Files\MiKTeX 2.9\tex/latex/grfext\grfext.sty" +Package: grfext 2019/12/03 v1.3 Manage graphics extensions (HO) + +("C:\Program Files\MiKTeX 2.9\tex/generic/kvdefinekeys\kvdefinekeys.sty" +Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/kvoptions\kvoptions.sty" +Package: kvoptions 2019/11/29 v3.13 Key value format for package options (HO) + +("C:\Program Files\MiKTeX 2.9\tex/generic/ltxcmds\ltxcmds.sty" +Package: ltxcmds 2019/12/15 v1.24 LaTeX kernel commands for general use (HO) +) +("C:\Program Files\MiKTeX 2.9\tex/generic/kvsetkeys\kvsetkeys.sty" +Package: kvsetkeys 2019/12/15 v1.18 Key value parser (HO) +)) +("C:\Program Files\MiKTeX 2.9\tex/latex/pdftexcmds\pdftexcmds.sty" +Package: pdftexcmds 2019/11/24 v0.31 Utility functions of pdfTeX for LuaTeX (HO +) + +("C:\Program Files\MiKTeX 2.9\tex/generic/iftex\iftex.sty" +Package: iftex 2020/03/06 v1.0d TeX engine tests +) +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode found. +) +Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 4 +85. +Package grfext Info: Graphics extension search list: +(grfext) [.pdf,.png,.jpg,.mps,.jpeg,.jbig2,.jb2,.PDF,.PNG,.JPG,.JPE +G,.JBIG2,.JB2,.eps] +(grfext) \AppendGraphicsExtensions on input line 504. +) +Chapter 1. +LaTeX Font Info: Trying to load font information for U+msa on input line 8. +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsa.fd" +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) +LaTeX Font Info: Trying to load font information for U+msb on input line 8. + +("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsb.fd" +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) +Overfull \hbox (20.70583pt too wide) detected at line 11 +\OML/cmm/m/it/12 w[] \OT1/cmr/m/n/12 = [] [] \OML/cmm/m/it/12 y[] x[] I\OMS/cms +y/m/n/12 f\OML/cmm/m/it/12 h[]\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 w[]\OT1/cmr/m/n +/12 ) \OML/cmm/m/it/12 > \OT1/cmr/m/n/12 0\OMS/cmsy/m/n/12 g \OML/cmm/m/it/12 s +[]; :::; s[] \OT1/cmr/m/n/12 (\OML/cmm/m/it/12 realised draws in training\OT1/c +mr/m/n/12 ) + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 23--31 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 23--31 + + [] + +[1 + + +{C:/Users/AndreDany/AppData/Local/MiKTeX/2.9/pdftex/config/pdftex.map}] +Underfull \hbox (badness 10000) in paragraph at lines 55--60 + + [] + + +Overfull \hbox (0.64156pt too wide) detected at line 72 +\U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = \U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = [] [] + \U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = [] [] \U/msb/m/n/12 E [] + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 82--89 + + [] + +[2] +Underfull \hbox (badness 10000) in paragraph at lines 95--98 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 106--117 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 119--128 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 119--128 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 130--137 + + [] + +[3] +Overfull \hbox (42.9944pt too wide) detected at line 154 + \OT1/cmr/m/n/12 = [](\OML/cmm/m/it/12 w[]\OT1/cmr/m/n/12 ) \OMS/cmsy/m/n/12 + [] \OT1/cmr/m/n/12 + [] [] \OMS/cmsy/m/n/12 [] [] \OT1/cmr/m/n/12 + [](\OMS/ +cmsy/m/n/12 k\OML/cmm/m/it/12 w[]\OMS/cmsy/m/n/12 k[] k\OML/cmm/m/it/12 w\OMS +/cmsy/m/n/12 k[]\OT1/cmr/m/n/12 ) = + [] + +<../img/lez21-img1.JPG, id=26, 407.27156pt x 228.10219pt> +File: ../img/lez21-img1.JPG Graphic file (type jpg) + +Package pdftex.def Info: ../img/lez21-img1.JPG used on input line 181. +(pdftex.def) Requested size: 312.00119pt x 174.75166pt. + +LaTeX Warning: `h' float specifier changed to `ht'. + +[4] [5 <../img/lez21-img1.JPG>] (lecture21.aux) ) +Here is how much of TeX's memory you used: + 5104 strings out of 480934 + 68864 string characters out of 2909670 + 334085 words of memory out of 3000000 + 20856 multiletter control sequences out of 15000+200000 + 544911 words of font info for 55 fonts, out of 3000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 42i,7n,50p,333b,238s stack positions out of 5000i,500n,10000p,200000b,50000s + < +C:\Users\AndreDany\AppData\Local\MiKTeX\2.9\fonts/pk/ljfour/jknappen/ec/dpi600\ +ecrm1200.pk> +Output written on lecture21.pdf (5 pages, 173176 bytes). +PDF statistics: + 187 PDF objects out of 1000 (max. 8388607) + 0 named destinations out of 1000 (max. 500000) + 6 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.pdf b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.pdf new file mode 100644 index 000000000..34655e95f Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.pdf differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.synctex.gz b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.synctex.gz new file mode 100644 index 000000000..b4fa4abb4 Binary files /dev/null and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.synctex.gz differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.tex b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.tex new file mode 100644 index 000000000..dfa09b7dd --- /dev/null +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/lectures/lecture21.tex @@ -0,0 +1,185 @@ +\documentclass[../main.tex]{subfiles} +\begin{document} + +\chapter{Lecture 21 - 25-05-2020} + +\section{Pegasos in Kernel space} + +Objective function was $$F_\lambda (w) = \frac{1}{m} \sum_{t=1}^{m} h_t(w) + \frac{1}{2} \|w\|^2 \quad w \in \barra{R}^d$$ +$$ +w_{T+1} = \frac{1}{\lambda \, T} \sum_{t=1}^T y_{st} \, x_{st} \, I \{ h_{st} (w_t) > 0 \} \qquad s_1, ..., s_t \quad (realised \ draws \ in \ training) +$$ +$$ +K \qquad H_k = \{ \sum_i \alpha_i k(x_i, \cdot) , \alpha_i, x_i \} \qquad g \in H_k +$$ +$$ +F_\lambda = \frac{1}{m} \sum_{t=1}^m h_t(g) + \frac{1}{2} +\| g\|^2 \qquad h_t(g) = \left[ 1- y_t \, g(x_t) \, \right]_+ $$ +$$ +g_{T+1} = \frac{1}{\lambda \, T} \sum_{t=1}^T \red{ y_{st} \, k (x_st, \cdot) \, I \{ h_{st}(g_t) > 0 \} } +$$ +where \red{red} part is $v_{st}$ +\section{Stability} +A way to bound the risk of a predictor. \\ Controlling the variance error and leave to the user the job to minimise the bias. +\\ Variance error is due to the fact that the predictor an algorithm generate from the training set will depends strongly on the training set itself. +If we perturb the training set our predictor will change a lot. +\\\\ +Minimisation of training error $ \Rightarrow $ predictor changes if training set if perturbed. $\Rightarrow $ risk of overfitting +\\ +Stability is the opposite since avoid overfitting when we perturbing the training set. +\\ +\begin{itemize} +\item $S$ Training set $(x_t,y_t)...(x_m, y_m)$ +\item loss function $\ell$ +\item distribution $D$ +\end{itemize} +$h: X \rightarrow Y$ $\ell_D(h)$ risk of $h$\\ +$z_t = (x_t,y_t) \, \ell(h_t, z_t) = \ell(h(x_t),y_t)$ +$$ +\hat{\ell}_s(h) = \frac{1}{m} \sum_{t=1}^m \ell(h,z_t) +$$ +Perturbation $z_t' = (x_t', y_t') $ also drawn from $D$ +\\ +$ +S^{(t)} \textbf{is $S$ where $z_t$ is replaced by $z'_t$ } +$ +$h_s= A(S)$ +\\ +A learning algorithm is $\varepsilon$-stable $\qquad$ $(\varepsilon > 0) \qquad h_s^{(t)} = A(S^{(t)})$ +$$ +\ell(h_s^{(t)}, z_t) - \ell(h_s, z_t) +$$ +we expect this subtraction result to be positive. +$$ +\barra{E} \left[ \,\ell(h_s^{(t)}, z_t) - \ell(h_s, z_t) \, \right] \leq \varepsilon \qquad \forall t = 1, ...m +$$ +where $\barra{E}[ \ ] \rightarrow s,z_t'$ +\\ +$z_t$ and $z'_t$ come from $D$ both +\\ +$$ +\barra{E} \left[ \ell(h_s,z'_t) - \ell(h_s^{(t)}, z_t' ) \right] \leq \varepsilon +$$ +\bred{Theorem} +\\ +If $A$ is $\varepsilon$-stable, then $$ +\barra{E} \left[ \ell_D(h_s) - \hat{\ell}_s(h_s) \right] \leq \varepsilon +$$ +\\ +Proof: \ $S \qquad z_t = (x_t, y_t) \qquad s' \qquad z'_t = (x_t',y_t') \qquad D$ +$$ +\barra{E} \left[ \hat{\ell}_s (h_s) \right] = \barra{E} \left[ \frac{1}{m} \sum_{t=1}^m \ell(h_s,z_t) \right] = \frac{1}{m} \sum_{t=1}^m \barra{E} \left[ \ell(h_s, z_t ) \right] \ = \ \frac{1}{m} \sum_{t=1}^m \barra{E} \left[ \ell(h_s^{(t)}, z_t') \right] +$$ +$$ +\ell_D(h_s) = \barra{E} \left[ \ell(h_s, z'_t) | S \right] = \frac{1}{m} \sum_{t=1}^m \barra{E} \left[ \ell(h_s, z'_t) \right] +$$ +Average with respect to random draw of $S$\\ +$ +\barra{E} \left[ \ell_D (h_s) \right] = \frac{1}{m} \sum_{t=1}^m \barra{E} \left[ \ell(h_s, z'_t)\right] +$ +$$ +\barra{E} \left[ \ell_D(h_s) - \hat{\ell}_s (h_s) \right] = \frac{1}{m} \sum_{t=1}^m \barra{E} \left[ \ell(h_s, z'_t) - \ell(h_s^{(t)}, z'_t \right] \leq \varepsilon +$$ +A stable algorithm is not overfitting (but they still underfit!). +\\So if an $ERM$ algorithm is $\varepsilon$-stable, it would be pretty good. +\\\\ +\bred{Theorem} +\\ +If \textbf{$A$} is $\varepsilon$-stable and it approximates $ERM$ in a class $H$: +$$ +\hat{\ell}_s \leq \min_{h \in H} \hat{\ell}_s (h) + \gamma \qquad \forall s, \ h_s = A(S) +$$ +for some $\gamma > 0$, then: +$$ +\barra{E} \left[ \ell_D (h_s) \right] \leq \min_{h \in H} \ell_D(h) + \varepsilon + \gamma +$$ +\bred{Proof} +\\ +$$ +\barra{E} \left[ \ell_D(h_s) \right] = \barra{E} \left[ \ell_D (h_s) - \hat{\ell}_s(h_s) \right] + \barra{E} \left[ \hat{\ell}_s(h_s) - \hat{\ell}_s (h^*) \right] + \barra{E} \left[ \ell_s (h^*) \right] +$$ +$$ +h^* = arg \min_{h \in H} \ \ell_D (h) +$$ +$$ +\barra{E} \left[ \hat{\ell}(h^*) \right] = \ell_D(h^*) \ \longrightarrow \ \barra{E} \left[ \frac{1}{m} \sum_t \ell(h^*, z_t) \right] = \frac{1}{m} \sum_t \red{\barra{E} \left[ \ell(h^*, z_t ) \right] } +$$ +where \bred{red} is $ \ell_D(h^*)$ +\\\\ +$ +\ell(\cdot , z) +$ is a convex function $\ell(w,z)$ \\ $\exists L > 0 \qquad | \ell(w,z)- \ell(z,z) | \leq L \| w-w'\|$ +\\ +$z = (x,y)$ +\\ +In the case of SVM, $\ell(w,z) = \left[ \- y \, w^T \, x \right]_+$ +$ \exists L > 0 \quad \forall z \ \forall w, w'$ +$$ +| \ell(w,z) - \ell(w',z) | \leq L \| w-w'\| +$$ +where $ell$ is \textbf{Lipschitz} +\\\\ +\bred{Theorem} +\\ +Let $\ell$ be convex, Lipschitz and differentiable. +\\ +Consider $A$ \qquad $ A(S) = w_s$ where +\\ +$$ +w_s = arg \min_{w \in \barra{R}^d} \left( \hat{\ell}_s (w) + \frac{\lambda}{2}\| w\|^2 \right) +$$ +If $\ell$ is hinge loss, then $A$ is $SVM$.\\ +then $A$ is $\frac{(2 \, L)^2}{\lambda \, m}$-stable \qquad $\forall \lambda > 0 $ +\\\\ +\bred{Proof} +\\ +Fix $\lambda > 0$ \qquad $F_s (w) = \hat{\ell}_s(w) + \frac{\lambda}{2} \| w \|^2$ +$$ +w_s = arg \min_{w \in \barra{R}^d} F_s (w) \qquad w_s^{(t)} = arg \min_{w \in \barra{R}^d } F_s^{(t)}(w) +$$ +$$ +\ell(w_s,z'_t) - \ell(w_s^{(t)},z'_t) \leq \varepsilon \qquad \forall s, z'_t \ \forall t +$$ +Use Lipschtiz +$$ +| \ell(w_s, z'_t) - \ell(w_s^{(t)}, z'_t) | \leq L \| w_s-w_s^{(t)} \| +$$ +$w = w_s, \ w' = w_s^{(t)}$ +$$ +F_s(w') - F_s(w) = \hat{\ell}(w') - \hat{\ell}(w) + \frac{1}{2} \| w' \|^2 - \frac{\lambda}{2} \|w \|^2 \ = +$$ +$$ +\ = \ + \hat{\ell}_s^{(t)}(w') - \hat{\ell}_s^{(t)} + \frac{1}{m} \left( \ell(w', z_t) - \ell(w,z_t) \right) - \frac{1}{m} \left( \ell(w', z'_t ) - \ell(w, z'_t) \right) + \frac{\lambda}{2} ( \| w'\|^2 - \| w \|^2 ) \ = +$$ +$$ += \ \red{F_s^{(t)} (w') - F_S^{(t)}(w)} + \frac{1}{m} (\ell(w', z_t) - \ell (w,z_t) ) - \frac{1}{m}( \ell(w',z'_t)-\ell(w,z'_t)) \ \leq +$$ +where \bred{red} is $\leq 0$ +$$ +\ \leq \ |\frac{1}{m}\ell(w',z_t) - \ell(w, z_t) | + \frac{1}{m} | \ell(w',z_t') - \ell(w, z'_t) | \leq \ +$$ + ----- MANCAAAAAAA ---- +$$ +F_s(w) - F_s(w') \leq \frac{2 \, L}{m} \| w- w'\| +$$ +$F_s$ is $\lambda$-SC \qquad $F_s(w') \geq F_s(w) + \nabla F_s(w)^T (w'-w)+\frac{\lambda}{2}\|w- w'\|^2$ +Since $w$ is minimiser of $F_s$ the gradiant $ \nabla F_s(w)^T = 0$ +Therefore: +$$ +F_s(w') - F_s(w) \geq \frac{1}{2} + \| w- w'\|^2 $$ +$$ +\frac{\lambda}{2} \| w- w'\|^2 \geq \frac{2 \, L}{m} \|w-w'\| \Rightarrow \| w-w' \| \leq \frac{4 \, L}{\lambda \, m} +$$ +$$ +\ell(w_s, z'_t) - \ell(w_s^{(t)}, z'_t) \leq \frac{4 \, L^2}{\lambda \, m} +$$ +We now know the stability of the SVM. +\begin{figure}[h] + \centering + \includegraphics[width=0.8\linewidth]{../img/lez21-img1.JPG} + \caption{} + %\label{fig:} +\end{figure} +\end{document} diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux index 2322218f9..ffe986617 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.aux @@ -211,5 +211,11 @@ \@writefile{lof}{\contentsline {figure}{\numberline {20.2}{\ignorespaces }}{123}\protected@file@percent } \@writefile{lof}{\contentsline {figure}{\numberline {20.3}{\ignorespaces }}{123}\protected@file@percent } \@writefile{toc}{\contentsline {section}{\numberline {20.2}Pegasos: OGD to solve SVM}{124}\protected@file@percent } +\@writefile{toc}{\contentsline {chapter}{\numberline {21}Lecture 21 - 25-05-2020}{126}\protected@file@percent } +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {21.1}Pegasos in Kernel space}{126}\protected@file@percent } +\@writefile{toc}{\contentsline {section}{\numberline {21.2}Stability}{126}\protected@file@percent } \bibstyle{abbrv} \bibdata{main} +\@writefile{lof}{\contentsline {figure}{\numberline {21.1}{\ignorespaces }}{130}\protected@file@percent } diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof index fa2ffdd3e..db5da12f7 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.lof @@ -98,3 +98,5 @@ \contentsline {figure}{\numberline {20.1}{\ignorespaces }}{122}% \contentsline {figure}{\numberline {20.2}{\ignorespaces }}{123}% \contentsline {figure}{\numberline {20.3}{\ignorespaces }}{123}% +\addvspace {10\p@ } +\contentsline {figure}{\numberline {21.1}{\ignorespaces }}{130}% diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log index 52869504c..b8d0b69e3 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 19 MAY 2020 10:25 +This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.4.13) 25 MAY 2020 10:28 entering extended mode **./main.tex (main.tex @@ -3092,18 +3092,100 @@ Underfull \hbox (badness 10000) in paragraph at lines 126--139 [] [123 <./lectures/../img/lez20-img2.JPG> <./lectures/../img/lez20-img3.JPG>] -[124]) [125] (main.bbl +[124]) [125] (lectures/lecture21.tex +Chapter 21. + +Overfull \hbox (20.70583pt too wide) detected at line 11 +\OML/cmm/m/it/12 w[] \OT1/cmr/m/n/12 = [] [] \OML/cmm/m/it/12 y[] x[] I\OMS/cms +y/m/n/12 f\OML/cmm/m/it/12 h[]\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 w[]\OT1/cmr/m/n +/12 ) \OML/cmm/m/it/12 > \OT1/cmr/m/n/12 0\OMS/cmsy/m/n/12 g \OML/cmm/m/it/12 s +[]; :::; s[] \OT1/cmr/m/n/12 (\OML/cmm/m/it/12 realised draws in training\OT1/c +mr/m/n/12 ) + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 23--31 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 23--31 + + [] + +[126 + +] +Underfull \hbox (badness 10000) in paragraph at lines 55--60 + + [] + + +Overfull \hbox (0.64156pt too wide) detected at line 72 +\U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = \U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = [] [] + \U/msb/m/n/12 E [] \OT1/cmr/m/n/12 = [] [] \U/msb/m/n/12 E [] + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 82--89 + + [] + +[127] +Underfull \hbox (badness 10000) in paragraph at lines 95--98 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 106--117 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 119--128 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 119--128 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 130--137 + + [] + +[128] +Overfull \hbox (42.9944pt too wide) detected at line 154 + \OT1/cmr/m/n/12 = [](\OML/cmm/m/it/12 w[]\OT1/cmr/m/n/12 ) \OMS/cmsy/m/n/12 + [] \OT1/cmr/m/n/12 + [] [] \OMS/cmsy/m/n/12 [] [] \OT1/cmr/m/n/12 + [](\OMS/ +cmsy/m/n/12 k\OML/cmm/m/it/12 w[]\OMS/cmsy/m/n/12 k[] k\OML/cmm/m/it/12 w\OMS +/cmsy/m/n/12 k[]\OT1/cmr/m/n/12 ) = + [] + + +File: lectures/../img/lez21-img1.JPG Graphic file (type jpg) + +Package pdftex.def Info: lectures/../img/lez21-img1.JPG used on input line 181 +. +(pdftex.def) Requested size: 312.00119pt x 174.75166pt. +) + +LaTeX Warning: `h' float specifier changed to `ht'. + +[129] (main.bbl [130 <./lectures/../img/lez21-img1.JPG>] LaTeX Warning: Empty `thebibliography' environment on input line 3. -) [126 +) [131 ] (main.aux) ) Here is how much of TeX's memory you used: - 5714 strings out of 480934 - 87854 string characters out of 2909670 + 5724 strings out of 480934 + 88274 string characters out of 2909670 336249 words of memory out of 3000000 - 21301 multiletter control sequences out of 15000+200000 + 21307 multiletter control sequences out of 15000+200000 561784 words of font info for 96 fonts, out of 3000000 for 9000 1141 hyphenation exceptions out of 8191 34i,13n,42p,348b,361s stack positions out of 5000i,500n,10000p,200000b,50000s @@ -3135,9 +3217,9 @@ s/cm/cmr8.pfb> -Output written on main.pdf (127 pages, 2680957 bytes). +Output written on main.pdf (132 pages, 2720089 bytes). PDF statistics: - 1029 PDF objects out of 1200 (max. 8388607) + 1045 PDF objects out of 1200 (max. 8388607) 0 named destinations out of 1000 (max. 500000) - 406 words of extra memory for PDF output out of 10000 (max. 10000000) + 411 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf index 43d1e65f8..eea8ccd0f 100644 Binary files a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.pdf differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz index 3c288a9dd..1228bc0ab 100644 Binary files a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz and b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.synctex.gz differ diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex index a014231f9..5ef2991cd 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.tex @@ -149,6 +149,10 @@ \newpage \subfile{lectures/lecture20} \newpage + +\newpage +\subfile{lectures/lecture21} +\newpage \bibliographystyle{abbrv} \bibliography{main} diff --git a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc index 5b8e49510..c5085731d 100644 --- a/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc +++ b/1year/3trimester/Machine Learning, Statistical Learning, Deep Learning and Artificial Intelligence/Machine Learning/main.toc @@ -90,3 +90,6 @@ \contentsline {subsection}{\numberline {20.1.1}Fritz John Optimality Conditions}{121}% \contentsline {subsection}{\numberline {20.1.2}Non-separable case}{122}% \contentsline {section}{\numberline {20.2}Pegasos: OGD to solve SVM}{124}% +\contentsline {chapter}{\numberline {21}Lecture 21 - 25-05-2020}{126}% +\contentsline {section}{\numberline {21.1}Pegasos in Kernel space}{126}% +\contentsline {section}{\numberline {21.2}Stability}{126}%