From 831312ce604b07a73dee7547256039f57eb9f5cf Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Sun, 7 Sep 2025 15:58:51 +0200 Subject: [PATCH] update book --- .../_build/.doctrees/environment.pickle | Bin 273661 -> 304160 bytes .../_build/.doctrees/exercisesweek37.doctree | Bin 36790 -> 37036 bytes .../_build/.doctrees/exercisesweek38.doctree | Bin 0 -> 30573 bytes .../_build/.doctrees/intro.doctree | Bin 45816 -> 45859 bytes .../_build/.doctrees/week37.doctree | Bin 0 -> 271058 bytes ...-main.c949a650a448cc0ae9fd3441c0e17fb0.css | 1 + ...ables.06eb56fa6e07937060861dad626602ad.css | 7 + .../html/_sources/exercisesweek37.ipynb | 71 +- .../html/_sources/exercisesweek38.ipynb | 429 ++ .../_build/html/_sources/week37.ipynb | 3736 +++++++++++++++++ doc/LectureNotes/_build/html/chapter1.html | 2 + doc/LectureNotes/_build/html/chapter10.html | 2 + doc/LectureNotes/_build/html/chapter11.html | 2 + doc/LectureNotes/_build/html/chapter12.html | 2 + doc/LectureNotes/_build/html/chapter13.html | 2 + doc/LectureNotes/_build/html/chapter2.html | 2 + doc/LectureNotes/_build/html/chapter3.html | 2 + doc/LectureNotes/_build/html/chapter4.html | 2 + doc/LectureNotes/_build/html/chapter5.html | 2 + doc/LectureNotes/_build/html/chapter6.html | 2 + doc/LectureNotes/_build/html/chapter7.html | 2 + doc/LectureNotes/_build/html/chapter8.html | 2 + doc/LectureNotes/_build/html/chapter9.html | 2 + .../_build/html/chapteroptimization.html | 2 + doc/LectureNotes/_build/html/clustering.html | 2 + .../_build/html/exercisesweek34.html | 2 + .../_build/html/exercisesweek35.html | 2 + .../_build/html/exercisesweek36.html | 2 + .../_build/html/exercisesweek37.html | 31 +- .../_build/html/exercisesweek38.html | 690 +++ doc/LectureNotes/_build/html/genindex.html | 2 + doc/LectureNotes/_build/html/intro.html | 2 + doc/LectureNotes/_build/html/linalg.html | 2 + doc/LectureNotes/_build/html/objects.inv | Bin 997 -> 1030 bytes doc/LectureNotes/_build/html/project1.html | 6 +- doc/LectureNotes/_build/html/schedule.html | 2 + doc/LectureNotes/_build/html/search.html | 2 + doc/LectureNotes/_build/html/searchindex.js | 2 +- doc/LectureNotes/_build/html/statistics.html | 2 + doc/LectureNotes/_build/html/teachers.html | 2 + doc/LectureNotes/_build/html/textbooks.html | 2 + doc/LectureNotes/_build/html/week34.html | 2 + doc/LectureNotes/_build/html/week35.html | 2 + doc/LectureNotes/_build/html/week36.html | 2 + doc/LectureNotes/_build/html/week37.html | 2756 ++++++++++++ .../jupyter_execute/exercisesweek37.ipynb | 71 +- .../jupyter_execute/exercisesweek38.ipynb | 429 ++ .../_build/jupyter_execute/week37.ipynb | 3736 +++++++++++++++++ doc/LectureNotes/week37.ipynb | 3736 +++++++++++++++++ 49 files changed, 15660 insertions(+), 99 deletions(-) create mode 100644 doc/LectureNotes/_build/.doctrees/exercisesweek38.doctree create mode 100644 doc/LectureNotes/_build/.doctrees/week37.doctree create mode 100644 doc/LectureNotes/_build/html/_panels_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css create mode 100644 doc/LectureNotes/_build/html/_panels_static/panels-variables.06eb56fa6e07937060861dad626602ad.css create mode 100644 doc/LectureNotes/_build/html/_sources/exercisesweek38.ipynb create mode 100644 doc/LectureNotes/_build/html/_sources/week37.ipynb create mode 100644 doc/LectureNotes/_build/html/exercisesweek38.html create mode 100644 doc/LectureNotes/_build/html/week37.html create mode 100644 doc/LectureNotes/_build/jupyter_execute/exercisesweek38.ipynb create mode 100644 doc/LectureNotes/_build/jupyter_execute/week37.ipynb create mode 100644 doc/LectureNotes/week37.ipynb diff --git a/doc/LectureNotes/_build/.doctrees/environment.pickle b/doc/LectureNotes/_build/.doctrees/environment.pickle index 98ef94a83541a27eae1bf14ac1d65c5f561f52b7..2159d6ab19dc4474f58d8079db861138a4ee0473 100644 GIT binary patch literal 304160 zcmd3P378y7aqjAz(MWUYmi8*CccmF=dPcXj63g!BTS>86VGn27|%IHiz-!UfB0XWL9NXcg<*( zJteGfx2GzzDl#e}BQi2FGV&!0UO(@KIXBS1`cki)b&D4Vo#J`7QYsdlVs)zCv%lu% zGl#UI*G=_5F?DIGuRhN%m#0qDJ2Itowc@VK-G@G$@hqrm9n?QQ=k6gim%+U$!H=IJ+dOu@ z(M+er*1unPKvV>uxWIF&HU3EFVijn*rJ^_W2tHw%cr#TkO-$sSv0AlS!WZ<)(sZdj z>6f(?CcSE^2qG-W)yk7qr;-{gmCoVT2a$Eg9DH){ZhlDz)YT1aygsbv))wwuTUn8puwS3h@OQ~Y5Fy>T< z_)fh&Zx<(O_5?u8B~~rYm(q6LNrA>hk9wQ9_oi%oN2yjxJ1M7_E@hZ?i)z*JJ-X^fybSNX?uQ%`^J#pGlmLo(H$>UCYNlXzRGhqHd(KpLe|fartEyabRjif zd+f2vRJCFkJ#;gc-^=xzwEGm0pQt#VmrCbt&x2aIT<<8@9tkSzoCT1wN{Qr4bZoCS zKJH#*4LE(xq)HB*M#o~q9qEI=)>>?cX~7Y@4RNnB)~SF86p7t{G##-Sch zk}WjK@=cq>GpTgeNuMj1&@7a|qQQfsP@bgC}7NVpF5_xtlgYVwYr`aT`*fs+95$Q611gw_=(g0?7)bBkK@l zo~-v!U7?hzLHLOgRJN3~MQMz>5Tw3Zak=!?=F*gx1xExx(%zX;!FJIZZpJ%>)HwKw zF6X2w&bWg%p+h_fv;%%D;TJ_OB>r@jCns1*ro8GTaE89;p=+wXG>h(+r!E+D8n8ir zAQZH!E|o=ZK$q$El~8J|;%1=aPSiWkIZiosL5x1sWUp5%m${n+h0uFTNV#L|q3yQNGP8raai zQsa4hf~eh92C?kQBs!;{-rVvex}eaj=zIPsh3{HYu#08VhpAd65Af|pw_0}7=O&4S z@;ZgFf;ty)SspF|A0{v`pkg700va!$tS=K+DbGpc3PVj9L;A>^dZ#RhmM+*;mW?(% zLlnk0j*2Vmc4E2^X)xumJ2A*b3##@QF|qv&{pF$#j2%vL45JDL%8p~MSH(y|HMRQ9 z1^c2~K;tyZ*<=i)@@^47PH_UroTzsLgRC6ANC^=0xX5`FWJfn+6i?I_`_G9_3vNB9 zJXN1Fh<2-aF9nU|j!)v#7t>uD{jmbE1gmPWUs510j3mJFz>?~tH8x8i5!!9amSmXs z=s|qsa=l6_0*?u{$k${%qR!3P=uhoz-;kk1M#ntx6PlGZi~~xDE}X&$nzcRGgF?vz zzicU=p*J8lPPsNlU%)!|$ldL1=iN~6;Bsja%%>X&jw(y(qQS*(2A}DUyAH;Hwp#IA zv2+25Eao~7>Iw=u`|f%>-`Cs>-H0~QCA2=(FYhK$=isi3AA}IFKIhNv7DXlM=+N<8 z)|IUm@~Jd5b5JC(^z~;@+{tf4;m8g-PvS-;FyEf62M>4{kASRO7Y#xRQ;;Tz2*|aZ zp`d$-qSOU9L%sV3i7$&b1c^GaTckcWRTn6tE2LPy(NYfsfen_fxU6A_7@`nXCL|hm zA%;>E_VSY#Dp1_T3DT*6*MVWyeGJdpdAgzR7i`LIs&}hf-&TUU3m(z9)(0aDhI2Zb z8nY`YF(54)!@lLqM$ zLxI-N7|ero7JL=s{uCtVB1{AxlF)zgjXl{?fd=de;xrrLPu7>%#bOEML-&UgAlt=; zl|ESof-KO>CQ2nt4pKpBibP|7eJ#Ya=DE;yT9wr1s9qIS&7@f8lOlM6D3F7>%bjg7 zNBVIE^+x^M#M}hkn<_9%tBTK%^AjjX?!KN^!#EKXcDU%Rq*%|fD9*=-RKiEq{7&gs zNn>vsE|>GJld%rM_OyoS>!F7%yO^;gUo16)6Wv)1*eB|XxaTrh%q{?ZuuK!bC0smi zpQrYTSe;yu({w-&mv~iJb7@}?`s$CgZbHjRgFdAjf`LKpOw}KrHTY_BsG#3JYtUKn z6yS$u4KSH5=;xk--9!?9{(fyj>CaijywxYTBlVpSMA8yuiL4O42G)71R>p+KfuYJX z1<_4Bj4x1mB!2Qy-!R7PnJpozS;jzwz>(f|(@_6U%J#|*%mdNypR6zECsk33du4r2 z0tT1ph3NiOsrxU>GU9Ja%oPAtfZ(x1lo4!{uN3d|l!A&@LOXo=yNjd`i34J+_d?w7u@#vrM~_bJ$&|&vu7U} zni#M)o;`cphK>FEtf3+6whdOn!E6^Jy<=T~G3Zz@bYY}rB(RQ49UViPHg#B=tkX0Q~fcv*b2XFY!Kp0HHt#GA#9RcZ=~E@Qbxt5mC&YgLVnJW)FULmhn#03QNTaxRv^({VfRIRh5y8iF96I!20y7!LjpNXCd7RZ4_8M5-nHFzFO( z-B&3Gpiuy6{rUlmbjZO{`XFf#F7i9Tc|5{%=(6Aw-Loj}>tK$7ga=F31;=8Cj|Em+ z)C@jOwSzjTx`bJ^{+>gp4<(_oYCIUPzl$LeWRQ_iCM1Eyx^+<)>kr=}(B^0LPY7^J zR~!tm@(V~-p{r=R4FyN4pRGLfM0h@EZcfcHc#;I45k4iTrwdA;xuSCciicI+#uxN$ zv^Ga_6U7_*`vy1lZ|rBV&?VrgRE}4H7gDRl(8jLP-vnKJH7F0gAUYxule)#jk0|bm zFFy>UKZ6?XW2$nZRMi;JT&W1I)rl5~ct=WA(HFHgsY~>!M|>e79zzTUJ+eOfHl8hR z?AIyHeGE_t>LB0r3E~S%67h&h;;|RK$kGuPw-QwyD7_W_d;xQioKt<&l!8vrhCXSq zT=TMh+UwzG2^*o)1O+^6!v?*vMmz|<#v}oqKj-0XkAy)tbsKt@{x!Yd{0Ty@ipQ-G zl_YJQV#a#_gAso|F-6x7{niFuCb<*)u;_!CJJKoZ_308Q-}IRfdXpf4U#c&yQ+L6G z@}sGhbrBk{k(^f{g$AubIf6_4738Pv>$mQ_^G<8y#%qNxq&v_$N69!~u$~;~F$9U8 z162v_GtSb7fvITaYLHgwJqu2g2^X_Z@g76;NgRmPpobZJ-+3BD@dH$Ol-IGP@kxpn zq(`Sp4$gQXXa0!GZ9sva=Ajq^D%=Gwh%rIxjfg@D(>`BvGZx&|rDUa6WHrgdF9uig z`2=`s4P)_D_Nnz}i|a#Ttvv-X5ok}s7=WiwX~dK?S>Sse zB&3TaIJXuH=Sm8Mg>gHdW&x^@91HMF>5h~x`MmFE%>Grq!v{eQoI}3Xq9q3oP$WB@ zCxY4iH94;mOw!gG7w}pmtpm_piCTo+k6D5Tr|pw+PAHt$SljfcQT4fl7Cf_Jek2@c zzAqNOT`?V#Qzp$dDw^=8QfWBRdGa79cCiytk>?QiaP!gwIk+vEIQ4x2y)#9)glJ}SzA{*J zszb$cVMsoEH6MI&5edt9qW)p3e+2cx>ln$%*DhW3cw;+F)2Ilu@;0r(9tb>)M(-YAb%0jors`XO z>NfsJQS~ApC;2Gfzex56ELqu8o>%Yo*8*in026xrB3iNL^$9S>FRPLB`f`1#e@~(- zLkym*uMJA&VjHc<^Gt|cWUKsY;EQ>&TyQKwgw`Cs@7OWx#PIP$)^iRWA3l2Q?%{(6 zPahf`eJ;Ed>MgOrjv2LGKxaHQeB#J`!$^%6aF!lc(h$A{}%Xn7Le3Hv3aVIr|~d;labXMd*%YS~0m~uqMMxdZHL}AU15N zN3>%mZ5L>1=)pH4M1|EQjeoaWqo6zNv~dg!;t2~Ye4q%=BZM0-sf9Qs8MbwM0Kqzo zFO|Up40kPZPa6VOW8$In8roeSh#!=&l)vycZe?L1mrz4W*ojl4^$ zp|e>mDpoiwGNqh%EsyAzWd7hcqp8jawWqrNZi{$1m2^!Aifd zNz?TDHJYwz_rnRGWBuwPmYJL?>@=vD8d_iNSEMT?tTG012Sk2?S@m%yrGTE=w&~8~ zi`Q>{Ar`^7R+@UKnGdai7^-DlOd%41jlpJz4-2d0O=d^OMHnj3W7R4-V!P8=BuXJJ zgcl0YmC*f`lGy__(NNrzhWSRTROg&YbfyeF+gERc^BU{oge_l{#(^D#4Te}7je>cK z^;$$;O&HAJ0fGWxxsb1sB3#6x3i*C22x3`M%|gFsQXJd`;k-2sPR_jezA9ZV3x7Bn zU*TAXUao8Wy5&?F^?mE8;%e>n&_l4FY7@Y+bPkKXm*J%i%D6*Na3YHPa`t&!tdw09 z>>6@lmbDNA@|_`o$jbuIIuUh71~R1N(HiKCrSS}hd&r=e%k|sjLs$EN_3}AsJCW=_ zwlJ@EDr8^C2$E%xkGzw0@@1GK#SB&eV5nf|RmB%r(jWl->jlxfJjrpY08v^(zqgr2vDN6iPz!;YUuEV1Qr^ zlo25ick#x zvcb)RqGG@rv@?(t4&WNhR7&LxhgbB5-a}k0B(n1WIcjNcNj)3O3{B6ZvJ_*XJt*$i zh90DKr4+{@5!5GhoiMMm`*U6Rb2-%EbTGmQup|Oo7nA2If{7OJj0^~f$0-U) zjA0Z=BpT;$qLEaLj#yfldKCnK!OKyDdU}c{iaaU^in6&$@e{%zp)Stv&@If-KqwK# zA}H%uczLKyS~39&UE&ppMyi1Ek!=KUSXjmo!INsVBPc5GgDBD50qXIg?#ia?NFo&onlQ3^k2TQVud0A!RB8cR|!@!L#MU8VF!w9J_8XaitU;ToI?c`&_$>*mY zkr!K%ccdQKbQ>&Ldj6u5X3>!mN%KnMuprjR*_5nC?5B(o@}_b>Ap}>2UPYK(mcv_l z-MZ6%nTzEMH-%o1mO8WrDQxcFYZNApwW3L)mwLvxXvxTKE{%GFN2UT94B88Rp?G0U zs0m>gJD94#7`$9xshSDZh*xh6y$a_bE$0_g{5DBxs9KmQq);?e{6Yb*2QL&Hth-_o zMU!5_fXZaAC_|hPN{hfY>JGx04hsw#!{ZPG!ndJ9_)7e)3d0EMG^Glc;c*5aoUd%u zqqB`-+`)nkRwJpeT91>30)5f=v>0_(+=Ia=;%P#~w}nMeHN>5zW-V$8m@mO1b}{$B z3O_{WfIYl2{FQg`CJom+@?f%Dxb~vXpM{B#DKY4U$=kt$v4;9BY7q;A+KUo+# zm8EU>=l7+J%gd^lDOwzR$*gJ(+=feEhhM!ukP8v_L z!S6s)-*8x*<`WWrE*eG1jE~s#0=FP6yfKa>5iZKU`eHkO!JhOm)v(D=*shGZC4>C% zz#vHlOl!Xeg9{e5u^)h=LJ*cItUoDd=#+z^Kf?%Q-7oGy=x*vXU9eWyyn^*nL_A}E~W zL}m0`0aiE_Lno}#%f*nu-Uo7dgx>LW_lf$=`YX~`;T$HO(bG2x9O{=qjt0dE_$3#f*#==Cdw-lcHV`g}GV zG5X2(eCHh*T$r+ANQd#nVZ5OzWyJ~Fl>|!yHsC5rADxzib{Tr9Z4?h3$1`fznTUpk zFlR?{52I(-+Yd??mU^fSM4)kfDm#LXSR12RE}Ng?jkYtT^ckA?DyV1%!v>rL66Rq< zk{v}E9svZ5*tDx->c34rF_kNGqi{u|6oZ{ATSVHGe0QP{n};M#pxK7U(0keMnTArw z#-92@3@-FZEW%(KMH9|Z?0LiM*55k7wnJdik>Lf^+!S*nk zkj@IZ?$BBk*mV0nYs>Ui&Al#otGU|+ReHaZU$^2Q!uW&8ZlCI{ecB2##=us`qdjcf zoBQ;VY04X3)O)iajd-!Syy3;>E+}4XE^l~I@1c5&&E*X*>U~O5_O=J`@%c##6GP9k z^8*NmK!6P^qJTirgua}+iVhOEVfz?w_S8FrE!1QCDN!|zOgA*(VACe=0yt3{(}s^+ z8ahyF>~UP?zDYk5_<3Y1_YM5v$Agij>GA)Jcw8HFnx6k&#PiLAL(^+Nig+zNZZy5} zOvEd`4)v+o^wPX}&0p`oU1UfUS)nokN}UpKw9IpU>2 z^ZKg1>8+ue-kP>6ZlCF`Y5U^tnckYVGwzFct3mrs+b8decsH#4n+S3s;+2NxrtOVK zBHj%T8BNWNM7+|_+_e4pWW>ASfu^as(TG1LlZtq^ zal~o*zI5|TwfX|=MlV&Tav5k{-1vU!C1U@J@Tg+$lp9OZY-EbYAbNpjPBbIx785<( z1uOrUOx0J&(T^rh7+K*#Mf~c;sqDW@liDEnJUBP2v=LWx_Vi`}IhfetQ$?TwiIh5VvUmngL@h|VW zTyK*xL)7}w%YMbZm@A4#a(Ofx++~ktkNeMT%Wn5CN3&=A%ZIaP{mT~wh@_tM%U&2< z*}>IVHtknu0t}qsYCJpPe|t9T`cLMv=lsi^*w~MG4{G0#eWU-%FJ#~3U%uUMMs8uI z=s}+K*L$V4rn)kADB)XwIoRi&eG0el$Uen?eT_P70KJ}@%RzxjaOV=ax zBLN+w>*HF%3A#RsAEsFW&E<+Tst^TFqnPz-Zj_!n<3DvDE?BMlqc6!9!n-mEY~mO1 z4_=i-?u$+C(3S9f+NbZ$X@hYwM z)q3q~sN%I+?d#}zN-KChUH`mR@K#(0%H)@*;%!>(+v)n3wSsrh^;25Gui!d(|DCuI zANa1|>fQPS@1bJ=47MZ1C|~d|vRBNb>|=nIzxu;xlp|q#s%EdvyK#TET~K?SFN>2XTNROmHSckuqZO zV-$VU8^pvZ<8nDTLHWFtO5u4GY5)I)=7Qnr544+lw~T;OU?OZ(f5`l>_Ga!6ag+N9 z{m23Jqjdc-t>EKy{Ryq$lXU$lt>BO7`qNs$XK)=1^qsg-wo4c8g3-KNf38Qr>D6zR z;zmJO7P?y=x?6!e5g8}HCJ2!8Kktu2&;JST0%`rMe)FdU>Cfm#YOT-H^U4Ka{_%f~o$p3_!z>3a&1$P0YuhKjJOg|FG zf5CO2NWX@gkw5aQ{*@m4H|^Pfr|WNO1>eGT@bZ7?H{aH8{!_pCf4B)S{f=JqUEFY- zmXE;q^hdu>wLhRAiTMxdn&Peef~Ok`eoO^F(VlpQu5X|@ii9zTuIFk6^XPiMRNm?8 zA6>53uE0$&7t5{Gt5)eZtM!{TxXG=hANi#>()CSR!Ogf1KIInN^!tzAN>5qZqwDB; zy;hK*>)W(~+v$3PR&)DPf}jK6q2KJKNB7Z>R0{{_ z`k+>Dgs$(QANl=9aUB4=mnufI+GDtuw&ihL3X}WUp>hNOqbkMzz{AxDHyl zk1FohY9GM0FX%P6*gC~_Jp^AN@g8q;K}HnXz$Z&6au#dk$ds}WOZt?*@PZs=+;YvMS>sd5CUKRGkR&@KfghT z5O~JFr#~C4|Mb$cGcB+!r3WHbo90q^gBL#dhItW{!KbuER0dzPETS^_udg#7k1$91^HM#%4Ltq#C@kadeVtO!Q)%M5ys4v_6=y$1TNf3p^ zWnJ8MC_W&!K$QigXjf$c1%ATtgnp2r6S*aL4rzMHQ9#{VM0;VE0lIgNdUI|i{ZI$a zt)d_5v$@rDMcp*FhJL7L=GM{`^~T(dbVWTdcN6_kugl#`Kh)E5x6l>!uH3D3MLjBK z(GT^a+&a3Vo|9WoKh#BX3HqTZUg-gZ*v>zhZvdbqbuTCuAhF0 zU%5^6LtM&jrXS)=Zh)?cA-N>|5DRjHbVZcU4bc_RJGX^?h|0OG^h31GZKEHeXl^@Q z5gl_o@Pn}**YGj_Dl-y6FI|%(#L`F^rJ0|D&adWEDC7p~w8r1m- z>jP-WM?bO%@qc2x@r#9v7thNS=K(CWA&k(F>DGf<3izKInOk4c08T^;PmRoPYM+up z>H@h73w{>z_leBSh<#fcDo5r5Bof1FkH1v}5tRryty1O) z5$M}R+k9!eAYw}?op60FHigwYMc^`pIO0i;VMO);q#&h`Ta44lmki%Mh;#q?xJ=x=wYR0YJh;W==q&_d<}mICfSgcgSw9ci{z|g{Lmk5+D(cwv|%& zBhAf&(q4bFDFSZCc~=h}?a10la?Lx~DFk&TZvjKzW<-FZL!lkob3}Vxm+3irg+qYi zCfO>>&dX-7ZOLC5;ShOv3$X17@!i#wp8~j6Z>Rl4K$m&NYU4`V#kN;o*!A3ntU9i= zeeSET;yY3wSK0>t->dnKRLGUK?$5u5??{bYX>0%bYx#~;$(6RR|HA9}j?~GOw$-&a z@Exg?D{X(i=#6|wYUN7XZJ+xEz9ZFgrET3y-^6#MUaqwL#(_8U9jTZrZO>czKlqN+ z%$2qU-w=1CYOb`+x&4>8j?~SSw!eE!+>y$;()L$B6nCU{uC&+ zNc~)C`|ZCKccg-V_3F3tlca{Ow0$ot?no6~X}fXvFLNEKqbqICdHfxGM=I$`+xl0C zJ5ozm+P?q$PjMZorYmh9`@Xm%^>n4}q4mGQb)=%MwEbS{-F!!C>Pp);fAC(uBUN>! z?N=83I^U7Hy3+PfZ+$=AAvG=HRVnF4F-=>#5fqR85$J2^Ds(4PiX($UmE&hAQ8Ppn zRr5%5y$&(6r!ia*mB?0oSf-)%x5B_O#WovfD7|aj?mfHGTkja#xo1my*RJh*w(Zzs z+uPIQ&hCt3Z{ISO8B1^3V&AcCo3q>5vD3zXJMG=}mYv&CvwaU|O?@@%`Wc%U+r4ww z?yWm_?b$Lu=Ilw2jc?6t*}Y}^?yY0v+wa)9W!D`$Gnw&iJI1!%;iSiQ?B12$v1fej z4rk}iJM8f-dmPyeQE#SZwrb%kq_=L{g{Qad8QYQ`+rDjl*Vwjk(F|lq=0$>Ckw%lbar_WH>z&AeLOYU0 z96oK4TbZJa=9FuNR0HBcDMxqKq4@LFB1H;SG*`C!yR2jr88lI!FWD)0M?*cpEFy0q zBAT&Nl7_Ga!g+Z7mu^FPf8P0qi4eAdUo2Vp_3Kej3Z$|A z6>SPWsg%b0LWijD#QUUV8tXqV>$^}-YNxS&aFKX_3F=7^HP)}y=tHWhvA##r2RV${ zE^4Y@Dd69PdQxAF_4YhbKOglZs>I+9v6;>*J85S466`~uv^Knp4wKVUyj#fHjx=}W zN%3qyhGu_6z=Ta&2=)0qDZ=?0;*=8X>QLhbt)Vb@Z1l)D1($Y}i7j_zWj+O5>TPmk z5+=;jGJu& zj3nD1%qrO!T0j_cvVD98vJKA{WLp5T(Wv=TA=_WgD%lv?ERgN*XCT|x<_NOQ1=+~F zXq4-!Cf8(e{8Y&Ltyv{2Lz@M%es>14o|r4hIuB$e%dC;C|15j!La0l!)f(&HB!{EL zs3+^~r$Y863ujYzGPGGB`%=Pc8r}K-zX%c&wP@aDV76Q#%dgb;uwbzR``@!s8$T@isfqM^L@W$+VhnAHHU7$YC z<&;D6Xb?=19-(9??48F|I}Tn^H$%E%i;uz!wS zYe&$o+(>dSF0n#3fUe!(rLhVVo&v9WGUH?o!4&f$WON1bV`$!cada zza}?|cXDa`;7f96@RvIA{rC%b4~jp}7k|!*KQ9n}UdVq&;A+W@;nGKtJ$^GIxNWqQ zKTr4_bjSHTt19;MSci}S@4>U8FPPwfQur$rbe6D)Oo_PRL_PeD(HPCt(!R(FjCi+SVDO z-Ybl;>}|l3-M2Gix%M>e<`=s~kvprT?VJoD^=4zF<~JZkbGh8iNG;X2w90*vA!Ocd zjLd=tWZ0QKGcwDLl_q#0Z93BYjxjbkoG>Js>`oBnC`vLau+)~9S2yQOI#uHW-N zVT@yY1CH$KpBYCh*yPF^yRU-U|29T#Q3Gl;OTS9bN;mGe-D~lFVcQyoEMgljZOf%C z!^iF7B;SY)jEAr|jO?D4zcY8hIu1WHZy3L~#xG)Hc2rmWN8_*SX!tr_PnntUJNM%( zwX}t3Eu3~B^LU=ZF5k3UhGsl~6nZ!{0{SR8?Wph6f#Lp8NBO2PzKa|1C7;JtxX~>K zJYO+}F!`}DCY=qK@T$|ylp8*&Z8skfi2y(;8jUGg0uhrsMB~X&78b+dg+LzlLJ0T8 zZKeZYTLbRAPBt^{w;c#}bF&`5uZZ&=oyvKa4yOptu+p``8e{a9G@wV0r>iiz_mICk z8cEQdbJq49AyRF;W?aHLlNSwVCQuUwJ3|=kXb}Tmth`1Hc7-q)ZV>}sB)vuqc84%H z(IN)CINK5i54VT`&H1m=6R+jSH(gw^Euu#5hO3~qPtIjzs*VBc9;8%8u+0Q%p5aOb znR{5a3rKisNiXhUhmQ~q5A?Nb#x*DB33D4nn^)9lX6b$z3}tP_X*EsCaR34HHhqL+ zokY@HWRJCW;ph%GQ=~I)`v)wnuY@J>rPqvnSQqfp|I8?N9pL;fbT~AvMu&vp)h#y9 z@dh(D|C~cINjUKp)El$MaP$JQY@EqDNGr@d9`z%)F!&tQ4<`&E`<81))@Ssb21avm z#LQ?fKO@uWFoH)aVtrnhk@sFRI$_pvT*k~uETW0O%HQLgw!hnA%Nz$XbIZdIh=FUb zb(DgJWIhx!6OqsgIpQc|(17L76NIFmqEp*k9HoHNlW;KTwE9Gg7<0VJ%oraQ81Izk z?a2xPDv&bSYI2#-M?A|Z;9sx05!*$UmWb^li$}zEk>w#`yT}3%v0XIair6k1&O~e%jSV8Ui`2A; z?IIm0V!KHFh}bSt2O_qM#7x9?QO^~zUDQWJY!`8$h6AbfC>P;iTs&4ow1hZ$B?$A) zh5dN!iCjb})>5uWLl))kqr;3vJQpIAgs-#Si{yBGunXoDHjbVvXJ>9B2rHWIF?K0V z>kAc*uWlTleQgdoj*sL7)CekC2qbD1cZx%S;7l!2wruu#!m~-{L(ayJROp>(OSIa7 ztJ*dBEG3Bc0K|L-k^2dl6-nUSJUkFplUp=hMb~r{Ovoj=h7YNiLA)0W4eOl=jzta9 zYzc#`Z)D!c9H)t>n}J4)MC~&LF&8fi*51w_rlme&P4Pe^UuGu%ng{5xMx5%{IMco> z#Ea8_9uF_pBrX7hr0XH|C;4ZcWF9H>__!EpKM`N%HuL2qQ{-A4^i7@pm>}x)>+Bnc z;r*ghL9n?G$B|*Q?Wy|_hO@cU&2m~D}0hZK1m!x#yM64d~{Ys z2}!wiW)w_8C?UxSe$KXvTH(uxjpOu689LY1$%p0R4YBcPMIaw`l3Gu4>)7&%0pO7- zmW?Rsm@=iN8VX)x3ZjuA&NVEeNg4eynGL*zBxKY8gu-I>b!VSgSG;}Hr9)D9v${CH z)z5*+Ou})4`aj4S~&bH zZKe9QgvSg@()ZbwB(mB|1_>#`!6#r)GE71xSo4|(osaWJv^3~^M9^WnL1)?_-gJat zcCs>I7v0C$w^gg~KojauKLJAVjP1)42>h3ACoyT~K}AIQ5u60d*i2)0HIIxx#oT8!u{%rO5 zvIN|IICs=dZc7GD>)j&CBt6C$x1_3%nu1UQd|1$JwoTHVhh283W5y7K36Y1JT-kK& zaRng(*sNrosHJH-h1ngY{|~Y2<-jZ+wZAJdF0jL48cED{*h`lzjVEb~w37sON$iyI zoMcg8hnXK^7vgBdjPGTp>`*rC6Q&>?WyL==yd0NllOv`2~;&)=}PRRtHT~Dh=ff12(i%A3vFjDAcnIxSo-(+4a zXdW(p%d`WiKJ*)=AY$jVq|*hCq#GoTbSZI6Y5`VBMp07`3!7Q{H>P}1to^(xh}cQquN< z!JD{DZh48J#@DuHzCLZr6vfvcn1YC%uZx0Uoe-ZA9dQlL9f=^(i9Qo+Z>GBi99p!j zZ#D%HJKeYWbhox*ZBfJ~Shccs-o{yCHO~+EK;C8-J{mq;Da4>7u`y^{z@X&Tq>P$K z*2+nVncS$C3}L1T(4uBW9yH~G5};vI5V13IU%<$1dvO%TInt4nWV_&0HQ%Y8v9Ng& z=Y~^V$l0T-#&fZ8bf4nrHrTp^VvUM(+9o7C3BMT|pAw2sJCXI;;)%#3A#Lg-uPY6@u zkaUihf^w9nDTPM3zmAPhnmp~q0HLsWhN5arLOuu!fA`!quSFP#8<_0G>sdTIwlHlJjEtozN5#@~qE#cps!rCy zAPoG<#^hal5Xqg`Q9}-ZG!C^3CWRr{9*T>k^&1mNb|c&}?6U<5!K_c|-mWNz?Lg}k z?>D8mslRN)p^J1Bq_fu=6Y-v$EZ!YQ94h(8EyQ;qPgHoX%Bk3hzX1avqmvZaB*WMw zeQW~t3O+<};&|(vxYrbfQf)^BGiTd%zWfYCW3)70#VMoF(ov0#K`VG6!xN21%SBW$ z$D^jiRJy2W3Zju82BVc4@PvFCfKXU$n7LONtIpwr$U>Vh(Rsw!B~E#Eco`*AqJoC~ z#9})(?NcWK$MFg*>>&;Nj@bIperQ-9)1(bIT1GCZ5T;3iDgKC^&o~{snoSa>B>VxU zngWPeXL*}xUs2ud7X_E5Gc1tn3@eL~;fjWFj5i$+8P|Bf1VY2TKA)^ll{Y{Ae4N4NN{ep`Ll|bkeEBn<CYjNJ?B$ug4{wQ$OE)JZGsqO4(f=@aBx0rcO{T;5$(62Qe!(0HLtBiu=LTkGSG!0TWhE z0f$U7Fg*&`^BFe4S9Hc@fUitomxO3Nskg56iwLuODe!60?Cv%N5j#IN9y)L55xDA4 zjWn`Q=8a$q8!jbsMNs@+7vV>5`=IF<+kqEPWmnp%OByTqb5j)RT4rg*T zxE8X=Yh9tu-biuXICXj?HWsZ)*xY=wv3X63BFy*ao03y;|Fm)5FA{|J-(#iqnl^nctGMC|mxQ4e;qcq<%5vNdz#so1!2Q$oif*?@z_ zn+S9K?WTlPym_lBh}d~EgKT^{Hg3#MxS6fGb0@+8|D-9&6xBaw3Lt07(vR|u;=INu`y{oFf6mG}(!;+5gp) z1&WLRYziWFE^eSD-2FJ*1*81|GL*$ml>w^=*3pWW)y%uzuDHzS4ZNs3#^@x|jE;!v z1U6#*t%+d~k(^z?uSIiuu_=hy8Mf&#CHAP+I5Z_Jc{7@8J7Z(oX6#W5P-<8;oymlY z2=ClxN_Hh`L#7~N=iJ5zvMv&oP-?aFY&QfU6PA}P)$$qZDAF7_;cOTW#m2Uc$c6!Y zk_w+7iXtiSNxIn8kL>XNHvX=Wh%nFlP3f&zHfjnYc9yLs?RfgovHKLO#_fWe5AOuJ z6dRA$us*Cf`I_R5L_p(8L|o@%rX*EdxnK$+cCIYjZ&x8mei(bR4gR*+7_vMuMr{Y7 z>!NLq7hTOUw!dgfNX3gcn}Ud)7i|X-3sb6vvl)IOHu|?GGO|65=H1OS|EMX!6wNGr??pJIY+2(&Pct9BYDz@KhcBCgh@B7p z!^k#PPG)F(6#Qg#avg?thrF66!l!10_|w@Pm-)O2NyvPR$o=eNM1IYNxsX1p_crsb z130#5LbsWMh@EdckGTbToH1|Y)>6*<3G>9%C*wF}1T2d9zYq_%#>T^4$l?-U$C)02 zg!n-^KQR>|?1Jb-ou5ma`8jCH3?+w~O+hsB(_qm^tviX000tlw7Hy_6WOAPeA3`); zIOep`n8wkxtsZEUo29NwpB=g9XJ|;|ii|TM024S%0!*u0vZLkC35Dw_1(?;m% zsEm`g!}k8=xF}j*8|*ruBSIwx$mc*91Osh%RK)AgwNCGsnu1WG^%B9d*^YQ!bK0SG zT!dT%c^k0E+O!+weKGO{S%eh^91r7Tpz$Rl0Q^0sL{)O|PE!!E^Q9*+M5A(=e=at5 z^eQ{lZ&>3(!~*-Dnvzg);g3y0#Lk7A$Vn0Gvhd{bcL>?CP98iIvRb|q8%J&yX~p~v z6cP!R6O`$?D`F}A+oq&d-1(*{h-b|m6bZm4( z0veTQDC=gB*-Jd(4TwNZyoafF@M4I!YkFp9R?#%f*B@lw$}bS6C%yn_qku?~zaSZW zfR=?we1Les6VaV`OW-d`^==Wdp1%@b)Z!du8H1SCgqYS?Yu$o4RVwpvg}LBFrF5a1 z4KZtPY|K&?L^;49DKXG=CO05NeVj*xu44Atp=aw);4bu^$kB zsFufq8&Z~&!o=Bq5jr~;TUWU~A#(OgWReacL;1=XnJC>I5%e)($}Oe4Go~O^pScRR zjCJ?Gs5Y9C+RnI$`tZbQt*1&Z4jnC8ep<$mgugO2wjqa~KuT~3I#oj|Sr$kIxgvEP z8xa@s-%V++*!LuZxPCfz9pz&~>@{NJd41ZZLDQ+waYe{Tl*LJxAx8XqY>ZfsJ&F<& zF)G?L`Uy;sc!)94RBl_8%Kg>W>Hlt15K4T0#T0~6xuO@usd9~l_?A;R1(Rm-(I-NO zddnAL>n*n?&}j(`h^so7I*?N>2&H=@B9!RwOZ#AJ&7vXCD*Cnt|o=;QBwHh)`|N=QxHlDzbE)J+s>2aCxiprqSHxSItjSg zuvhrD*jTY5;ZGYvjfeOV;WSwXgj%%xZ!rbY$PWXRrnI6EIRg+1i?~diTLWh&%}+@u z!5W;-nmH3&54kn)c*6K))yQZBdQrDuB4YENXUb-!7EdsUY1QJa&k=WPDKg36_gw>Q zj9bICYKazE_tB}Eh&svR5a$Y&gcIM?Vq?hegqA3ic;ZX2m@L>5KQ=xIJyNoao+i|b zqLgf2k$;jGoPaiK$2iR$vAtL7Ulcvm2{QQ_Hh(>-I2({7+ zV!hG;gu=o$bz>72NI}?1r*&VyrRkdwl#1s|`5NVC!~AL{^)h$q?3nxXg^{b!^{7?o zXG}YW>RV4Uh-ryB`?IE14P>;=d)9#yC&Jo#U0md>uZGE9t6-Wc$9*q(-b<#7MQwuG z6E(cw0*qR;*>5xjp#*3(gP3)jZH-7BL*dyOa@X<>lV+|Qij6C4Wi}ZK$i@jBr6gm} z#FvP<)P7T39E35pvVOO{g#0KaWMg`q)6BG#eD zO)06Ek!BFrQ6CLyN*-yCIVq;M*E&_oPZmoB@~buEK+wHBS7Iao5WKI!;Dda}0x$T# zeYB=c&CnBlIPuF{=grFuLA-wM4Rdawe>o*FPY6EGwm0aZX}Af0dKy_Y*j+bD5r^fH z=W^avI;ZFqyxOGmV%xOuFpN|eOIJumSM(?{#_JCmp}!Uz-yfKU@BRk_A(Q?G z@X|9rh(CZ*QNg&zAI6^!K9cw=M+5bb2u1o{L+0j`?7qtsL}RxwFfx^$Dl~`z2!%yV z!|bXv7jRm#w#0UFEQgrtu<7#;v2`tEHVn6)EXnqR8B9-9Du}-`Wvl8Le`5+FcAl(0 zUaD|N`$5GQCVj+A165DCi|YKd~rui8L6c8AcMG$mhq}HaM2$>adh;IpM%#z{%SXT zlRU{*!#x;?Sh}QX{DMMAC_`jwg z8o6P>P}L`d_uBx3!r}*}K6tsN@gwXzGZ@s3?Y7OQoph~IL6l6RRkqhUc^>(KDmXnI z!G}#J{ob^EcIH6z!I4WGZ;D#tSc6tuG@@5B2m>0Mu5$JaTOOw`!+B}!w+qSgzPRXE z@5TOT(Xj6@RC3%KwWzVDb>i+c1)(Hpo8ZrE8%nE(5xaa59x(V!IgdpoH&U^2V@-n2 zwAT@c;!1?Mbk>xlO1nN}3Lt!Ic*3~=Kf8-~J?jBV#WgHWkH;F6(ox~n z!jK@nG&a^fmkcP0B_@CuDC`_q`eD#~KnT{UD!wLdm?Y{_7hzGo#FROTzn4rw#LnNo zlkB&l&{$rvnvRoUAI>{tV`YD$L%>7+;$YgpG-agf3!f7toGzGU_Tw&Y)o44MfuZMNlL<$)MCAYYer%MsbV}O_ zk02gHDbJ-*Cd+qPC;higK_~(Err_RmCX2BEYxfLgyX%$8exk3C8#Y0%K|SL`d8GEcoMH(|OYMc39C) z1+kPpMfn4M#9#w^GAy`9i-inkQ0nmNH)+g{A`VlIUAWd z%p1rDFHyQaIwUv{Ac2mi$s(em22IJWBxkcJh}b!{5j^oDu{Bl3yrXpnc{W_wVz@6h zmi1{C1H06uaudwc@zHn|5g&Ehl;DbICrv@b&a<6<1RM5`B085M3avgct5m7lSbBpl zLawU{SZL=bJ=9@|!l{IVj3;B`;Eo`$O<_j|?g->q7=ax4DUd@tDU$ziH#>Kkjt1G7YjpD4eH#8*wX(8rleHd_jDSupGi1jzzU)RnFqInubEpH<2u|AU8( zWpGJ^C;cIfwrEvcX8Ha=dPr~-rgGSEgXLBY(+2m1E@}~5uNMOM7R~s1rXXVH^T0vu z`Qc+=PaYe!#w(?QWpgGXd(6!vNFBTDJ($j8yn8vs(qwEbO(rrDsbndS2^Ww`+MK3H zB85PD4Jpk$j5v8`lPT$ylx{Qy5j*#W_S=(=_obKo0{M~P@O;#p>CaLo}(Ddc$mQ zLDZtkQncRUETEe~Osit1U4u7WU|h7nRI0*Iw#y;IeOGLB>`06SEo!=DUewa=_SWgS z#T10<%SlrZDzrmzwIv%MCLOmHlLBkZwUM02cQ&FkWSr9Yc&I~+$JQZkAe9vqJ#d}sf%e3FobWHFbt0E)`ciC2mNwj-J%uVYYIY%Q5S=lb$flEV($KC zs|{f+<1AemE_@G6)k$ZI?xVGaIGcU3@phl$ZG=@vC<Fpj<&M1N0 zX$m5C<{nebwHmEI%l5Fw7Qt1VXqG`x?}W&X!=sRM5vwwx(JvJnx9?TlZnPVdm{lm0 zH7+Lw#v&(L(+(GaI2W_B+zWs~CcB~uXBlhcjc9*j8s&g;SH#uYhoN=A&%Gjlp( z_V^A{HYrZO%@joJoW{{0!sy&<4ReGWf+7(3T5%8w(KLlh ze=?n+M6=nDI7Un;|HzcWibFEgSd{WufW?uQvq|doNh=fa@W5Ej-)S0otH|?8%LbA|H;8l`9h$ z{MM6cZoR>XT{sdk&-jojF%>^PXbPf{9|o46G9HD&VE{s5(PsJ~(s=yg^-UAdk7E>Q z!;p19Qj20Y({Y>f20Gq=RnoEu+HN5{Fn@oRdLVsmq)GV~Q6}YgOlhn7*|!;lfk|n+ zHnaL3Mv#IDsTJ9zC}P0?@ly`M3>y|F|1vgytWBH~ z7>Eg62m_5T5erbCGbO6x%b%Kph@CHe2XRO_()J@efTRy|=s4C0oCGXfQcR=`KqxG}X*!*k zu99c2udBp5=pe+B@+(y<&Q(1mle5$z=_4aOBwvj3keoN|8>)MG4B|S@*?Yvo4>`SU zCT&Qt-xwDian1rKcjM)H8`~lrZH~SuYKH#$)`|OCQxHmOUM2W5+cWeYk)b$m9d_($ z4dFx~E_^6PE{F`o`Q*6Ya9}+yidvHUpeYGe7yp1Mh-b|OV{hS&`~9Q$ge(kbOFSWt zd@(kT+?eo>|3lUX5({myMvZQZqL%0WuPI>_Z~hO1xQ^2(Y6_x}M+SD8GVO#28h}t(TxF0lR*x%&tpn2W zlok_%Gc<1NyZ7<%t0H2g-LoiPoE}h6r!RX;Ypmop_Q#Tt}UF+x@Or!;ZYiDE1xx z3Y_TUI0Fr&4^jPmY*fE3ab6+AVX-_Wg&CnM(8x^IUh7;an}SecQ4oxrZ3UVTr}bR0 zCpiRGqZc!e=oL2S-V`IZ#6dlT3r&wTTms%{{_{m6^P_CMH<*%I$;^}~h(>N1=s~4Q zh2$E5P*}vJO8bWM6D9bIvjvgOc@*n&B<{7cn&^_)hkMRvV(U5m2|GYYBoY<~C2M8M zhR!o@*0-VV6XB}-lqu&`=lM8;xQ+_)gl$HviH(Bfeq^LdnmU1TSY>=dPQEV`^tnh+Eya#5I*zKOMK!&LMqt zj7)5^1lYA`x-K#W5j&?=AHaTg1fQP^dAdnOY zXj6*Fq&HhkiK`3|St0mgNlL3D?>XDbu5Iu!F+l85| z&j(^--g6W3DAqD#33q=6@jaA2(awO0;;3ah)k$L6FhK}QUc~I+j468*ho5H(qLIS} zW{R?O#E@wKLSYe?rE~KkO1f5_pF zN-5Zd^)zCaube&E_p1+gammf*fT;d-q4&fV%h!& zrVLhnMjC3K|NN-hBk!?}y8f$?`ixZDk zrXWynWM1K?$J`|IILDP!kc5#w#gc>?@5wc%!ETrhf=?TS&0lY8IhPk)KFCrWvD#7yD zEXfYhDKf6(8&Po`DbpUR^v_uaF|GcY{ajhM4i~MH*hF;RbuN&`Z{$?SKzJf9vewB! zNYZ8|!h@8)BYRA;$&TxYQc*8%o%D~Jf>2_4QE+dzt+=-R_5}P96Cqwa6&o+w5ppBi z3kHUcD3k2%rUX;n{jH`To;A&l7cmj=#6<9uD~1iAPsc`m3V7m%eN$PpdOD)QIX-Dh zL&b)VF^Flk&a7KK`(-r8Bpu6Ex=iiW16AzAp> z*toIZ4*?-mlRz~YByUcpC8j)=$rGk*t-@&#nv|-k4cHf@yZ*U#2L6*N2qj1VXbM6Z zHliD~WY}CqgBnY7*J-%_U~#}l>3G$!EU&d@$Ixyv7V6yzY?VT@v@XNVY6>;N&A1AP zv}k89XAsv%mx5K2Bp=lCxvpEoNL-?sLFd=Hr_as4#FdVf+o5V%h_2|?x;c7VGz@b?(CcXcOJ!7U&LpfM-E(HOdFjXOd2(g0uq%FblH|N9D<_$HtR86RwYCa$I1k2mS%cq^Bc^wI8n_&e$xC zVHQSg>Z>LsQk47}z?h43@2wz2D8#_OjE#ZVge{RG zW0)N?$Y(3Kg%KAhMXi{1j6^P;f6kQjihF-*3ZjvF1|y>yT!s2J0HLtBit%fjZe!#K z4ij-{udC0ly`tyH^GQdmj#Gwtiik2VjmL(36_4$G!K<<^)Wznni_35xNg&ApUk1n%(NKgK;dKWU+|`^(-la>qU{`fM2klEPg?JQF|ocs;NSkhg{tQ-nixaHu;Y4Iilveb|0G&}0N2eD| zsjL{4GX)Vlqq>#eL;5~1yftMiHm>yGKtunG@P>3#9Z_eezs8h+iU+SW1<}X@1H(qy zMq*XV0EEKgp;p|JSn>Sc_1%WPQ<&y%#=)r-(tmxb;uh0x8CDfJ%}PbG$bzKkeWwl# z_lLB@XJ@G&(w9fBVLlzThWQ!O4x{?qrx?U_bY$%C4~g}c4Gxrov;4@5aiCPGIC-%X z%}<~kb^-r;T(qw53=We8a=cQI4v>>kE&+0Z6{{`zEROP8e64jRebp3%63H(MKF_w- zVyVt9V!0)#>Y7Mj#$h0_n=r9HGtT|!4YO?A~ z>4II!a5f|h3W}l!o3bM9h>ahs6IIbZ%S}0rjqb92jVTc+#kQJKROzrm25}vA*iB~; z@*H?_kcW|$j3ViMUu^WgnK!AZ-9!S3NTNq+rPHmG{iG=fB@4#{=Vn_gt$qNz@MwJT zm#j$~XrDo7%$|&mH)|3X1OiEaA)4fY5zvJpN=KbHC8`n!&lE)Ld>J|D+7l(5>Eb@d z;$rauVGdXmm;geNQ+QcvtV#jy&>{$F^^Gyw_ru0m*d%yMY^*$n@XG*aR&4@jaMQ;* zDR8DeECOet0ezhDWPo#s`w`0UO{T0<+<${9h(_)k%mCCRLa1Z|5DJT!ES43*H-qD8 zd60xAZyF>&8(VK$nGlW`w4R)1h>Qp|_>3t_Rj>GzDTvtlao~tkLfBvhYa^H$Agy=B zsS4|#W&=pUc~0zaq2SY6-mX~p+xeP98>&gHJm~pSF{IZ2BR1|GOicK=VxyXlD^Csx zR}r&7XXp~HbR2e#BaAd$H60MKHvJ7#Mk!wZn<> zj%1KFYF_X_>%=`{3PSbx=L!DI_Pjs_7Q*_>(BL0p!i!^Lf*)8&%{KEOB1qzKQv#|E ze$f;}>^$fYS+<+#|MuAEzgVW#Zl-;N1^!l3!YSJS4^t40v^OxHl?5(ToB;@hMNAfW z&q?U}ilrT+ttT7%i`e=>Z$fM~)*9AE|A_U^KQm>d>I;8j3Zjt<28>jFK{)vhKqxFe z&i-RNcLwVam!|r1_v7y)Q@IDlpXZA|XT_fv@Sj?441aTk1(tMrukR+d?h`Yd+qU?& z*)cNdn<5u$I_R6l=XbV)2BSoXcC^}}ExCX}Tt~Czro#@#EO1rleI0_8wCZv2$n5X(wI73KbTPMW2m6M6wba zYt|xyL1NIrmXOp$7(pddqAI@RO+m!YmsQgIz!7e|+8<~V-3s~n*toLVw?e3W+JhFf z9~dK@Q73f1&Xk~vC$BaI@vM1b?21^)$5Z9)Nf?I6`;>Od$m~-LJNbVvHlD0X6zFs; zZl2Ok1Pf>6To7lN6yJz~#-Fjuw6Jt#DZOQBdeiDV ze(Y2@KyB#unC3k7Tf~V=Q3^C!$d`2!Fv7z=0K8h974mr$fOiG`Y?T zlmvwb4y0ZpKYFc*4L}tS_Bfx8dC`T2;}dQo78*~*#+2tI#*won2_(tCPDmyNlI&{_ za1uAVwezA4Zzu6RolJ_?X?#;G&wR1lrbvTKMTZKlSK+-FxGGL9n z73{KWk~;_cmzzQkJ`o%1b|%Ioc1f(4``BS~HIT#hO)Q`4auy&LVrYbY_EA%AC|UZj zDTvq^dh21A++ktXL5~L1(I@&}i;Z13W@pig3ex)s)1FOJ6nx5j&Uq zwbKzuEADR`;eycC&>+^iAuc<1Q^HfnBanh@5-){Oh;tDUTOGi%MXR>W6htHE46GYv zAc+yv0EEIq%zq3(C@jRfhXDwM#az?rBw>N|tECIDg!2tFX`NhiBMH-bM{ z2Agh7GM?NGpVW?O6%z~E?TwWW3mJT>v`dL59>?u;7Kc<@$0(4BY(?&(RXTzsBsHhe znY^)US-q>bbA%fd*wmBtg%xMKR&l483Xv;btD{!DPMUUMrN53bh-nEw`xaYIu)*Hn zk*<#xkH$qu9G%&?O53PeWoDPNG5pQ(nxj{ku&WTYeuUGj-_ROUlCbR8t|*Jb&a*q zfC7W0kAa^tL|26ocJh}^`J!a-i>4q{?}&>p(1f`fygrWQCrhzJn%&R{-?1^K5x%|_ z8!aUUDlvh?K$F`DE4U3fwP*#;Hw6(pU;0ivIEhbfGdl)547(GwWCf$8Jb8jcY#NM> zP5oHy_YvZ4XV{Jag!KGtob95Bnpy|9o~o2E z{BTZQ7zqB?2Z9t*a&L%{@~~G44_-}Bcm!MsQ9jC&oNAr+uQ3IoMC6sGAe1F3dO)0( zq_Lp(9*2!ztkw$f!Ft$gapYjAdweRk?y)pckS$YcLqx8IMKL18`{SmJP@?z|1~IKD z&c2JH{q*rsUJDFS^Y_xY;PERvmcxePb7JfXWxv@ZH>>r8h@^0Lm@-kRwq2$mV&}k`;aassku_;6 zKprtAs^ZJ@O+m!Ym#))JId7*O3mM1} z%o);=FN=*C*kLJ}O_nMYL#mHci<>#{gemzH2VQInVmc1I{)QXo+(7@D7nc^bPqXUO zHC@Gth1i(XC8oM0DC9T_vEp4oKbG|%oT4FSvm_ndtR@UgnpyFbDIFCn-fjwFI#$dY z2eypbaI|xz)>sX@K>VXxeIqrPJwXgSAE$M7bar7EDY58kK2R)cmUpsJCeQFE$0C z1hL%|gtGlb5BjfV`}gH|TwS6kFb$2W!`yMiQ}gZ}elVi)TE6I6hG5hw$p>~zWol`1 zBj@aky7|-HEa1;IWsef@eWoB(ClOP#YoUj>px=1QhCaFXlekdgKp%Xk_Hi1|me`UiZgM`-kdc z7a7F$8$*^KfH@>ivK=l%2_v$q;f&|E#6`;b3M{drak1-7H6EDGOQLlBn_8#w8%#kc z0h%%ep>)0AbW37fYXu94@x1FR#=VCwVv{L^7Kgex&N*xQj*!g$6~1ORW@q$O4Kmv# zssG!Qa!OMFoIy-0sna{ajV+VxcME`c{ME|BK(T%;|{_tW=ll2$L2c0}^C$tfx7bf|h*ht=q-9lQUO$6U*N-d=^wwZ!xB)Gw#p~f8{LIxld z7XL1!aJE;P*7)Otwbe>mM-a|k8?&%qqE-&6wTamJ!0Ln}N2ui1%957-V7@_T0v3;(<@k~i*OO?qGYp%q`noWu|+lVKW=R8S6kIyF!qbAvX zxhbU;vtDKjB6eo&2$;3Kky)oEt67|NeBP~8Ygn{%inuBjDM%s2wfDuwwe5;)+rwNd zGio@tmQhRUs5P6>Ktrq7Tw=;?5Oc~gZ7RuQ6Ls1hbI9b6Q z)~-yF3lTCb4iXQ~H|~v#y!EXrc?4x=cQwRleS{ICO!3(8E)Ea(nvKvStuy4HDMR9z zQ0HQ!_SGlU6Rp!SV+ukkCtI*%wr5t;PpDrJ8_BOWp}t~DEj1>++!RD3!3_o!H6{rW zG612l7>PX&+GC=mVx8VrSspfz3%C5hVCv|1af@Py~DS z%=|)Z%)CpPq+5}th0$cwRH1o|Z2fyvo+!5dttp7u**XxgmHaCQWfCdQc%}L}r@6u; zFl6q{9f--?t8Z2;+$O}4JS>zDM4`mnPcRQ`wnUyWrL|($)21L|XV*gkyU4?GpyEJ_ z!h9UZ-x>};6v~RtOWlg&2|PyyGGlGzl$aF3(~6)W-rgP?Zy!{=-JVQK{F4%YU)eTW zp_uTCs7C4q3V*FbGycv#wwTclf_cOMb;2KZN@#ZF*M`? z2z)e24UeXA@{lQ2m4xm$1ra-Al7WQoBnic_V>SniOwvi16l5R{Q}fR8vRa5q=VD{h zfRehMd>SI-lq8%)5GsXECViZm7?w4qwqlrL3Zju=2F9(jkHwtd0EEI~z3JE>eKyN} z%08Q2p)U8!v()A2tFGp=`6bgXpnB0;7=(e(#?(cyDX@r|TsEiO%!H$58wk<$qjAv{ z#t|E`WDzwDlNCZq%Qw(GCVsee9(>3Ygp#2T3O3HRYoQCyGm(U=IY8nou`%T8{+utF zQtba%_a$(0RaN@glg`ph_65RAia-Za>7+Y50(LeCSxtuo7@&Ps^}4$%)m3dRNg9X$ z-~x2CM3i=8T$phk9i7pU(HUiMU{KW2Q3o9zcb##=1s4!S@&BEB-+Q-Sy{cYL^@&b? z`BmLlRej!h-*=aD?>+Y%Q}Msa6(J6Khd`S=_kFEg8cZ`>qeRpkescE?8AxCXW zEs_O>o&91wtd7`alPxv-9v$(k>qmSKSA=OhJGmmt>_#P`F1t}^%rlNk%H;wFqG>%K zv`2pn9$+FITEm@enWGZ?D)jwp?8%?z5@8x(fh)pP2O%OZ@q1F*lMig}jo5X*NsO7b zrt91k+|(Pr<=)H1*--U5t_X3e%oJU@Przw{+Kfu%fh}394|$<-n6=}CCJuT8Hz7VI z##Y%B9j+8wCmpLFIz=mW3YjSnaM3n|{WDiYC1H$y(sZ?YipLOPw78dREW1C9M zNW-DC2wMYV`qL9s<>;wKo1#C-tpcVN{X&VTX(H7=i@=?tk2LFBvVLNQy%2U|aEo7v zd)GN}XyJ-5ZJ|+#m~g{%J@0+Ax26Yo$jUXvO+ER zd(xp~_pt9sOVeAqh?@qmi7R5vq_JmvD|{Pe<2s9MHOErR(jjy4CNYXu7^7sPG8Y}a z3SHfc^9C-KhK>_TM9uza!du4Fjdm~!;;mv>PKnsz6~0^QhwL<0gsH&=t_WiYXyWQJ z1cW-jh~JFb*I8&q@nV{S1CuWV#Ry2kr!QzV{;gOQh?wTo9`#jGd*w!zkg@nTTy{*e zdIwj8sUxGX!|3JQ_yCU2Psci|>b5uXX~W#Nb^b2$Zl`t1%9RyzS0V=zMXZYcrgJ~o z{WLAG0!G?@D27uf8vH;(Ze9u%uwcQBv1~e#AMD553vWa{DcH9L{`^aB1)-c4rK-Qy z=ev(X{+;(jqy0`WBr7&bWnYLq&%3smj=H{{RR?yoL z*?8LlIAp=A;7}IRkv7LWFiYoUJGQP*4h&{u!#WHAJ$*4Yo~BwovO?I*Dw_J;QQZT; zqtbH-Q47136I^ocA}UqGaH)){DIQT7HSEw2aJ(VZV*Z_5-A#-6q!LkcAewl?cxp$* zNc5~*%Z{OzQHZ7Ect`4#>-5_V`t3&jcC&gLD^1fP$;8GmYR}Y<+S6PSrk{CI13%Hb zO;bB!!GV5No77}uN+cErcO`CD=lo&?S41Va<7xVC+cE>aZWs&^MvI9bN^}gFEB%pV z)4JfA9u4+-vH^E}>Vwqcyf@}n4JUR)tbUaFJXWmqK}|Q+2b}jwP1Q|$YQniLPf3l! zmFpn4W|;o&8YSXf_IH~q`nv%!z^T7$52My!KWZti2-6sPHSiPO-)*Ys@9q(Ww<8Si z-CS%;rF;cfL?yh8?#lFdx@j;(7%fKW@wkg=+x!l1^SZD>OjfdOWt`Ji*IW!u4waLY}d7U=x0aGN@Y)gy9I>IpFN95F^NMbu_bB{gp*b1_CLopM!* z;y|~nQi!bWT%=9a-@+9kPS!}fO5I|FjP6T#BNs6P`t@89;-K&Gt&fd6$nOd3Pb#9k zb;?c`;X#NA3j4buSZoqJzeHV zRptEBUvTL$4F5S-ggC>-TeQ6fzI&lVtkebh-t6IXQjm%wkrRc3%z0QEnE@e z?CtW~XUC2a&kBxpXnRzQ&0VH-c8qw2a9l&)BV1|>yAN?ih_h=#ZEf124lWgyUzEqC zvROP+CD0}x7h|#PB6w|~=@|k; zgwf*X+0XUsRNM%}HusZ3$YCfrB16O8T4^+j-#)+A|(BqCa^|NrV zIwvdVD-q{#>gR_}FTh_XfZR?g?KFXoCEGa(Gm485JELe<4_=r|8AYddchPG>FJ zf68gciVClp_lU8wDneGDRwN2VqEIC2Dnf1nP$b06yST6#X5OwuoWtHeij4AwB&F*G zz$$8jH;B5)4E5|9r}aV}fS(ove?4WH_q0NT&_law8Rc5hp}gu&fsfYD#3Ni0rj0zL z$(-o%qS-TVu)LwpvIYj}pNg^466sf-Kn?|Z(>D$^fycS1nI`Z9u82zD8OE@&r*%VQ zh%j1=VlJ?|$VEqK#LB_)J?6ywGTC&tH(6mboZl{F1Tuh6tmD z+r`Z!$I+QcZCAkI{o#9t*S=_&oCjV1R`OWw+o`=%&*I6);hm~|N7YPwCK-WLA#NIO zJye>NN9LbIG3^ZBF^sDM`KVHJrK^U5bEXi>W>3Rk5O3#45PQ8|tcs;$` zix($#>lApv{sI3|x1f}IAzn(ebN1=DjeO3S1M5$w^`^72;j^hf$Jaxpd9KvHWFp~Y z@bUk{$NMLxZliCDDRQdaKUsU!?&N#as+?i#`xCActqb?dptS(RsbR=f=+5@dFOBF7 zY)IPaUV=0AM*XAI-{L*>KD>nRdl?`|y_vpNrMIBuqL#W(l?keY-=A6qtJGi7pLLE&PP|07SkT|I4s*HktHFLl%LztwHWzof=v+QAcq%rEHWSM>4}y*!PVP!srs z50onXi38Q-1kOI_E@Fl{2cgRqKcLRxP)Ywlmn(jt+mil)E?4}3nwKX*mn(iiHPqlf zy#qkcq?ZPInL{u0=%tBXTIgi~y)42@h>P#}T&PlYE~*ZG=ek3OEr7IuDQvV;rL^7A zDzzF5yLB_|+-}`mb|9I-5`xv4?sZ`2 zud}YkE}e3E!a7rDE@uzweVqO*0+zZng1tI^X>e`)x!`poVwTYJSM24eypE|CS4*6E zIqq8L)OLpwaSo@pwRd&8r7*1A)$ucBPN|kD^Tzr?y`C$=G^FF2mx-R;MiJ~Kn=9Cv zf*Z+P?UbKFHwAa4KCki*VC%u|t(fz^M2w#G5uAkAT9rrD)mmZATQ^B8cDq`Y@i!%$ zu=qAE_-0ysD_6vr`Qxv-?X>gBxJO>qzUZA|L@kcQ=B}|qGm5O;qI>syRiFd_1OK{VqRk%uV@XAih;Z+lE=-X`sNyHx_McppIWWkdZd1o zAL5EI)&4=Q2xEn4uIsWwggVdWL&g4qLDgg0$~Fk>cRVdtBU&OEy0O}Ja&KH#8>WR@ z+>=}uOpE&kSA?kxLM*I4j1AG8*8^g3u-s%EhY<=(Z4w4ev-36ynPy1mndYGwYP_9l3ZZTKHn3-Y^RtvAPiwOLPtbBXY$$l)|g*5O!=5LquQ8ubNzJu8CQg99oKP1m@!RPiU}Cg z7$VBFP<=W>gu!AfcUW-e_fMYNaCJM6=rtH*Mn*zc&t1WDb6ba_?>0^M9qTYAKy=1z zi`CknOk|S1eTB8_to9A7u}~s)a;8UK8(jZufB>wmdW zn}+ut4fV5Fv~%}I-MweM6Fu<_@3JHC1jIPP!Cty79np%4F2A!xImgtl_wcz=9I0FBmJVjej$dYL&-X=<|$kerak>( z`h_DKjFrV!E(NBYt>KCg zXK}q^F}}k(;f{YQcxCo<#t*ya#obYfmI4FaAu;aOYwqIy@TRgVL91wGm>rdBDePIi zp9{F*>}swEan7EnI7{rX_GPf0t{PuImf|vnV!tv>)Gm;nI7R2*6V_?k975Q=;s9KK zbwowa9aXhXtVK8hT3;YW>+>|NNQdkxj+FCSab=v4;%Kf~@hB-yLFTmLs>AgZ*S#+( z9_n*5T-prtC%GcTnct|Ght=w}R!V?Ija4NV z8nstaOdZ=!lVT(0QF;{@cEj9VToK~TMHF*AJFG5v{o!~7h57ONGOvdXaP=WEt|FSN zo@m}zCDg5k+tmlT&>D8$&lMrg&UVF4?+zV1#5?_J`GGoAw`y#1OL=l9Ux1JV0nYwg zjI-^Uv))>fL{#@_6x6QfTES2c`CnWb43m#>MTj%GUoqLYc87&buGu_AT}dY~@L|Ex z!s#djQ5lEpENvnpzZTplXhAUToJkf21Kj>zjN7XWp@u%gtbti&Y9>%ToK|FtlkGt_iUo5Rtd<|#Wo}sz370R>)RIWrVoqp z5eXd2CgastCF-jZBIf`XS<_1QaYcxevw1Hfq}nQ;tL3EOz>~{nVEZ6lU=lk&gS7FZ z*Dlo<3y@$H^#}RNiSe~10-G7yDGSwThixIP9qj^X?bHD4YZsz0%f;VNnC6NQr*KtQ zwx@8~#@I-^*cqgo#I({qcvWR1`!!;8t%l2q)UO-O_x{p0qy5o97J4ZPv9OatGeWpXtaiiOarP6M??iKq$p#BXL@UbdE_)(o(rtg)xf zE5ZSjNvy#R&~`F?lt|{|IV39m$= zS=Cb#sc)cUIcwE;+gv|y?OYM2Z*0@l*V;1X4nXtKZ^1*ZfCi*`vE6%vhUtxBtS!J! zzjtdgLI)B(Wv;boLHnzClwZ#U(sWtJxgzAb2*)lBj1spAauJS18f+I;oXKzH0%^Fo zg)2gwi`C`h`ECTm$t7cmB8Vlnz3yIpu${b3j2oTI7Qt`juL&sNG5ZbOaucH(%?}92P|TdTd<-K%hUY<=6)>3+?C|Zuk}=cdtE(! zyTRz`2l-NJ64y%2fZB&~MO5;_SZ^{*R(eXt5Mi`v+t<{O?W?&WOe1}zre~t(-xqnQRIN@N&%!D{SxS2m zXbw5i5%m|Ft#tIEBS6YW#7Ma~qM~M@C1GRUQwQf?(`n>F*Xl%-Iw6ui%thEVfcv>3 z#7SyAlC!b%irrQmpr~+&`Gy!7O_=xvZW{vqt6ank=wIfF7&GYXrQVsfJH~ z;Y!@MPCI-BSA?mmOSvM9rKO3i%hD1WN=E9Xj)>7-BZu=rF4CsST*DP%_!dICYR{Ax zg}6%E*a0p+2I79M2yuv49mC2HCbg%qr9>w~>1e3V)=T_FUpLya%#v=Q#{Uy8 z_J+P6aYa^y}3Z$Vxs2%)2 zE{cYT7ji|26R{ZB_hV#@>rE`29n55$iijg`6Qf~C1jw7)NB!G}Ncbx*f`)`Qb47@g zFwgZw)@K<4P2dw^*f#=u8rf+&g)<7_djD@+ybR?3$`v6F`7Cb>D}eTo#L#ZgtNXND zAFA8m=b~hQ{vWOgaiCZ4c7sa=R}idLe`>!rK*7ArgsjKXs)!foN@x1^mJo3A(OqN+ zQ!^Xd)M;PNL*;8qr^H}yLRvoT#+ciN`ujW=F$4MlSA;m| z3v^uCf$Tsrjg$feLvXqH4PwYIbOX`4_f`~C8E#+8#m>Ng4_AaZ{42c9CSUAD=z8)@ z%d6!DtnUQ+osWvqaGu}ql7^2W2)EfSYi>0CCAGspL>6jNtXdEEmn_>hs!g--A zgK-%0uW|7*kbi|MLLBk~$Fc*oN(mQYfIugS;rWb2^m?{K5ppoqWhL01@*FAC^gyJb zl*W>`QW{QK@N`b*-6e2@((!f1dd0ZY@BV>Hmf`z%ToK}Y*O4)KjToVIXG~rV)#{vh zT)-6}PMM!E*$Y7&=oqgSV`VHEllO2@Gi_xjSA;n5i*`HCfaQg`s~V2>q!{{(G5>Z3 z{AAU&BhU76@iP=SToK|FtlM)2Sy}s?^q{3v-0Bd|>nu3rW;4C`I7dO_acl=+Faqu3 zE-_}-MPQ|Cr4Zp+<=s^woH6cboe-tO=oGTxUd{#E@HNC0AV;ep;#4f%lh0!-14$$(6q+9$=iq?=6PJrIu`J?v3Lx2# zRzT5`4imA1i=t^0Te%{{iMX&cW0mJ2Z|hHHaLD==y~j==1%OU~l;?|)vL<3@qUFj& z%ax(9fl8TBr*#t-T|>|fToK{~Z90at@Cd_-Y#ce;B0ns*>o9X8JKdW_%=*6me4Cdd zqkRA1C1QkaiookG>M2*PTZ;STSuTrdqE_i;rC^3j?)Gy`s$)}Avq z#Wkj;WZ*!=V-3eTD?rL86)E~JI|}wm2dC$Ah-Xm7rdp`*t-9DCW?iR}32Sebj%8L6 zHXdi&(yU|#Qm~FXJXD&2WJbkYayS)oH!XExTCARm8libApX8`gJxZ#Ve;r)qhz(*k z2G!waqf`%dv>)fPYAVLVToHm5L$kZKjMC$9enB2t1oVj`+=ohoSk`Oh*hc&JFX>EN6zsS{0Bamc=pi?C@gZ{ms&Cuy8f;6E-#4=W1% z!(3bqH6P=O5T~Z)$RHLdtV7fd@=FUDbR3dtBTM^xx%* z5Ql!rwQ4)oav4Y`V>vrl83t$Wl~QZ#OC#=fu*-wFe`qjofD(0@zB9NY#A!eZ7*zqu zi~Hz0YuBZhVn0wf3Iy=FrkGCJfhN%=#>fRKbQjtJZ8_zl@u*fKij=V_rACODbzD?U z`&i2rAx_MNHHlfLqGQ^YQ-}wsIVwhtRf8G|muW{yj}SM9xxgB34sbL)D1OezjA_n~e9BM%`z^!46MYS~$ z?#@2UMb3bKKUah}_)UJK^CPH$cCgZaLk#(56XSe9-#1j}zskkT!2V^fh%v*?j!x2` z;kG*W1?u|b=Sms9vym{92#(sxa20=+3zWh7871PJ1RQZ!_SbY_IK$=~Mpt+%m8Eai zi&1byrC~P!?OW>pnzZ&s!?XvpUvIal?Y_D2Ej*blXkXsI-?h55T!UYif4o9ln%Z5p;Evhg-I z%*3H=!O6E#n3Il8>yrb6nb>fui?pG~Eb1MMzMbWhTK)VIyAkb?ykY|hHHnR~oYi`9 z4j#B<+XoO1HTjiF@9`|NyCO_AT)`D#n*36Y-bBY{x&Bxdetc9<;mJg6$UBg7Gfs|n zRxFuOM`>xty%+P~_0~>IhbhM+HWU*G*o$R~euU3;1B+DtJWy4SL;HMeJQ$93lkqwn z=Yc;UacigbD!BUqNU*FPAnH9pG)DU4hgyGf-qrdmgR}D2;r^(L%c7|WN4O%yE5d@q zxn4VyyoGuvst{Bc0u&C4QHZmo{=F5I>fWJtc#4aifj`d`ArAkVPFVQl9B<93r;o2aKvxPiQZ3$J15wOkRE42@?v zajTjM2dT{lh6tmDP%N#cPB(~_7lF8ZApNvhjcBHwM+%=-JM!_PT;>e$22ZKRZekw-N zbFkSpSjE(Vl326UN|G5#ENUgyz^9cAx8}#WY#2g+z!f1*=$bMXyxjHKQQITdrLpz_mV3Ee7&>3a6(LUNzB=e!hgClio^T3wvLe#w$Hd6pR}Z;Z7|W4o zlvX&@a3A2(WSIVEt_X3auW)-y;pkuu2OR%n(w!`j2KeC&K2>0iqsi9 zrKSKi(#uT64ePYEYl#= z^IgQn(KM0>SA;kfmmJ2Dud?4PgeX>KORa5yfOUC5jGId%w5YFCQb}m&pk}Rw{qW3q z9~Wsu)?Tg%ak4J1*=|NKH?m?(U0k~rjbcosxeyzcl3WquEJcyv!^t5!AjOlT$dFi5 z_fD#h(&N2ajH~tVn+s^AW^3c<1z*X<+fY{GiuhBe?0zxI$TFys73~9D6b%u7#})CX zOvIN(iRcIqHDBbSXo&a%SA;kbk;A7mh{lw_DhAAg>N_0)gWwr4_%DoPPiGKuDWS3` z)Y#An)AKYJTSL#2ToK~*tnEx_7mZvF^msc5YNRRT5E+}snY@GDmZGwE}9@IoVmt6(LBK>lile7}f#YL__W@#XzM`+6ORl=1s|wJ%(weZ5j!uvMr1v`8)2Ppi~m^%OHm30;vc zU28s`{qeE7^DIyiS@7i1_NagB&|K^aF3qMuu~IOQ(|Ys zHHFj~rKV`jC29)UMO8wa=efL^sxiP7@yDnd?@>gLr)qpa{lx@UjW67LQK%CkwMMBL zT63wYu{k`xyo<}LsTyzRiuhwxjc+NU$5S=Ful`~Jt490n7l*0_sWnQ~(3(qCjVD%$s*0zz?F~nU#%@k0)aN98D&A z;kAMF+XFc7fNS4dFEBeBcH2R{k<txCPx=Z6wj1qquWuQu+7wL;Bv4A+4ECJk@~XR_@s^nz8lm^p1cv z*7`!74O{AEpz?DD=)Hy=0bKu64A-UbG|_4ADijE#`?>nj{R~%x>7_o!6=C{H&2e3Q z<-3&@6J8sZ9Q96SsR*BG@B5HD4js33>_zj{}5T?P_RM z=aTFNToK|#EheJk2o>9w&gL;JRa@!2T8x$@dZiP$kJ9Z&Snb@yMbNaCom>&(B-F7@ zeo~B^y0^*uxOf}N9Il8zWyqB3660PY&t%PEizA?_iB{#>vH#2%8mX8b208KjIrjo+|!<8ly_6v-%+y zSwqhExFW>K8Mj+=(^XO?E7PrcE)=TMm~G&S5T}Rf*4!>eQeAG%ZCsp9OIpVjAx_o0 zu7R{0-+-dV?WU|a2$*}!^FF6d^J5P7JVNp0t`%b|8p(S~`QhX!l&l3?Kq9po0e_qP?Tt#B%R+Fq;`d)FBZb87IYo9{Q4ar=`Y1d zT7!_(q)e2QiPAY7qfFE&6EY%R&qdb|^ct=Rae@%$B~gqMMFq@KDf0luMD4+(6tTQ? znrT`QlkqES>F~TYDolJHRksIuodp}zQIM%knmNm2yqh5-|Ozd z+b}8y$&l#M!LJcoF&Er6nRd05MZPlPc>*Xc$Z{AtK=fp&Ql~Bc?4K+p^AA!0qd@Hp zbY2&W@qj!C6pAQ#>rkh3Vg2B*;))P&539WFlj%X)=1o}nIO-(2C$EME*AT7|<6?Eh z%R8AKr0v}VnuOBCcU!3vjqfThuBNNn%@rX|%>~~24n};k&bp9kCt>G&iUN(KUyK;b zU);e^PKFtBXDK~Ag2E$6ii@eCrI#y0oR+48=&ldiXE1Q6`Wc|%ZZR5=IRy9Zx6jmu z{S{oy4D2uCiV%l=JI(PuQ}`gZNAhkQPc^;(H@^|% zrU{cxeOtXJ8XkF`;v!~1|0P$1IOucS#6NL6?*tG(Z=cjmWv-XrM%^>iO)bNX>ztP? z=86!9+alKhnyMGm$W?R7P8TW6D)rjgJYw%w1gyAHjDuCmBfwK6suaQUO1UB-VxGfA z)wF}{ToK~LEI$xBLU%;RHV#f!Vq%o6h+vLXk?sj=D->S2$}FXui=-jrMy?2PG8XSm zp7G2wuZ~t4UAK!7u_V%?Zm(>v^4o`6#EZEI8WR43D?*%vkPT0da7Bpov}mWDPsWj84ttZBb)*I~W7x<=%`}GfToK~Hx9ozegpOTMtuHzuhWz{pIcezF1vUGjV_e(}^v~mp z5QlyRrIdB^BBDF8cG>ATW_>EEa-b8sMT~^=A{poq)!7g+5=5~!W^@R3L8rM`8afJG z5#n@Qe9&Kl%a;w!+9@A?MOZfsQ}ZS(H0h^BkF7#o^i#}y$? zQ=_**1-f#{pdbFn#Mo%^_oHAt(QQMZKfp!Ifd0>15#pd<*yZSCPsIUcvTE!t#1M)( zaA!bz!Ge3S z^_A0sJK%rkDHm1BgN6YO%bjLaWXbxdoY(R^MWknggn1@Wpg-lqRfFBE-MGbXxdDvEQzFg9)lieY{34Z+y&K!Cy_vSXj$NQ^l2v$mwHQxr;K^^(ezQ@GVUh~9qCHxznjPXRxriIm zN?Z}*q%ALNPp<9fn&H9CulvOaT2bDRbi?uDPgL3b`T!S6L&o26MTnCzzq|!iX(eA4 zBVj?Hm1q{icG?%Y$Qkgzz!f15{>m`;`k>Jf@=px5hG)cxxS$FVQFXj%K&ug&{XES@ z)R6KdSA;kz?ZH&GWt|n!XhWdFFlm9w>dFIBQ=bkv50xh*uCNA>OQw9>snw!ZK)*U? zK1;bG#K~I-#|m{KOr6)Owg}xRM$aO0sZb}tRO6|+4t)g|J=1zFqaD7mkOhZCC%oWcHPt_VTw-SkQGIx(N@NoMRc z&O!w9DBY=WuNAR=<&CjyI*}jj$J-09yES-q^RWPfcexCztU06oVLf@5b`EUR5$;QS z+pf(c<851iU&_hkolM&?$42_2OmEvk_3o&nPjsEg#@h}!akBYoLnPluv3)x>uTKsP zW@5uScN0NTjEyHaarMMZ;po1t5eyMgmw>2&G{RlnTMU zdZ!&fNejGVbn^Kpw~9@8koZyrP7Cpqw7feWXPz>1pW(khU z#dINPgnmtomcP$5#j_?xWFB#$Z1i-1#b00Dciro1rBlJ0}hJO5nSLdfKqiD{O55+i1V}( zYZeG6hhrQRI?9jLR5f?T9x-+*8zN?-FsPVL!k*2*=hr*5tMm$WdJY$K z!&{6iqLR1qn6U1Hm=qm4vkgOp(cS#AX>qW@jYax6-UG9q`zEMT(ld{ix9CmW3S#Qmy-Gw)a9bxluSzIw z(Ut&w4@$vTC2dia)P*0dAJKo}iZJcvA2lhpDmFL45&4>#Auz3HzvyVOggj_wZ6jDBW#P5q#+=86z+H#N@;?-xUSlrzIubFnhr)E=$~ zak$5KW;i3p0q4x{NiM2}m_Duuabm`LX82WNl#K7p@LgOa4H+-ziV!DbEN6y4C`QEC z&kVnxi=ZLleOwXZB#iaU@PCOBF}^dyk8zPSWPF}0LY$0>1yj9;FzT7%--=OFv0$pV zd&YKV_}5&J4Nt$~iV){%3}=SVKP)xP81tFo-%<$!0 zKuxpQ!4)CS$;fAh<6(PSZ$dhJPl;#^`5; zf6PV9fc`_S2yxKIe`a|85h){<^I9&lhMc2Z5#r>G$4B#pV#JK!NAnC9QA0|RD?*%<(S0=EEJnpx zeKh})i=H9i^;{9+1XL_Jj?qW+U&TnNSaKYXkLJH{K{h;ngeyXvr{jkQ3d#QDEjq>l zxmxvBxLU1Kfndn#DhE%_0XtWqphBtFM`zUEoA(o&$&UY*AqiIUbLw@aL+J{^Vzs)7k)PZksMTpmdHAfv} ze?aDE3(lX1oYMvPS{}6r`tsS#cW!$_9w)^Lec52tpPA2-vfwYoK}xMn6l!F&HaV?L zG>@ZWN|z8p)1XkD2K;2M2yueu9YY{X<3)b^@Qa3Tbq`EkB1S}`id$*!8iM>{E?%Zh zT*wt64*3OcH2hvCiz=5x+|b~ei+~5^K`|06FG9YrM@7zv@G;73xR@GRuHuRirzNuI zOu@+{)M==!MftTbIw4nk7Q>JVzk(p?0cr-ssJRf4;#^(m%o^!}vBu7z+j0;_ALkxO zmr$$e=VEK~x?T`idCxv(+%I?-=TAzQo1J zK>eRw5#mstk(#(to$Dj}rK!`{o4r3#YPQ>k86b%tC=86y}Vqq8N)X05?*yfa(*-dO!zN+yaF%lL< z^0+y&PteUNOmU5@TJPebX9##ZSA;kLtByLSl4Ld^6Z06-VSZMOjn!D}^E8Q$HTC7E zxwsl?9_5My758o%TKG7@Ma+PHj4MJMbTdXEk~VG3T5gQyKv#6D82r;B$;8HRFLMhQ zFoXGNt_X3=mxsn>+ACKlsRF3qCx-fp@W@ON4b|bPFd1*+B5BCDmn%Y?jElovz6vo_ zd9W?O%)?^Lv{o4hRII7WgKc4&KE}n^(DVRTgg8w*j_6UrGhr1+o_Xo5&FY}qBQp0r zG3G9dxFd*X}HcDV2XES-rl#Ugh zJjEfRk?e$%zzzzo0)6H>G48epqw=Ng7{U)}4Nak-6;ys*r+o!OOs?h9VA|b9ToK|- zuB;vqt!hHy!(w!;tdf$RtdByi#KH%-fErHraYcx8GFl+@oEQ&d3Z$Op0%x#Kb47?_ zKi0q!uNjqt*ZlaclhnVPhEq4rm^2A*H>3o$~;bA)N=VEE-_yAXgI2{#k_LXLI zjZny672}4QL`Rr|FLMDj9DI>0LY#wzD!ntddAs7tlgMtN+~EUl;aM@f7wNptv3vK{(?oPjVfa@< zfjZ|JE4U)W;h#%6&2>)KWH1i?ZZX{FMJS26f78&Epp%Q1sqa^CMTi4Fo)d??ViZtU zzbRbR6I|E~@-|n5IP#+(D}0$K-kZaC-^m5b;Qdmr2ywhC%$qTuB7CnX2US8?zncr7 z;ozNI5#k&icDMLkKLoP-lj(FZkK=#^r-u8;y7Fv773nzwBlG9Ps5&I@o_0j zwk1+|COSOUEWq9CG-I2&BE*4TaJ^H28JoqDX0Gan`gSqw7e)r10<2keZ*MCq4F47` zcBbBUa7Bp2KbQ7&Xi#2Q*w1l?*y{EEAudt|><76b#KBgbO;0k1 z3cstefB%6P;JUL>x2@5`|1TFY1NwKkBE&&o>(1Qb*@UBfZ}QpnDRp!nosS*M!mAnX zFLVGR(D}?cA!TD;;?2oOBjsx2YNYm&l}6r?jIi}H3u@JAd`{IqE45`5Mi`<9XD&0n+$R8 z^mCdF@iLtZaqAANiyZ3m1W8{8WsRvF5GZ7N+TOF2;tYA8=mmCxUk%_=FfS^&AL3#zol>^*pW! zaiT^$5PXXmC1X1fe42}#0l&Z%ArAgX2ZG-u2K?v;g74)bWD~9^`4g`OTi=-jr>s%4yWUvkd&$wR7(yik_@Kh*O zr!6~)D?*&6i8>IxR*bocJrH~mmkHD6B3u#TWU>zg9}r`&z5~JgxKJCm_Hsptv(>t< zAA5aj_s=^n9Xx^GkFaQ&LNT}%m=&XHZ3Ge6v_j$Y+iE+?S19C`kmjOnh)Qxrh!eG; z%SLcKwPhPC7R(-$2S?}xeqbHo)nbHgj9{z5K@3Uk+(I$473C=wqA|Qs^GYu2hPV<} zgg9{>hfg`VQ%UEvrB;k7?q^pa8}Gxq6|x8T3w54_-r9B5?zI8#?ib^(J%YV{rCU_# zM!`VyzHTAnqWRuLn7j{gkvHW19an@nc}sWNg*YO4sa-XsA4x`I`hx&NUlwC%S;W1c z+Hwde5bFNE$VJf*@dd63aU$j&$z?SQ2$CAy&3r}-{YDA{lULBIyM`ctnv0i#{7J3| zambP788NbRy-p_XSahf(2>Z$#q^3!f<=MY;RNpxS{R-T^&I!>{t_X4HXLZKo#he`v zMqTL?L%bnkdp8ULeFYaKQ^zmkiVz2S)%b?ugct)z=ri7d*XE*Xh`E_7LY$ZtBM=i9 zd+!vZ%*fWltIicR4@d;%1=#5m$sb^b0D^7t;zu0Q?1iCS`yw^!(AiGs5cL zp^l~*3e;(W&f|&@hrjtmpOdi?S$G1W-e(7{3}C-S4EyHDY1}xP$ik_I79*-u`wlK> zrrK}diV(+rMOQl6+gC^rT8U&&kCUS?&h`N2p-4*`px}Bj3SjuSIz)XPqP`BHW^kN~ zrJOa!#Ul94k^{pgIsJ4J@4m=5T|FQN5-`S2?|1Zw1`BahO)rO^WS33ToCbiC@M9g zIjDiDyiOU2h&~}w{)>yKA>}cy2ys&8d!)dFW5CH3k`*1;@5P8%;1hx6Gw;?R@PEri z&Vc`Gt_X4Pt)mCK2=^V-Pw8SJ?4z376?pjlakWi@Po2x!j>#Tx(65FmS4nu&b|4>BfziYlKn@#jM=``X{ zQEb|Dm0yl0E>iWpM&UpW>wSC5S2(Kl)sYx_eXXJtwKg55^WV8#7&<@66>$#ebZspV z;!Xu?)A`hx>D0fJq0?`Gp2Fj!^OsyM44qGKMaa{+LJeYL;PU*N$!Z=;)6~C|;PO&! z+4yhe|6X2Z9rx-HJLpbqw9{t;4OU<2v2jb^7fF z{WgXiaN)kP2}cLq+eX!GL#B2o7ctX%uHcFg2Ypjl#<5S*nsPE@*~LP(A1j)1wnz<)X<<)K|{T`1ox!o{ z{sH(1Xos^y&gnun^PSt?kav@q2gapeit*7A(dVZ#j=P_e$FXWVgQL|-Iqi0qQOZU0 zQJu>?JjZ&1i@l-mCtMNY^gS0o;mNd}vs(4$K0M8^uVQ(b3g}e)S}%6F6*wl3m5-|` zO^n6eQp?>}k^?y@ik#)-3PEY5oSdIgK^>CEq6~#t4<4GwGDvk}V*zxn(?oCPiV&xD zm$ybm5qANB-l*`r0F>XYZkl5~6t-QA+1>s|26U~lnW0-MeAnGW{{CSlYy#%va7GEY z+AUmeOsnnSiV&pNUFe$I32&gDWCji&gI1t7y-^XRCu-AZ!d>NCiTQF1LZ~O;ZK71F zvG{mB7f{2=ajpnKPBhr(AH)WEPQ|T9nmhZPCk#EGr>vnOB6>EljM~VObeA3 zYF(II7@!6EOjnE0leOjdH*iH%QZpVS$9)i!3PmT#V2ChU+{TS%uT!BEOHD9$yKy9{8?_-vK3=q*-hc!l z`xm5U(A&z?Z>Hg`B{h$(8dJ~ERYU4`^wyM`OdmFbLaa?9?CVZ9!_1{_fZM zmo0SFl$uX(i&9hQZAxk?y)~p}(%bCR0)6!~{lUD{Y`U72YS!OP)89R*h0UR>$*JGd z)$G($^wyASqz|X0rqkQisaf>anR=GKyE;WH{rj&hmKy1CF}c%+z|_;0X6L=MEb6*% z=l;pJ52r5Piock=fASr}sjJ=M{>gU^r_$cV6kOcxT};Kr2fd4FxcIVnF&!7b^DbuK z!rG<<@1Kc_-QLBlJBN#<8L$|z7epLDiz(0|+kqcXQCi%ii&LP`hjei&ieGk%O1D{? z)UT)E^L4s79mONMI0MBOxJCMPu`~-4odFnf8{HdD#SO2R1rbx#4P&}UH++d(RCjr& zcR>jJL|>39jScEg@yGc53VlJUvz45gD0b-LEEG@Z zVgrgV)Wz8-zC{=3p!l#Z&PDP2x;XESvWIMUA95NV^8Jf&hiU2|zo?7!kiXGIdX)1Q zs~?c|ow`UmI=VmohJow`WR z@g7|y)&D~mp?a~jz^$jQXOh~3a>dSs*cV*{u``v}x9cK_{kU6H&-0YNpnEL8SpAyr zafL1tJTYA)c<#_eg6ExXQOWo(eL+|~?H1MN=d~(&fI=xX} z5GWthMZ)2KbdhkFb)J$!IIPn}!r_Q60tdt97Wdu_xc9s0`v!IIf7eC2!xOqlcW60Z z{Xq8zo4pIt$k7*c?>pS0Qv2QB1^wv1^#y75j9XMcx?rUeJ6++u%e$bu+3Q_U-F%(C zpeOmTTU3AkHSdD%G4}!`i@1)uMf#C>;H~Zh&&C74iheO$J@EayNDurKU8D#8ty@(0 zxx`Yxrf1ym7S-o3)ED$vZ*+_5^N)EKq}Ts=7xY*QS1C#KB-?b6^u19RsXE@Ki&W{} zri-X_<{7rR&oBqi@H_f`j(P@bwJOpx?AAqk2FEQboz8j}q|vT!`=IsfW8+7wO?%tc&z;@6bik`ct|{T0iaPL6$3wpTsyG8Z+7rhI5xZije^l;}>2Q&}86sl&i)Zh)g8(i%hq5YL}@RLTR{qxeXADur}Nt&{s?%e5JP@{0X3+k}XdKYAsyw|%RtK{?E z1+0?Plio!GF4oLbq6y4>x=3L5>mq^qDqSSV?$<>^=_|TOmG$?!NR@R}qY_3vDi_?s~^y_t<^<(wrkv?Li`qeK@a+7U8Dzn&@HN;{7hfaQ#ZFLf#|9M z;wJBc$V%u7!sB+gsD5<6z94YEqKg2h?0GzUrWwz(aXzFstLM2^7wLHhb&;OuUbm|Ice`=!264cUj^vu9eM z?*4_iQHxSHtBa)WJ-SHhen=Ndsqa&9KK^JHS!eUrAFb0x`lEwxkyI+W=6z4H*O>RE zW?1}%oa1!s9d9+mQIm?BFwuE!J2FAOo?h;ym%pW#_tDEo=;Z-=d5~WIm0ljEmq+O3 z3-t0By?mQqzDqAp(919AQ?rO9#iHYbI~^|K^_IwJ%bY zq-(~cNry^v^T`-3KX+1E=wWlc(}DLfuBRglA~v)RmF6H;4K_TrO?@cUGX;O0`o=7h z(Ui;|b;`d!TWaZ!_n|f-x>0v*useej`nYFlVc^q5Htv1V*o}<08UI^=QH+&lW@9Pj z!lb!&U+4Zwx=X_e=e$!;k3JKFBVGF6&x$8?1Vj3o{sf)6`w9K1G|#(2W9gw!ilwRX zfdTq;>ZLXuwwG7JH!$D>n7RYM2!fY-89q9cdO5w88Wd26bB6~2R%5|IdRFXx9HHn0 zdBWBEq%+rx>(ndMpO&GRx(lD2Ej1VrZi2=Nmt8KD!#4qz6nQOrh)k#bgfrgbriM}5 z>0xMD<|a(q^cyY9;EI-I@J7orc%x+*ywS1@-e_3{Z?r6fH(Hj#8!gM=jh1EbM$0mI zqh%Sq(XtHQXjuktv@C-+T9&~ZEz97ImSylpeGuMgSq5*kEQ2>%mcbh>%ixWcW$;GJ zGI*n98NAW54Blv225+=1gQqW+rlg3t*3@f&7AT$BpH1NOrC$e9#~U;R?)UQ$hP@k0 zn5x?9za(&m6gER@#cD_)Go*kSQn(B$ScViTLkg53g~^bDWJn=0qyQOGcnm2xh7=lS z6&OPbiy;NYkV0Ze0dZF0Fr;7@QYf5NAPgxC&MFAbDg=fU07L5eht%T_6;&(fJ7XR{ z^_W6Iw~nc~ylNLNH13D_f*KM^bFh5gOY>sd{kwJ8!*2%m1t82A?8RfduDK0wTQmn(oFoZlf$8a()4|*PTn~jyK}fS z3;bxA&|na;8r-YCov9z;9_WhtdQ(3}sWkVXV$v%A>lt|j3P4!2)Kf~I%0HRYrM|X~ z*uBoMzF*)bsb}KY;G{;4^SV#i4=Dirn zsXx3nlYGJMWW1oEZB3<6UuoE&wsv=EcYI6x)@@tj8@IQU9cOE7^VXQt zzRlU#vu#s+WBV5Tdc(HZhIp)_y=P0Ty~h<>Bmfr2fO#{$EWpd%7$6d9W&WT)!3U*z zyR)Y=>8za~B=Z`+(tI4+@1@1kZUuXHF_#1tr3UJbyWPRGGza5BH~e@C&KYV#7^if_ zgDHsawY!ik=q{$TB=`m0ohT#LON)Zv^&>Wqo|TqX1iw+@y34*_8vGt!6lJ%p_+zb+ zS%WVv4*oI5%);P+)+qdyuk4)u!F+)p88+~6X^FZ*IGenaTdzM{PsYY@sqtU}Ll{2Z zrGM%TfTbpQtsz5oznweTjW#qmd=~vjPrp}g0WR=<>IR@rM!Oskh5tDj?0ueIzCbUZ zqnF3V$PZ=;_L~8k~x?gNti*f#pY{U3=k!p z?if12lXkWNjSKbYKoNEeN>y=O~JR07!;tr6=ammh}d(c*oUPZxW;X_pE&Yv?QV zr@OJ!Z)3tj!WQ(vd_zz(r@KHsYmz$F;h|FVX*iVSyK$zy8=Le+geBbnD0<^!3^_## z=r5=fPd!co8u4Lw!b!s&4fx>GpDD3~P_kzb_r2woL5(29&QJ8)g}!cpnmyg^_IpF6 zmhw;Y0|@X|`O^fN5bUs_(gOX{M7BueCNsE=gXeG5fe=uq3wo<0P{R6!{Nh!2(KB#|i~Q$QjJgTgjm|uAp zsLGp(q2lHoxAvmq9*R}3xQ%fv7-jpr;ob!uRm+`CdB3R!^e6BzbBP}68M;q8gXrm% z-f^79Q@mAtG5;hgrIYT?^_~H&NyJIRqfVMa)r7_XW%(wrhf9asu0_+!xApg>(2AT)+c5{JMC(WwC!C2`Zz_g|hbCA?>+`1I_ybreaaOynze1Y3e^_{4u zQ=KZ!RgVwF)C?nadMbXpq#q$Sy9aVfisLfaO{sBm5R@NYTbid8z=Ppoa@cw!(Jk(? z(Wua!1~&L&HGP3cR1dAN@xHl`zFFvgqsJQ0+)7#Lyud`0bAD=OSRIMi`_4& zKTTj5bF1fY>Js{PUUFdYbTZNF6!P7$e^cw!hpx^&*);V+r6YA$7O$@8Y5;q0HpVbK zYIi(Ojla=v80hg!2AJ$9c>-2f<3HxnLn@U5rea4HV_BXtX2ZXZ61S=0Le E58s+?VgLXD literal 273661 zcmd4437lLcJ!x!dSKisxEm=awtCeKStIJyX!ZO3#nYTMLn#=QM zB&}@$1_F{82!SPu`G-FwTp=Mp5=aOn&H_ggtLmz*uBxtn<=nTHq;r|k$=dwH zNTqxdXOnNii!%o%Yjf{+owAEN`E=GPxK06wryV;948|vGi;wY#HtxVPy z9(9tHYS}qntT=diUSgE6o2<3>OC;RM%H%2BaLY-;C*9tY?np04FHA2=ccvGoyTsjO zu`rfFGm9j)-ub*Cal`+_xvo>G@<%!sDnK(+EVz@8;uDsNHxrfO_;}74tyU^Ud_lLo znk<$kylZXw3Ad6cfCvk-)zU=8DJMpY#j`l|K%|{f2cPV}n%2Co>2kr&p)rtP zg?O$XSNk&s+!5#g$0SmYn;F-Tq1Fb;ba}uC2(WBNT&fQ5H-%+fVlTN}ZB#S9#U3;}M z7T*&GR`}=wI*OZ+I3KTdR-Al^FmSoEFm2BbaNij4dd5ILQ^<@J%j3@W0rX9GAU}aW z1L#--pl7;R*xrA~fSai}@sgc93nbkE`7P?x2bcgM*OSG3saOyM|FJdvc=KOenn>9d zTM|~VTYhuc9qp@~T3e}H%tEdpy(h(yRj~7pRVY?0^nXZ`T?Ls*x{nph5CiL;EdHkB z_e#O-mH-me@5`C-G~SuBZOgXp5KMI`{o-2NIdoUZ&}3}}`b^o$IkxLerVrHSR*ETm zf{xnxC^43=K((xbltS7QUhhkcWpWPT1~GV|Tr5^D_CMj>yZ8ho;^Jh@8h0w_(3OOp z%N5Tj#;T7$K9Q)D?ShML#`1f)wpKe&0QvE<mCLn`yzP>pvd)^p{=Ik$AmQuCiuIQQMQX+5LH_C6@g-$^N|lOlblK>HDg+F6Tlr zS4}z8<4LlGMp?Fbvv?+vOgqW5r6QVz5@_!~I0EHaEaRKoPp2W1Zhs1Jx@+?fZIwW4 z%j{GtLmzkEanA18=3SnD$UAguU712dqqSL^H$!3v+1-$BZ62_q_>NK__D+?`f{V3|V7xc~vl=ZU8HM06nl=s*WbIHgRkIF45N4E<4@ zEn%|Iz=rOX7|Yq?MD5NJh-H^2&^ZP5W|bz;1%+Nk-}6Q(eAmLfT_}-0OjOG`fS+H; zR7#oT*$E<{JWgP&pw0zcmIW_?591gZaAO{Z0vazb)s~8*gzF@6grTN{A$@peZIQeV zEuFV3EE{cjhA51091%y>ZNzjT(qPKr%y>Uvnp?3)iHY-1(_g;Sfw9Ahk7888K-qCL z<5n<|(4A`S=Dd9&lSkt;%GqQLByyPoew@NMkU3sk3=Gn8^dcod%;QVWV<0=a8KZc- zw!nK%e42miI;F|l%zm_6$+-z=tjyR1K79e5CD9+t5KFMC3i~Ai;=)J*EEg=PJXU41 z1QMa0CTvND*^eE>M=sNENk!l?!P-4d#v|&itd0IOpY0nml*s6q4SqtivW9U$3DJcU z7(vsvn{lB~a=cUKj?x#f&b|At`E2LSsCDplX%ft#69|sFme56mFK1Ht z%*I5?>Z=2oiPTOo95`WKE!mu8?5)MoV1`1U6W* zoM8<^#1I!@WkRB17h)*I#cqD`d>M+nFituZ@H#NeW**0Lc8*Ty`vsfQn`?{Jsb?!e z-T9AbTR=;EC@8+snku>mXk;Sy59Y&o%c zIYuhD*2aTl2KMKoQTo(!8a<1&3Wn@XX-#UE<^pRNwU=r9&)NQy7-^gm>v-{MwYHF^ z2rz7EsC5}w*(nua%oU;G=%L#E=}M*K?j0B)o28VVa5G7_e>_t`YyFwxK;1-P0B{EM zNy31bA`B3r2AE0xbcmrqYiJbaK{5@#igAAul5+tj0uM>(zxc+kbTLl@_Be5x4e?90 zg?6D(1o_bYp#;cwv0Fj(aiF#aVp?@G&~@4^sm*bFW!P;d z#d?nv{u4xjEX-Z*Z2ei%kIShy>fa{jCg|RD1GBV>_zXEefpX;T>$+8p6aK}H40|6IxCh^eNo{3=C># zvi3;J;H$}@g8o3uptIm9zz??!Fqtms=dOa?KoW2Metkme%~`~})g!ng^_>tz(h_8e zEEl~7)_I~@!i2|xp~^D_(M?>8FHm_Te)3VzFvjeeEg`zIlz|9=BfXtTLj7No?Ufvu z2cq9!sx9Lu)ujaY%G%5r3@*_N(fun@_g|LRh`$LjR{&H2g2xVVjbP*Y3h_QqDdwS{ z4Lz`AbD2C`2(0XvX-}|qkZ{k969<-RH(bnKlQfjqx>K2R{r<$lf?b2@srIC>IC(}j z`OuRcGiG>mD+^YFU2;x2lt{YfAj4x ze(9}GTCBm1H*V_PXAKNkw{EoZ4raR;=^g7lj6uhOp$j7| zC4qHZ?C2QSyt%{LY@K2%yOuPlQWmk=8Ut%B+d2-5q#sYwgLh%lX6LOZ=-NfAV#60x z;Rmcr(L#>|;pkESIf3>?53n^fG6GkEAedDggKGiOPVdT=#mnNGUF(U1CytFUH7;7G zeT-fvnthpd9)rKYRP>S*y*<$3Q&r$YFZPI6`n+a5;-Yi76)Ty8vN( zmv=x+qNaGpW?W&oSTvIxn_$}JGUpr%m@>OEV^)v+7OxTFA|MDHJq%Lf)A=2GfZ_K) zR&~k~Ba%LYxttHGAARQ#T%kQgFl+yv{5E<62}e(F?>>feQSUGUV5U%T%0z*E%jg>YkPu?JbPh$sDGIPO9H}Fq6w!qmES~R|xt8`Vco=N!u0cJm?^w z?7){n!&GzpOu-s2+oSw6_aHI=WS6x{83+26n9jEtka)AOsX|Rb(WR_R!75fOrD{cE zBTv+ZfT&<4zGjG&QK8bu=nL#mvL2!*`tTtTCFep3JRP%huG43ct|18GsiUN5h~eOG zpJa@6Yq;bVc-7Bz#9Q|;iMR9(WX+HlXIQ-|WvSXCa3H@ui35oC~&P$ndS z#rpMO80!z;Bhcn&^-u6|OO_oBu<{E?R-vnCx(x+Ks-LYq^h9_*Xl_o;F?f;$j}aav zsHY1`pt*u`9*T!m-lmuKY_e_-h0;@+`Fll!9tgSqf$9u1ztd{5<}~|N^b*n z@ztO_^n&P!L`>=yk36clC%*hJjQ$kva351Q$BPw>0gX2m!L@s$MIzqa#fs>QTAS1* zdekGH5D||d2K_EsA3dAS6gKthl;%DLCDj<1^_Qw{x<`9G_$*-~bef=mYi-=9H&%}a!Pl51p!4TEvhC3z z=!R}X@6x}9_ZvS!=vDE!6`+!&ty4(34`MLl&nKqn+M(ClsLLdGVhl`8DfWf+QpvMp-dJa@2w9gnz9|op^m90Wrq4z8}O~x~reTw%OqDNw1tomKd z;Cs%|Ac`NL%A>rFEsal7v>-h?RkCo#3pw*fTy6sj{1gwx7*OFZ$bcB*q~3@qq%iGs z#Z1bA+qxJpR|~8rdHBWPNJb7&TzE|Oybo+;gt(kY+!{EX?( z)jB*7#UB$tjcO8Wl}=Q>i4J=sbCl6FUlzrlcbgq@^jJ zm{5%(DasLsz|0wSJuVW-5U%PbCM%(!W`gK-P-2fxiTa79&$XUK>K>5%X;4y}}a;tT}aIZw_tOq!lJ z19>3R&ZXGu#V6!wp(|wX;C`&DAlYlh zb^JOG?^D2;URkX!l^x90 zA>V1o<{%roX}5kCrVxBth=^)eE1A3l#WXp5i+=Tdv3!=obi;G?s}zSQ;ws#A3aq@S zedMf{u19o`%pLjt!AB^_$d(+Wpp*{`_|KqtF_lf55az_jswf$Sp<78!%OR5 z;|@W|iMZU8wa?iiQg&6atI2^`(ozh_cZLKaHw{4RMb;S^$dHo9s-QDM<0($}kV!F@ zYq!dWuJ!>No{e6cxfQ`*b(`ZFunp9evm$64XoKeHP$6W!9lu}q=5 zU@oXzNZo@)Q_idOj1}r8zXvm1*sY5Ps&07z(dB`3hSEnT^(JV@rGyXpY`6cccLRxE z3NVOCsU##HUgl&G1_)xHWaly&gJBePoz$E+F91Ov!-xZ-bE*?3Bfo zu+PC_(=Y2o*K$3e2*u!q4Q}oiH~Or8I|WJM1g`#6xmZeZdPR5OJ>gOA$Dc^R?PC&Vm&x0WqhuXgx;*z#_grK9=_`tl$es z^7HtoCEYF{OQDb;I|rYLyyINXreq#O?%RlOl&FJTvFa$M*A;XjNlpTE26oojpxjYJ;fBiY`o_>wnfN_ z`5q5B3D}|u$auBlOb&m#wuJBK3}BN=&>fdmm<`%#<`ek;;^b?+#v})2#wL>-jR_K@ zh|4mg6Zs@?1>=_ZApMre7YutT1qTtk0K|uhmr}(l^)flD#lXSni@-952%c1<9YIlf z?q`W+hp5K~x+|Naqs0ptwvkp*7)>PMpej;kBlS`y#bR6~8gsHP25!nZK?)(Ih%TcS z9CX6K{p|OtnMJrKJODJ>V*roGtC&Mn;>axCcwJfO%PY`~rxSfro(`TY@{zISeDD zzG!ryv47P|AF<<)55`}TcvK#2iQkcUbn~sSWa;?}PLf4OW+crnj=_RhEoW2mHgZ2@ zj*vT<{UIT^GVm(WFaF^Qr{FJVB}WUnYeoRLb4#5U>W3>w4b6a&Jy?uPJ{ zcwH5S5!7iyU0jOC8GvxUvQdxDHiB^n0UJaksjpg3kc9$$QUA0Ub(Y_Q!6@WuLdCO% zMN&1youy_C?&L9Ff<>Ic+yfDQ${<8mF7Ed72MY{A1c;XJF@9-Isa5J8Tvn8Ks@q3*q^36L7g*gw=jn^Vwxx3Po2_Ncb%~{djl4hTn^pJGcWh z{lxSMp1Of;{oDI@_!ppq$$dw#+9_>soM<;$+wP|AZ98|x2c2yXJU+0)Ihi_A-Sv`* zBLh2Ma<}_%Vf6O=mV0w2^1~;ym0b_LrxpcS%W)%g^JiJ9tV8}_?D+3PZL8-W%PkBhMB8BMrY`oXw$rMKL zo2PAZGP`va%#UsOsa9*G@gy6(4kYyrr^RVLA>rquQH0F+kVP+W3&O%16;79-6ISZi#gM?>2Xc7?-tlzz@!HM$E7DLS z|IC<}2t5iz4}!lJ4js5Zmbu9Og528Y9hb5*+0z8N&~-WT`qmVWhpV%r!W%{WgKvHu z-Zp3#Pz_(v?cEf5m(o#dbJ%di=qKOvoOfh!Vakdj9mW%<@dmP#6(?v_5-bVWfGZ_^ zbXxY?CFrHL5j=DZ&!|;rA{!RMoF2|Tf}UNQe^9!x)I)6`0*&jF>0xxl>L|@}+58l5 zw4E*{Pt(L#K}9ndHsB>4OfO za?WLteE?r4JjfMVRgjcX##4@K!#P48+|y1nUC5A|k5e3uVq`*k#b9pSL7MPn_DMjx zl)Z>w{*vs=`TQ05y_|g|9Z!RGCmcy7bT@C|j>6fy<1N@}yt7^V9zg@|ogEE#R?Bv1 zfXZhc=Qu~hL*DBg;Q(UimQSZ%yG~&g^>vb~pJ`t5WT)Y!nX?;7=C$NWRl`ejr+LW}iiVfkLtdI@ z3}|?NQONrqrk>z5ywp9-OPrtFM6L*A;>epB|zJ3`(K>hK1F z><@XRuDL0D<6y|ULCxRL+@X+H>YAIfA0G*MH#lB2GT z;Qfs+Rcmvx%DY&Z%%-5xaN_xU7m9Ty!c&PkNp>_&R{kUn3iJZaNNA?9SWKXB=10^q zNvSQDxX^?ILmfPm$P2wNnSOqXk^$NK;54kz0$0t6(v#ks-dCHA+756Er(cxbpB=*& z;P6hA4iI^0IJqnR;%r7-A4E_>J~E^Z530jM$G!XUbia2vklx}Q9!}rw9o}=f)+Tdt zsP!Y4y&Lyp4kjAOz7);+XX&HqW8O2{(%Zemk@RWr@R9Tx@9?EQB8f}hwU_xvwtqC5 zPI|XfJ_e3|G?pIszCE4Jcu!{2XT8In>0RCfx#QkZ-haQ4E_zQs=3gwQUGK??k7m_B zI_D#IK7GM^d&lwg0Z?fo{W$-9!hib7%Y5Nt`sMuh73o*<-=};auNt1|0eN-$HQwPH zd;o9s@%l;s+ME2he=7ZE@0GXs7bpD(-|8Rzw14+!(m(6H@^t!b-r+BJ&B&#}l$FP` z=~}lmpHx@I${&2|yZrUj>G$CD-Rbx6-}k29M}OrCXJJaE_wgmMK3Zf^Os4nqHL(!c z&n`^&^HsSZ8qo-(1i>32AC|5KB(RI38tE6))9GJKzn}m9lKlJ2>Gyk|^?_dR=vVxs z|C9cpckx%#za|f}50ORlFn$=+ucv>*d*&m?q* z_95g^FgQrYyfAToMyM+#3#{KoMgArkHmF59q-UC?4;ve+J!slc&~P0KaM?o z{mj&Do)gu(lf9E(d9n81K^$Mox;(IC59ud|adO$ab2s0SR$sVB}dEs6<9;P1& z=qMc@(=HsR;}iH{nkCR&_GL7x5Cu=+GV9gs2t9S$d+L51uv+y-Uy?7RO{I`9#4kSJ zzxW`I#I=WTlzkXK{KQLez#y`Z(DgIk^#q;0%saE`Y}7kDNoO=pGX#gu#=Ns}I!nv5 z8mb6jv?mjRGbiEu2NL|p2MN|=JPnpUce7ue;fzBeDnm4{7g5#@+7W^cP@HIh!r7IO)L_In-NBhSJp@y z32Mvb=n$xoWWlML-fs`jghj*PUk;#x+^_JCP|XXTBINK;l-*ya?5h~gH{(pWdebTd z3yd%Ovs>rDfLnJ0D-9@ih6Gr@;)KZMWZMvlAjdK;MHcJ=au8iqM-v#n5_n~w!Vly7 zDmr_$clH__2;%$%U4N~2{dG7HUv_@}iC@4&IIOx{U1eqOu#3v~Q0?ZUh1_&wT%_u|-p|9v^=6I z+MSw~mxfG-l1M{@sVzY2`DC0!h(X)Gx7hzyWTSF5iPfR9br15dJREB`sh21Q%^6{c z4P)gMqMiQ7Yk6<}Q=)tloLBN|KZ?)DevE$Ph2Ns%-_|ZXgJbWjYhB2%6X|!;8H!YC ziXWq_mTo`SOc|HUz=@pIaRPt);dvb?GPF z`pFWUCD?0l{ocWOcjo$eZ z{YV_2$FZ+SU%<)mr@dQWrpLabJ^NKU{8!uISijpr9t6jJm$Ns0>f|FkF(RK8c zr9HZyjyGr*Vsw0~cHuTU-l$#JM8`eagIC+IKQW1Mb1(JCPl~Y~F zInA8Qta2=7u@-hs?CiIE-)R}Y|d@8bZq4iLW4M{~P+ zaGQ1+#{zq;Q9AU~DSYr6fIX9Ms&#QDyR3Z179r9-YF?zl_x}GkS^15jgs%oJ`%ix( zaM=$A{4j9Y_xAsQE_*Hd5C4yT)qi%@TS%u^k@B8 z&TRczKkD*bdUm?7)_>6hAu*os(Dgc3yZ;Se3c2Zj%C|#q`d{>)AvgUG`##;A&k{hT zV6$JN^SaRFm+4xA8~-bGb%ED`_~n^IkC1TaxA;*{PWaLP;CB2h9lrMOd@T?^&wgE9 z^S}0+>YD$NKf*PXtfk34^-_`fgp7Oi)+}|->^$I>rT&<0qa*5q+4*!ty)N5MN7T`> z9dtyUE4zS>s6%BJ(hqf_>>@g%j+5=ABkC;K#q>iRB-=$t)G4yv^g|sXyM&I2?%4Uj$~KU5%D3rhK`8#*_-HyNS<9wKSbp0 z&2&Vx&E7&sMA7UzIwCq|Ejl9VW!KXY(JZ?GN6_u*L6Yk(sj6k#e}8gvc(zxJ+bbX2 z4@Ff#B}xY?1!sEsV(MmwR3j2daL+5$?n6Tz_ThHqy@~O_yDVI=cwUzF_F<`oP=tmo zGw$D|MEA+zS+(VL;6#?}XK?t%AaeE3c**_W%2;0GapoTylg zoUbK;>sYam0$Pa1cuORZNr*(t3RO%H39g;Ayp@*8A*Yhc=+ivH3xOO&2O#WYcR z5|`b6@%^+AdjJWZ^rW_65X*wfV@d7Zjo!PI*owg$<(10)8K*MF03c1GA|)WvdjUwD zoHMG58l<<^I`IO}k`oswL`OuU<&>26M{{TYYPYw@5{af`yqX1%cBE|-cjlGblzO>> zmp-9hF)|;~q0kPkwV^ehOZD2cV!}@Of^0RVXQxwGuH!{6I5ixzc`UaVFpv$~s6>+8Q(raV4{Nw(4tR}9sO%!h9Gg1{-+P?hcMm{5TaiwkWflYiyD&tDq zqi1{gjMT=Jww@EW^BJj*D{cSK6X!EhA6MEw{fsyx6>_C*{VxyjJyIiA+P2tR_>5G^ zm9}r*ww2FFom^>K`$KU?D&I-4q3kMhF{z+LAxX=Xk?@V&fAlAG2Rf7GvQZtN!nvVTgc#z7TeC?! zy#@K8r=b3c5@atvB1^1#%RFF>VF`&dkleLx_nuwJt#^#>+_NRQYuENY+ji`+?d{1i zXLri6w{ICujV8BjvG3Tn&Drhj*lFXxo%U{f%g*h%vwaU&1bq|h?rEDD-Mw?y?yWm_ z?b$Lm>g-95j%`hC*}Y}^?yaL^+wa)9W!D`$Q>n3SJ4UzN;Uq_Q?B12!v1e@b4rk}i zJM6J7dmPye(QCS9wr)w;TgG-eyY_6~y=PC--sS8$+t()qIkoY6lk?K#U9m=g9kO+? z3Xq1Wxx#^W?47rwbUm-?!ej(XEm?3cPG)leL<4^Te~)FCXN&kdd=u`IDA(U#u~6J! zjQb?q_4hx!NZen7`y}r5_q*qd`>3?dc0>LB@tef`819owsK0-4jkv!7_en|A-+x1! zf=_Cr{(ing++T$ENs-jw|4Vtl6Zc8A)Zg!K7w<2`eNr~{_t$9jA@x&#ze~~w^?BLO zX}G^az+a2|q@wEY+q1>}Ik-=vN(}B03&q@$lVpZ3M2{0yl`z(JlyECe zh-Z5-{CZj+lO1UV)aLM<1$#$`9XPOJL5&R*$6)YS#E^3G8Co$VmXlHEc?)P!YmfS=vW|s@u<}B|)?V08&Dx?ZAbv*yvBLTfnbVIcGo>rj zA|U^++BQHg{h2SZ*glzk8(}pMJBjpf-8oqe_VhQ*6_jkNR@tVcuXqDCR66e!qqwRTB+I@S_`<^L0C@LxfY-;SauG1OC)D^5$@uZd7#n+8 zhl^!wev`6}qr@G`F&=j+uqe;UugPx0JK5d%!Ixxv@RvI2X8Z-bKJlkt{MjP@Y!iQW z@SkDK`?I@n=po3irs)yfHd4%;Bm53##&}bq3RYuShfuSH=Q@@sF&S9S5n}@I*5hd3MH71@;L$Tft-IIwpVvJW;9bW94n;x&*Phmb+M$fIFxFzF+ zVuHk1cdvB<)*IEQCmPe*x&ZZTV=TMtuw=jC^jNMrMN6v1l06hKDr(Eu0!S5&k(yJ7 z6iqs^(<8M+TURI-rv;F?XpGFPVebaOV%%X z+-vcoP1_oQEMhGrt&OC0o5$?J1fPiIXNM3;M2SVqTOQeG9fMPpSMAR)O{U0@~8j0od>X?RnkIg4@%C0fX&2OJrjSWiCXiO{CiI~(O z>bK{y5Xi-J8O5yg0o*@tjC)%h?i`?*9`{>^{3W%l$L}v-(>-^9cFtPccLZ?x{&nLL)R`Qux=svs1~8c4cD>e05Ce{qT_*;+ z0vIfB5(AFTT_*;+0~l;*5(AD3PLIL$?0TgST?0*`#$n3oQQIeH60%xBpLGwa$|6l; zoaBD6Tt@K{maIH#K3dY*ci7=w6obR;i>@2jthCZ+)d`jesX{iUo<W$i?s6LNE1E%d^sjq@JXWt1_r2kLr7v-s`lgHHk5~iLNq# zIV>>VDUHh$WyID|HP&iySkOmY;{@#KCeh~Xis=bGAkf~mHw01In}~ZJ*^jhp0)GQ! zeQ$I4Rcjq6QaN7dDq;tYaH)_3M5M`(cR{4d(0pE`$L%hB~ZBlc7E!(qxF*RqSc2 zCkzPp*n-h2g3@C120x=Q8}vf4AFZ9LG$k^kl2O2qrMjJ4FRjz z#_m<6^z?1`85&bP#tyM5J$b@y^BH{X^|_Ncg^X+1B0pmukf@e3lblrnS73n(?y}Dg zemNR`vpex472i-T3rWH<*wM zbqyI1&*$-8Bs8ooigCK0U#UkJG(E$!hi5tsMExSrXp*R3Fa_}pUKFhTJcF2$`aP!P zU;QBM_lAw(>gS0s4)9_jc9f2W7pr6Efk8auqIw{C7oB(xRk(P^32A>2UuHG(r5z|W z$(MNy;s$p1O@r{=(8dB-$Ale?u*Rue#8=apO4(yDmc+)o0ounSqkT`zj*lsP;vPP6 z>|?}b8hw0pR)h)3=H{8P$rOYVl3N8oTdks&dop5UN3LRuwuW?aLHW2ZG9E4W<-?9s z>v3*?tLY)}F=*tmoq(MJ@K%dU5=wLwq%fyBJ7<22E~IURK~#U#zE)h{1Mj^(`>M?o)C1H%EFp* zh&LVKm!2q(+l9>I>>H|*8{MkMYd zq78L(8YS|JrhHNZ-4__d4KV}dz%d8jyWqIc-cCX&(?xh;hL>Fu$itX-1~&3bw*};P z3BPLdgt_KDAVz{N+`68TAp5f*#l5fEx^hPLX&hzWMn62YB>bfIuC-g<4%`7e#3x)& zcfVFWzBC568MZFW#J9!$ruA+SW|GchjGI!`vrR!L0iGe~)@qY<=iv;yqcLOflU&F{ zOs-(sIkk*{Hr9Pu$E!)2PGNRO6~qJV8U$w1sQq2BF@YWG@y9XS;S`*BaV$=YKAkwQ zi(^58>%cT?$j5jnApDA+&Yagguj&8_(EHcLKjgbg?e9aSvaaUj%=UrIh zw(Aex#AR~JOAIx>wl(tgBc@DIeEqN~h{*Zc?x&gr`1B8v@u?%`*HF-BV)Gm6{_a%>!nPfL~RB>8s48Jz@V8OQCUyhdeXHUg_6sPn@bu_Ye1CVYq2(r!; zMC7cseAaH?YaOSp^>Wj=v5rK>KTGj%d%U3V@U2l@kuGRt(qU5yEBPBT1ra%uVm_00 z?6n3{RFXmFhcqm55uSv6WPFM#KJADj0f6KT8_JD4n&Vez)xt(rWliap}{ZwRp(&T9;CT|FNG9J)?9v<3wwy2RwZ#1Q_V$$nPLDVzJ zU=pt!#3I?k0EEKgW2UJEgoU^E>$+nR#_0wo>&X7FMIDh#X@%v|91P2)dCs(ls804Q zgD~(b8U6s)F_kXrF$Gc24};N44R}I64L~R?{z-`VQ~?!Z)j4<&1vhg=+R_wj zoT;1%hfz{Rx=^7E&hm(9Us2udAqHVE^qDH+wL_w?6|Xlyxj6bSc@_H|JR)4lIBC8-jIx0`~9oGUl8z37OHMrIB#@vopl9JW0Vn2gUw z#+zG&$>_+OEQ;m@un2G{Bq!@xQ}QYfecTj8E>`cU?})_Q+Thi(v^4P=URr&1iW%QK$rX6%wJ|VoGQw zHE~lAk@IS;tVclh^7P(^2P5Os%`wRgGKHu0Kg=2qi8P-!C9dMneWoBH=T9g6@zor{ zn_`#vfJ%EJGL9@pzDJSH?MXHr>69stj zj>veiB8EgeG#{^PJ;0KX`0>w~(o?bIX;TpOEHUsMDEEVyl^TFhSX{;ZU^+coek6|x zD;F?B9Tb=zd93*i8sMLeuwP)6glIjkx32Yz5VQM_O zBaJLnxx$~q1|;`CBI8g`%#rxWNs@+7yoglD#_Y$=;SjU^8>R$TJo}m{h{$=iVlb7h z!nKe_`QS1w_C_V)`l-{(MN!%AD`Pe{AFpp-lcEsweHjpH(tPhW1ra$%7Cn%0tCY0S z=pWq|86!GlC?ln{s?k5hB;R97JS7o3O+iFX|C{t=CySTDQ6^g>HxiL?V{J^wAzp`r z#+wjx{ER7K6>lCk1ra%Krjd82j~F`~P_7?{X|fYyv%lMv z1&WJ5ZwexEE^ed{?g8wffzf`53}vxWrOzsWbrkWk8hQ6ek@0RL2X#joop_Sb5m}wU zM#SG57#5Pr`Ds&XD~5f_6h!0<+kBV`dsM2Nni7<}Y0b5-N5-_N4@8I_g23Dq!w zPh8>SXHmojKJg4z^`o}Aw~W7YI3&%p3kWu8buTgn5jo3Nk#;2ErXV8c%F+XN1%l+Iu{YY_k3`0hWwB9e z+fQ8=ZEL*fY|OEJ$dr(Z7x$Zjh@2N~2ayX?tOlzAz9KUE&yS^Kdm7CbH`4q`Q-Ud) zPnd#;oaWtrDaN5(rbKC00pHoXBBTD27>fR-GMM)w9a6Te@u91c5AQT3qT<8bO+iG? zhu%R{8!N?Av^)xaGTI;vL%Ty>&Evs69|HV&E;9aXMiDX(BXU1`7?EGIZZ4#c>fMce zd)Acbif-P`n=(W3^NXe+>iKC9G*WRVu@Jxjgu>$ErpfVgod<8B zbK}-|G`k(ot&3YTgP_k2t)F@-Y|Y1dwA$pf;1&jP0~24@_$v?&VIM@=D}y#ZN;ye8 zXzve2MbX+Cf7Jo)#3(U9JqN5HhSFo(r8Lz8PITXi5%7tII z0YTP=)ga>$@d6$YFS<6)rjB_shnWlB_~cx_ViTmfYtKB$T)JdC@bbIppZzgoZy5QT@%AC+t^5LEdg2REHVTNu`3vIy2WVA@#0Q86T#?<0 zw*>y;s@^R^;`x7O%2U-X{)9nHX+pGoL7cK6P8Lf%TwyLaUM`-mqyxf9R<)kpN z^IV9|zO^eN^SyQ(Hi4()UWrWHA!MjtIU^IMyF-#b)&sp}?R--Zs?S`7TgJM(Z$ukS zNo}XFr#kPlj)}p^Vg_4g7A&tUV?e_1iHvQi;U|z19D+{OkcyWCQhu#SUB`yxg&Z=a zz0$D<7{ra!vFoWG8*;Ca8_)4+n+8p%OxyjSB2gOqGzJ)viHs2&utrg0B1T1D|$hcD%V(uZ#juQ9BC#WzMpZRw>%wLZ&?>Z zrzJEXu4-FlUrx0ol*Pj#sYE|*$|5DGZ($Hq3TjWwt)a!Ikv>Wl6uqgkzN-7TBBOLy ztb%sQb?UXNN#Xo3Dg4dmiTmrOAe0pTn&3~XohQqV3kSAEyOU&SAKOCRTH$|;j1|jc z-n1dmcz_=vPLsbgC8p}^e{Bk)o*xD(O=(3Tat0t27Ezft>wIS?%}+@u!5W;(n(6G0 z$_QQOdpu$MvT9^Be7&gKFCn>k3xHUY=I49{F{N5;xmMh*mB=K6-*XMHF>Vc3D@6*j z?xS5Pk#&;87Q|(`5-fZ(5E(;u$FxG3#1l_~#bm*j__6Ux=#i3T^faMv4^y&z&9kc4 z6oe9|jiw-!P>T)|rBItLed^bzE9Y&@D80FfFqLIgS`G-ogxo}AJ!&8@VIj19=@xeL zl%eUx!y(z7XG}S$28V}DL8yLo6}4>2!JVfZv1t$j)2O+sS&BX@#TKU=n)agA0TcGs zkuh=?7NAP(D5F5f&gZD`^l@m~$4=7>AvvF~G-Zb3=S5Qx_53stYNZ!MywU)K!s4c8 zd|xO*_(R&A_Qr*#ZyqWZ&J}aj3JeF#uclKk&$Osl(ietCpx?nGMD;+DN&UO z{I@BHYvzlwZ)Yj00Kn0PynrO9ay1A_Y)NZoE)9C8wv0*ucy04840S>h5{I^cO_S4= zn;FCn)JFrFl1JHNE{f^yh4(u*Q7GoguU1zBLHF_;jEwvP@V@$k59%EYyx{-#(3-L| zLs$6b#DmTA=1x-(N>KI*_O@DqJv;>y;YLrQcm{jvMkvv+RCF^moO8hWB%ImB6n3&Z zTr8skpw)92rS>z3SfK)4!KH*4w>MyWrXyqbgHy2E`v5;v()$38IOBu(11QFY55OPB zpY}VF_$&Vcb&C-F>6r3V>CaJ95cS={z_wK8sZbmSAQTo6jjk(CpT{Q2T4e3SXcno| zL5t^SBkNiyXc%liUX<A3J`AB$ zSXbxDgmntGQ2A_-H~$nFZ`Q@e1RA>ZVxKH*5+TsgI24jQ@DHZsRUG`V?j!^*YkNZ9kO7no3D&gJ=n`L(Mqp`WufD${l(MiV~T7?5R8 z;4Y>dlrR!Ny?$W{ZznkIBKxkedT}%HF-=;=Kjgk}11;l~r{SVMcKpc5X|D#ah5FT* zLck=tEix`)+=6>PU&zSq8sWE(i}p3K3(Y#%&^)o%nSxM)x0XS)JifAQWWue8DIr3h z*dUS2*#Jw9M8=ZkG1qS#O3-UOz>g3c^ROu~l_DH61ra$vZa-OeIQol;g@cLXHCGqU zN5-bxW2h*uaDqb1#(rrtxs0x^vrOc&Fb+Wl^9x7~iqNmk2cWXaT9WqYj?=TI-G zjNQ|beAuwl@8??72kC=DLmY*$5XWDb_7m0B{)|D~K*OnP&~mwa8TLzKzg<9mPOrUzdH;s zv8!2jXFmsnO>9j2MpH(rzVLbmaRZH#bsB94Q!w;gEHdG&mXP`%ABv39mQHC~;o;|F zDCM~%%w+js^Q8ZPDF`J1?-$&g%49JXfGrP};roa6eFEE(AI_9fwQ_p41qQ^=MaH$Q zvGW2k-pG245Q}RP6T(NGjUhJ9UzqYhNzk7O9$a@im}%}}r*MU0(<$rj!Mc2 zAw2~DGo6QEO8>I44q9`HHbUnFiKE!ist99z9Eq{CdLp20UbMV5Gbj4IP=COaVg7(S z&}x&Dqg@Q*1`fSDRTdWp+#?0-HGuhrD6UTH$4-<{&Y89ZKFW29M;@!%Ab5b-k3>c6 z+Ab~63u%#<=V;iG!CaKsQF$(?eOF@VdkE)*X`zRjXVd+rAe7{t5}clDm^K{=cCoIR>y<*ky#c=23hyjX(VIG)_o3(1dK2n?Gv<=Ra_ zM9#CFUIrW1k0MQ%G77C8FsoRu*a*Er7opZw87#DO6E5x{MB$W!*~Z%=IV&Kik6j_9R<@Y3LP=oU6h!3QTyxBEDr`aw70;313_FPkO?e?; zP&^nJ%WjJ0fdY?TNgoBF9WUsd35nL9HYKa#%zdUHBInGqqnW~4@|bb4oG`=UiO6`e z9JP$OaS<4jYEb7!$ddjGrleHdsG5R^oEz8bJ9tNAJgM8aYEIY^>z^~FpJKt&rXV6` z!JsD)Bn#M4SGsQGNRx5Bs!}fbA6zUfgG(Yf=?`eMPe#V61HSZ-;3!Pxu#@q_ts16v z?g?GgLY7|tzA2LwpMS>`MC5$#JBYxtOJT zUJkJIOOdfO9!p82;>8>$TtF&rbD1KE6cXt*q%`v|WapiKH6^{`-angyh@5)^2kZ&Q z{lY7sCSMW~=)pY16|T|qx?p8grhOc43*bd&1|h{3eVN?Be3T=1uqLUx_=RlJGaqO+ zY2VK^1ra#|_jqcyfst&8Tq7q3BI6|Ovmk7fAV&$i__eXLvvHq=K2v@und~(MQO{Dt zfQpFVix_|b2!+LVQ-6UB>&%*aLyK7~Ipw&Nj7YZfF{hF)rh@*0bc?zoeO72V=So;O z$1&{Gx4e7s!n zrPaZg6<7n9DZtL43PK~E5gY8wzZ@AKR-#6bXr7jA`j)8{mY*B8%K8^g$*H>edrU!G zGe?Xg#=2_3PU;P+<+G8oqi(i0H!P^~$EJi+wEql)m{P?|8G|cWtpGU@yy*@j_lm6JLi)jxqgm0HH^tbI+uE4p& zp*86Lw0WNWktqlzMxPdJZ}psXpJMKTMym~JEMqKP7%n^yOvOp23z^5Nb$K@5iHx`V z6mLVUIzmx6dwmq+8j89&hD~q3Wy%>PkY6_i5jk^@D&|`C)}Lj&h_Qum6+4=xkkmUa zs^jn|6RmzuIkarzNc5Rr2lTZ0IrbFVeX8E#05 zK;moJK_*1QH1a1RVyJVuH_^c1qQ+pJfm?aC&!DQoaxsDQk5xka=M~`+sYmr2k?HLW##e39hty_BD?e z`UE3@i`KNpv~K9RwKlP`M~nE)3!4&k0D&g0;xkWoD>yXa zwI1Sum=_iW>^CK*QYyEbf{2_SC%wA$WXI$(XPsOoT`Z=2|B)+pc0jRKk8HurP;AW2 zn_~C{S_5;I`y*rJeK3J!VS89U3UQyQuGkO)m0CT-we&M~SO~;Jsz=NVE8=p>l!dAn zoG=9uc`sQ0V8%u5T6ngtAzGk->dEy%k%`DyxgvJnYdxOi*6WPedBY*|jB}>MRQzyF zLDciZ!17bZqcAuOKqxFeZaTmc7BC*acVmsmUhJnp3x=!*P+AnLnU2|1H_&nWtfE#$ z(9Yz81M`Pk)C1{jLru!}hMANfH0>m+pM8Kq7?_mCv6)r(pk`IMl(Ruq_8;5WEwPfe zgQY?LJSu{sfL&T4z{(Z~z`Xzj{HbZUufKU}|B)%RP2)4G$fT*;D8WkFZg98lZ$(IK z^4ruQ08I)6bYh1Y_g`q?~I1}P$077B$>1g`H=0OYFJSWe> zWuekQkJ{LxwJ?rlnbxstu7VwZu!~P{+&LUsN4Xsxg;9cOs!~I9K1%*ZO@xxxgF^bx zkSX)kFm;zHh{!nyNp_246)Q&zH96p-X>WDFunrk3Mc%Nid-LR#adL{xl8nSzL% z537frasj0k*`z2SV1WE72Wf_N!O1s9#*a0zvjPJ#feT=u@g*bx^?FmHD!yDY1ra%4 zdJcNq;aZYD%%Nk55jbV66muyvgjQt*#O6bhv1wDx-vKwSGc3;t#r;KMx=st3;(gGR z;EHD-Fa;4g&$`KK%oj1~4lOSIxyYEaL~M>iEqi%wIv+x!MSo#RM8$_cGX)VjAL6P) z3W9jWQYDkmJT8iU!y`r;`ZyPqH!L7P-;0b#{aT$A#PoEmD9|UcQm!tYe<68@-!&z^ z;@`JTLDcilz%W#{rI<(?fKXVx-*mbnT_rPa?9w~ypo0`ks;^WjJ6H9PY+u`&DGhyO zsE6d&!#pHg&}x%YmpFqk@Q|3!*}FvGhn(IvlQtmOr=p@G_F3TKZXB++u`I&T=IHHV zGxQV96Zfbo2qiT~1boI39kaReP>AmN?Z{}{38R8ql@tz;HB9w0LCRP?E+4dpq~j?mCI+Wz+}2m`FTObhIe9w^WGBwBB<8bZ7s3xQ9Q zCRsa!xPdzH)(0|f6)W-{r`&h=E3l)F$K0x&?kx~8D*g1s^r^WJ^6l8?1Ktn58 z$D8NEW>XMKEH(*7wpxM4#BM$3?FmkS)#$~{BYFkRxziDHOKj9bxX|=i!^P*F=09&A zo)c!{-DgT_rAm*Rf~e<~fgV(OoJSC!BXO^m)I^uW zKG<_!6IsvcjoCgzqL8paC|)g5HFTbNv%U>L6(o#*8Y;sz?*+sFesG)g-+Howq3Bi>;OLdnn130}5Z=dPcE zV`^nlfLou5j9VL~;+9%Dq>qlFg>C-8l*~$GK4}Ufa!#!p!g_YpPn4$2-I+oLr7}}a zu;%aAB4f~M)JB&G@DMl-1mXe#ZAuYZ^ybT^L{)tGk|~JD`LcgR?5=up0=0&a>`3+O zD@9tslLCJ@>yble$eu!-?fi7sXZI~p+5C6L4+iI0EEIKDof|)LsWFFG{L=zW-~_)Zdc=M zKur%u){$$e8k?n3m)mXB^|eQ7p6!8lB8GF}LxGqiNbZbh%EVd8VB) z1)-$sVZrxS+do)zDL5u`|ND5VDT3%s$bekEHZm4kFgapM1KSmK3paPgQgY7?7{?}2sL!*0%Sk$SSZnn-4hDq zeRyB)4RddmUqm`WSb^oW>sm9_AUcKSb$lW$uVXb@ZE~8hfW+FUD={vH=#2f6mjxZIqr+L!vGzFo=a+}~@tF5@U z1NJ!l5#s?~BqHO*e5Bln_WXgNBg`Z_V@fbJ+&*jy;+ko09K=Mz6BEHJt{60cUL6_r zDdCA5_Dp5X>gfne=Xj+l4HX+MGKeX)PRp&H12UUqg0^KVo-dGs7d4poT0?1CT_W9p zN1byHa!kdJqJimXKo;H|88;4iDIkPu9H_?q;?41-#FXbUIl`2!RX7a-lTtOc0sF#q z*UvZ4!2j12gp#BG#}tGzY(zI|%CNbL1~r!E&Qoyz!Qz0A(s3(6S^mF~^`6BsER{mD z6qn&z#@{HAz``46!a;fjP|!p-B%HkgNHuBtuVWBXif-#o z|NTd>)ED_#=THMT!)XP5dTJkp#2VoZ=u>g0Tt?Y}ArQwp99AdyaAZ8$AIo@H#>WJf zdg33DjJrCLi2b+~f1yW!YhkiI)I8trG6kVzd%s|PtLF{tRSMEYC<7J$*(?)x!CvK z@>7HY418l`48$UAi4+;bQeR4LVZ;SWVG*;A;n3ju>rF|oxOd4EL_PNmMn*Nb3iWLO zLSbr}?lrhs{lzoMdwSWGb@}90hLV~uxZQ6TO2mGihh{##r>+_E_ zJiAc!`ejq(L46@I#`cCVRIw?S;^O}rXV6`)MBOgP`=L%E=@Te8CSZnp`o`&cwM=vjYJ!5DJT>&A2}h@tl2Qm!aDAanxkx>Ee02oZ@Oo78Dd^4>nXq`uE8Au_{&(?X%pJl33_2+t-*9Qc~#IP7iu^2A{DioNFWhP^f0Zo5QsHt+qav7 zP_i(OL9{$%z3M@%!lUuU3t5vm&^m*_n7utR-mH$D7YM|?Ks3n%BcKaIn2y?FN>n8d zaZ?bH^JVy8#vU(XPnXQ&EG`yr5axh2z6l@{Ii;5sM=O-z4lRO|R?irtbw6y31xxPPB1hu;ghzvGTZYfCt<}MowL_@iUd*8o5BbzA+2xN5$^?v?`M0*l!ffdA;0~2*_mQY$ zk1{tGg*iLt0<9*E%~_@(BInNPQ%#J|&$JD*F_IPX0<{JXsmb)9zT@Je8gDo7Y$p;;Vbil%9$u z1qN{gNA!&%NFP5ZHm5|P7vXGWx160s_Bx*jMvu2fM)-|bu}L`t5*<_u66lCU1qvIT z4`IXno15pvPnv>I!ch~g zn}Vq4hrzT*&76dM8h}t(T*de^R+$z;SC-OSmZmqYo?}N(1{2g?h^&j;B6eH~Q=svD zuB4lQAs+UBGi9^tDgVMCrqrP=_aeTC%b_3&O|J7i6+z*F1ErV9k6tZc0Z`e6JdA(25IWwQd)(h-9}ZyOpNc!62s8 z6s=#idkZrW96Kj>swQ5d6b15H%GF#j9qEzC*t1TsN1(x`70{rH7^t?&Xy}9wGp`82)N^q~$=GD#v$kgE?CYcH}`Z$VSIpscUEK|lRy9T*)uztBA<=_>Ov2JH< zRALuLyxhYMi>rYgmTw|_s>_*=T!5h=_SuuB+)%PKVG1I0hORrDA$M4ibbQiwavzTXmqOey?=&T`;?mnqK}61_UTt>-(uxP_M>s#VH86-h7a6}c z$6U2N0x8G_@lqHCI2V$!^{gq`73V%~3ZkBK2G)%-ki>{-0779Q=064?6c!@xVE{s5 z@rc8g9m@%c{Fy$1(cyc#*LffiUOe|=%HzFSvD)>}rl@d)nj@ijHHmSCb zQX&=Eirhsjv;|2(Y8Gy6%|wH!5E}9NYFNap9j!JwO_;|ZrX+mpEw(OyfxWjPT^}uO zkBW}iIAow$tO=jHxyC}3Z<<`U}B4h)Fvz!ZIzOJ?~ zPuEdX8Y)eHM9}b~P&B^l5;baG1NVp?Ki*$5JDEwKp$%_&jp zH6Qe}mw=HwYG-?Z0O7M)AkruP^ou`R#Gh^ar#7DxVqx!0UaHM2J7d*yW|Ea=KwZ?9 z$6PcX_jZ8r8u!Oj&^T-YlHb_0CdrzDP@R3;6oeYd1h1PK$*v-DroCeINCnBU6z2C7 zkrk!^-w7H-(>2yW0}2e{9tK{;5M339*vW4<<%^QRx0!-ay(22VKoe%G@cKBGS1iR6 zWp)E2{Ie1A1u9Wspb8U63^ch7v4THtN>s&{-!%mhIbV8CIoOF$Ei*d`I}EE6v|I|nRUiC z;rfwE5l&0gFiSR+y0Usk?mifdcx{P{V-`j#50gf=x#KB)ybAGT$4zOhBy2N-m~vj! za+`78$#N0H57*>{f#7|;FGv9;cVC2*hrLR8a5O>T;d3EC`7lfJc=NO$HU*(X!VfdQ)iLS$5JkCpQ-#~F0#nja=z?`WQ!KW7R;N!QbYBU3pg zOlL@)g9itXSw-QPbIJkg|3PHbUrYr&5y|(P)l7i-VIGN3ni5cT>EAa6QO^Sd^`tbM z5G(@_3X6#5WSu9=lomy6$Savr&`|u(2zx@wYc|f!YCR#ODBORTGEwz|ZU)+DV|W^Bx%H0w-bV0KMUR95?%nC)Q@ug5^+ONUdvKXhF7U)~1w8#&BTJ+V$OR=7Q-FG{+8H+g`gHTW`;;jNC6S*n1)*$z z(S!bX+5SD*4Y1#83-tn~f#LQ0+;PNHv+o^zDCFk#e9^VWRH?`ZR!gO-Npd4+?F+j3 zv$#>f|I3s;O2EHk3PN=fF*Um$dPqxJbrO*rVgN#6G21j!L0BLy=sPzyE$G&vTou7l zN(to!TlW{-DssLN_ko+uO9BdXyFI=C5+6fx;>sBjEa=ViXfv*Y^2SrMnvPj4ZSc-*Pm{l#`l?mPy%$^ z6oe9E!Re;Nc%T_9AjV(1v0~hP=mHj*LTFvQ7w4Uko~%biNK#EP&X`W_(Y2csfrEk=?{!!B}%t0i7r(GYi^ zY#{mvnkV}EO^I&o{q}|zYrJUZfPoF|H9v6c)`k(DIL)r{!l%K`1r&2Z9}~ z4mm75=;UZQp^Y59Aj!WK8Oax6wUE|m1Hr#;N-d=^zG@1hp5O+9h8lN-2pND-SiCnH zwy*Za2WzVpx9&zdcXialdWmW&pw`xJj%b{yt%^BvgonQ6U!%C6V}o?VHIXFWR4yzOl5 zLcExB#e!v8TF>}2FzR)tR91|7jVXx88MWSL)DFnmX{_DDx+E+muF$rWZYh@u3e~Sf z#vn^EXa_GmS9rt)9u187B~z*@#=OrIL_K2+EDvS8h)I_L2!+M#Osz2K`}+2edD6Ty zpa;I#qHacCb+t+J7fic=>P4St5I1ttyoGBK(6pHJV7o* z$gtQ*JUHK&b9-xS%?9$;wyNS0RGmGeAx7~LMvN-OW5K)FJlt(GLT3WICXLV^nldDc z3H7a!QTyr>>h;aj@)lDNN;%!cAX*+NoO(iicVr~L+JyR`DYcX~88ih^PjG_)MU6>9 zgbY9^EWR(Ku+@H)9rqv!gEl&_&pODuKw%wG429NF%Yt|hh=O>|1#Cd#x~SL`)PLt9 z>mEB}>Ee0b1p&w*^TJ1tibDp=WC|j32Hxe{oa7Nffe-foQr!qe zuxn@L(~&Xr#mXeziYhIPCYz=T&Ff_APn+^YvGpycAR=dLpU+nEuN;&`q`2ah>g!zQ z3X{Nqx%W(DEWBN@aGMZE@~}`r5QP#iKfye((GvNXDXkT|K4Jl}6@b>eO@%ACb+wJkB#6K?a_mpj; z6^aSJNN=QGpzzl^L}N$zds8l|j_^08AR_MwK^K7UtFcPO+(e<6&!7x|UN&P^UsRKa zfD6F)(Zn@88pg>VnNn3T<_D%AB4iLDVzMz_?ZRv6%B4fKXWcy{YFx z`fUF5$LzD&73gx0wy4X|S6$6#^Agi8pv3+`24Uc{F?A7a_AR0Ym(8h6YTVJP4Fu@= zWK?vm-9C-8rc5Yl^#&Tp#EIs4aLyEjk|9^HvDI_TPB_m*5w6AriFZWCkgNN1e$JF) zs^dRx3L#ExDh4Z~eIhc>Y*VA&Ruo$$ z%y{aDK{&^WLJ~8&=wa3{_WhnIpXW^0oz zo_$}P;OG-WqNH0G&9B~H1(f@tywG{oq4MAk91L?z@^ zi1w$5$!~4GJM^1^P~9OSMr`&q>HlBdm%zzYRq1End&^D&$sHg?sPW5kcB|V zN;)C9VEd}-b$3;2qn0F%;)n`##u{OUR&g0d#|=hDX25lHbaWiW{qKsq<1#Xe;xaDa z`v1T}QizPsOZ&!IDU-{IalyUr8Fm|3g4&Q1P9z0q6l zF)7Zvue@F=!Z=lWiO%K|Fq)t?!+|_-Bx?;KE>sSyc3joOMGyZW#4X0ys#v1KltSsG zeDy=8aJ5c>Y07CS+M2L}R75plf__qWwQ7kc5TUg=)<}nsy9mGi6ED%X_{Q{yCaB6$ zsYY9(KPasNx)!~aMbw=l)xV1%U80|B)%RIb*p)_K^?kZ=EI%O?q1(bEijIk1YOlwA zU+OC!l*}L_iVO&_uvc=NBqBqpl7M ztteVdl5=3Pg&-dRaoF_vXN~>Fsz69Lr><~W6+SE1sstvBQ7JpRmwS#>gsvl_up{W@ zT>k)0nV;?stLCvc;%Q^LZ>#uSWZh2tl+`;cWUfR8Bup%Oe^ap^>=Z2vEZ-#UEHj*F zn_c!G@4Aj>hXo6s70D!H`Joivp8G7+lY)J#@6YdcD+sxrFV+0D@?q5q`Qz{jUrgLP zYtp3aKJo4kuqrr| z!E&V2@h;5LecA3U>*Lv>bi{S9LZvT8#?w@5dsYYz7e!sayK7qjcvN}@A-u6$y}%{o zE~1hfhD&9bmv}^FShGVv!1abei@8Hu-F1t35sRoh5Ka8VcxrdmNc3*EmNgnwAV|}+ zP&~FVh}t_EN9`f02;I*-sDPj7De%rzVW+@)Xo4Sh;3G*9Irv(^$m$?C&;J^>^!x0Z#p0R}i%|jiYvn zRD^B}7c1Z=yuaI2)!!X73U7B1-s_~;=pOGHsfcQL1>KeI@l?|gh|pS$(&I@t)3%m7 zyq=EWgqUW@wv~2HTfIYRnkm8g>mFPO#U*IpboP11YDUFza8Ece%Y%Nng$@0}c~3t+ ziX#02Q@SB(RnWERlvG4DkAkY8Ym?e>6^PJUJRse(A}z3E{^RqzW4;dlp^|Yp+@i{7 z?Id&Nd|Y{ss*cC!tCw$|pn6Hg9c?T9U!`?JSFg9Ri1RptmL2N_;_H6i6mq4evhO_3 zG|)`{U*kA_N-9D(kdG@WCVEV5^VZu|rdWs~a)N)~=vT&2pRc|8lyeS1|9?{4bQS)o zRD^NpFFcZi&+?m#6hX@MI!EDV2XdKW+V8XZf(;WhZx8eurHewi>ZX<8t&0jJ0)#Av zCJoNY7f3~nnGnIgomyB)vePs0X#+kT5DnhrBUPb0aj~A`p^oB2wz-q-ez4c)tupN#wx*L#CTA-!#=4Z_dsX(7AcLoCfv^=&SPiT<~pc~r_ui;lht-#pEic- z{E+K^sv_qAqMv9S(MO~rbglj`Mao2XYD;h?)n80nE(6JUB!}y1Rn_`u#t2x7=#P|t z*yTazAE4n0DTcaYKQ0wfO@p9+(p{WtUIGzX3%85=AT|I>GZFZpfU6xN_l&slv)$8A zH;gvN-&wYCTxXXma0AboXJU(*E5uDB?T1RU^6(2CD<++hJ4WzSpc71LuJkm!;G8K$ zGMSU~=#=KE*bGj8oSb*!uM=xXiM8}{8D5;kty6G5`bYdr+=5i%xp*nf$=Ro)HllrH z(j)4VX#>ejWaMn(dH8y`)Z|JXjK^Y58XrF&AMc-(cmaK@j$HRoR_Cp|9U)(?dzS8EXw1rxweS5jIocaqX?KUp$Y_F+o@IR`iQ~p-9o$@0!9^DRZBV=Ag zFSpano%C`RFM%fT(K1k+`ZEWr#cvs-x`-KUC8N?+KcLRxP)YqjrK^6R+LHQ#N>}~B z-@IN+WNN4x|8{_irtoT?3Q%@OT<_tF^P5eXszLnNwcwmSqR z9z?0^>cRW5a;Z4gNT~!fBR#oN+x7wxdUEv^>9(nR9`nQVdmeN70giNU?Mx0hn7nsb z*Wx6OFQQ@dXlE{G52?eMR0ck3+-d$^<+8@VsrM=HIuUUfsQgt2G%8lgK02?ejtBX1 zPK0`d0em7KdSkWDGaNRYRbV;rql|K@`J`f{k~L$Zb;u%yiD|xHH`axnOwn6 z7hLCcY(RQSIj|e_Dt{K|3HCtMLbkQbP!%t&58?8I(yG#tlF|wr*{Vrm2iw)Eg1;%j z7*$Qsv%y9&M=D~>{K+@ccG>xO)FZEEU$n^>QHw)}jUJDBB)QJ7@;YePl>JkKglv?e zsW%_ivxvIA)P&!KSlEjrO{?EdWr}%)b+oEA95)8?qEH^0hgIeZYRbH#(oe0`Z5?eK z<-<}Dy4oL*iqNwV#dSlo5JR13>!D&QJH$Pv&2t0aA;#^-YD8NoO&Rl9lACcwZI~8p zaW9mzpj+HuNk!x$|BC+P__2Jad6er+gOgnf2}Mtph2Rfx{NZMVf#j%hP$jcJcJPRIXBMd;S?KT;8TOjDI&0>(6f zhzc!uO(77WvG}lbW#g{zA3wkA`&~!W9t?tEA%>4OyxP0M}9Ivg!zA{bRR zTkWZMEFB*hEUfLYx;AwABSmy~Ps|#KV5nN=2Nw4mkXM82m@tc|+ZRpz&8@?*O|w&o zIJe!R%Xb9&>0F$>lu&FbSRi@#8gouR49L=Oxdzw2aNcYuJFK1icy7gmD(*0nljKTQ zut!aSUph$UY^*opXcy=fe%MtTJ_%WE_X=NGYwE#R>#!YHHa>!zHE3kU>`LOqG-kZd=`!Ic^M2oJU3&0Q!K4g7*29n zr2tzwDb#urkWo;-8=H3Sp{IMGtd5r_zTsVd1m=Ah>9^Mo$YQnQuJBsE>4kb~9WAEe z0kprEjt1s!aXmZE96MOa%}sc#n#_7&d^;_5xPB<6v_44 zllUeJQyJNA?Tm4xg23%=pk(nJPtjPL$9Eg!akJtv61G{t;ANJvviJ@u1-hL*Bo$$t z#r4c$bh~xJ9sk&0N#=CA>}kQfYRSH}eFNRsjB&SKaThHQZ|q1LT7|1`RT75sr`6KNzF+dPOTDV0nro%Aiu5CnuyhnyXSgOp23V0i`&1xSryw_a(&x zqvCI+v}xv_l!`FU{6=OTvsSOQlFxf;tnBxmI@|+xKA*Alw8F>i#hVN*3lQNfOM|9U zZm{I|p4uxZ##f?~q}VxKXRm;E4K5^>OGOxGF2v0BZ?}43%7;4-bhFdz%e)@e$JHKV zT!j=@{o#CBl|Z)|Y*#y_(CT*eG^q&V>}+Fp2DV$f;<%1-3f8gIcT&(#)ZR=#_NK6X ztLNkvn`g&wE><133>f2Vo8oLB9H~?=Tvm{~PpzP`A6E(nddQfR2F;``6=9so{mkUx z+U*vC_hj-!W-^X}4;zjat~_aoDmd)0)D#ge)nPtC8-hvaOuXO^cm3DKxV=_!JBVmK zeucxRi(26dPQ58DkCzqt2B^MM%9E!0#ZnQ~R15a=^p>Dns0u`AEk@Z^kS>E3?J8et zv=A4Vj&|VE$=8*?$5_>&gKqCiWw;{uKt*_`lto<;9+rwQPQj{uFkjEaio8oeQ(YWG zV$+Lm-MO}H{tcb48{;G7yK7C8*Rm?%vMK>`z9L0dlk+912;=1R?1fVtn_c)=P7($@ zh)R#S4lE+wpgjrF_&(|M~@wGXGIWx558LHL}$3jXwIt5bN@c>)a zEK09ZC}{ zCizC0t4V;Gl~AX_WyOV35!KWP1~5Gas#6ky2(86B>B`nU7MnaB7SHu|iznQ=sd=}$ zs}AonbTs1|Q=c(Gm5fR>+6mVQDgC-}ACrpE++8mf;d1x$^LwOu#nmeKdY*dtBe_h( zj>IuT86qbH8T9+qmGtVH=zdQ#KB{k@AgT#~DyrGaB5`vqo8=cu;nt-86^j^udl8>- zU9cyUve6xSL#c~-^;OR|m;$u4EQH`wq=7eFR%lSRXqjBggyMCL!+qbm;hy-ztSc+_ za(K@G2g(Y2I=mtru$aU?LY9t`>7!UYkBF?;gpIchmoZANgue%G$4TvMke193 zRKodioHpr{mb|U%DG6`KP_mM>TD*Ozao#>46`}ja_bKWp`iN^j`Yl+<70`gV7u&Pf zKQaA*G1eB~q~FV$CZX9_e}!o+TG0L)7UkcU0;#*K?@C3O=K^86uzFBCIoQ;x-g|4> zJTbG*z$j5#02j&StZ6@qvi7x$8ph;vpiYCc&Y4mX#<^HkIiBx z(X%j=Lhd}Q=*<0Xr-rfdQ7M?3kHbin^5?#day^S%ZcNkq^i%12(b9 zESM?8_Ow5==a4bxc91Q<(v!jxdU}@U4O-8#=Pp*0xMnqd9b^p?=2KFJG|73X2;(F- zAI0(}ld`}ytTp|e;olqMtRGiXNsDk*@PgI7pJ80TQ=&LSqhnOx36 z@DMuwscA>{L1S!O5VHLWlt(0g2RJLp%loCEYF^$e6;aKLV82OkS*azNK!n!f)zTTM z)ms<4ZT;J!mzXi=SW`m*x%OK2e4h$IJPUL zB6K6YP|-8d>+egw=sDI-T+hNTKh2c(!h0OT;K7wI4qNH!L${BVYmJezCd96=(2{U4 z@2P|PugN6B-zjy%tWJQWtEC9*{(P5IgmIFZkK}BeyyCPK7bvO>Von+(qXi2;U$z0z z2c?K<&>g9WF@rAN>YZ7uQ-7&3xMzjfSPg?n0OyxTLDD!6vxvIA`-C4X&Doc>^2Joj z&iQS8-ewHHx#-ZtUX}p}-_kgQ_e(|Ss`^H$2tCtML^d?jGBlK&tC#wQG1}{d?fR+| zY29QVm5R`O8$!Bf&opVPq3LF+PNbmUOYzYV|3)gpIK&qp!_E*EwWn~TL^ng}YG|Hs zO<^~be0lkI974vxUxg`=s|jpoJWbe|rmRVzx3i#5gUf-7q#~-R5p-3$w^NG}fe5X| z@1@Ih(gN$sC(gC5ocLAI>|8gYr6M$MM_I&#PfWTG z#M3AFwhnf!9NdE`VAG!BHDDpHHt~ zbmPl!=}J}ws)lq}2jHuo?v;>L+t%z-0Z0parR-(KG2r_t8r{UBo(3C(gzeh6TSDd zpx5<6UXOvtjWMt=S=5!-DC?1--wG;f=j zX?>t4DP4g0SsWZ&nJUpG7g8Vz1p@70F0^QHLODw+!Z;C&fqe=iYg8R#;qG8M?NoUj z*=UT0B_SZMvk#ZE50J24ilA-{mq|q!C!xu;L{@hhd`;k(G3=XxJLd1>#D z!u~2_6kHtg{9NUITL`%M=x#Cusdw!+4=0EO8L>xD+o9dB0SIamWuG%VcSn5+=j|fo>ARJRQ;H z)oO=4)JadX`z8f=q1onazvCx^xtQ@11~OY?nJ zD#AG54TM2{$QYpwhe3W&inpfhtx^%jDQm5ELG0@ozi5n=v4lbXoD?+;{AZ*hj03-D zkK<%5&&{3X`F^dO7(ZzY{l!>+JK1ss>iV8%e<{UJQ}A=C2;&rVTzv+iR#Q%L$Wn1@ zm5b*N3kJEF^Z-82k<)lw+kqPlU%Oa-xv3>f2d*HxO2NZ32h(L2&T!$Tbb^->ty5qI zw-kysxSm-g6=9sO?P@)v24dXPO)FOd`DmQoc9Wlh>fX$v)AZxK zk=%B}Bbgm`RpdKW)#2aiOuZPdae#iglnCASwn{}9=W?f6t(95ofMlZr4-#nP+uc^qXR zhy=MpQx|_>Vsh|+kBN^PW8#94<0*h(LrMXrB^e~*qf!(#5g(R{FiylJJJVKWaPiJm zJdI1%x2Q9Aaw!0Ge5Cx?7%8hmb~;?COt?}RavR9X1UjuBO3~E>Jth@loS;p|VAuvX ztO$LPvn{g2a=Q+_HnNig8ThOpOyxUkEt6WWFf|$96e7nto^rLiwXoK${vxUg3N|QsQV|CEXiuz~0W(eO>N6DMh6Al>INy)NV$PYQGeN8uunR;J)eU= zg9kiwt4abt)dS_Gai>W({HEF}5wuil-q3`=}$sr5OljRLsRk5U$prR?eo@+zqagB3(Eyksmb_oG}TOtjpkp*qwB+v$g;7-||mC>3FxhBZ5_y*UTL zA+XZJ)d4$;DZCvWB*)~m?FW|}>U_}Gq5jYqE$ty2BVbvdu%{1sC+BU*vOa;4?=dOH znx^kbMHr`P5l>NZ?rd8FnM_P|q5ipP)0L*y5R1xlQ#(w#t8OmXQ072`1{Vo4r6Pn}4m&JGST~b(QW3^U z8mII3USsr#oWCED;;N}RC>3FxnzkcD*dIxIuK3Axz4D;=lrbvihq9bAtYBdHl1$b# zn$Jsd)6i$7B8)@7_S_hlcroa8dhqA5h*so zY6OURj}%o+%sZtbj1zN7U1B=eDW+{Xg{Y63ZyBS;szVKV$+W|yM}V8JOM%tgd_^k4 zI5$_pD`|Gfa<|UZ1COZcW#b8@XxLLUTvjwd>>s79XkvdS6=9s%Rz)mX zyx6%MqNnkH@J{lDHUNH`1#yuqBv!TEwh>u_@KYm81xHp zmj%rLSz|61R} z?np(98FsN#lGzGwYv(?$>o1!+Y4Rlcck4)E4R-t!FEs}J90bfHro(nTSl3@7MNES} z%p%T-)QRxU{UmuS@=)B{map!;mbfp**vJfSSZcMa*ioO9u;Fn&kTs4z&HZu-@a@W_LyCYIwg? zgl_w9ROn5#ZM1OlXb*WiLT*OM(9Vj)(|nbdPTU8uAYO0n!n&AZJmNqx z28X>=r{O-AJX&F%QvGvZRsAls&qv0C;aCqDZ^U&T*aH%`c2ezvy9GdiWiCM2D?m6x z`s0U6e=^=x`t!`Q`q#n!=)a{b>Wc6UsR-j0VZq_tfSr!tLcJ4J2&xM{3TNyvMIr8z zmh)yRIqyI_oC*aRoOe!=iZBlU>YZppWaMJm5m>jn!$jICQq+*NQ#hT;oy_~U&(|6w zgzVU2VK%v8ZQO2BX%qIe2{w>RrSR$o5|WCjW@tRaiCfjQJ4iKc5QxxP81kjnveR{< z<#`~k97u08RwG*J)BpR}#RISiBV zKW$+)ghS%qn~R9AtH#8BI~7k3`JJvj*BD7x;M8lV##9GNV)IfdNz+JTQ7Op-pHec| zn$Jkt(1aGHB8(Hdx`IXT)V#x@8^LxiI!J?{9j}UAeV;MnR^t}Gw@d&f?w#WbWw1Pg zGJz5C)lzshL-$HW7-y(`Obl83)SA^>s(#QIN$q1IDIBk{kbS=tWX;oir6P>;G|M$* z^Y7Mt&lnH0JqxoM4dUBUlr+%al!`D8bi1;?OCv1Dfx;oasz_Lclg>HA3 zNJSW@b6*2=c3{^JgvXqM9k25AdC(ZS`x+q^8)G>#kx~i=8t!#cnl#hbNJSWD`e|+t zt9C&eYr9!v^zQKbT~>1OICttrK46ZYivqnZJ>WFwV>chqDE` zs_AVxX6>Af`+wEOZ|^h)`m&IFSrZ!Y#$$yBfu8SSDUOmt;zblRD^M|*3@k`=a_E%&KOf` z>bIg%CKFFdA=WJYN-DxQOJM}~aB}btNdDx=GbA07>HdZfSt|OQ%6;6GJ58X=a^waGIR;k@Oa0B&`eCtWnq@jr3UtohN%5A;dBq zqGf|r-7m#iQ}srv2!m9)hGEl=VK2}}!>5^sMBD6|c3(4|g>$H9clrL}i!O0&@#XzM z*?P4?%J^-)x)!chwqA+(7(3_ss^bwO4U%Bo2nXHf>q-mq`c~?@fxX!KSkAel!+cs)%Yg=#ROK3 z`S-64R1H#Vl&YaLH&r#Z2CK#wq`c~?@mZ;e>Z&oGV!JaQZSSh$@(M&~EnX{)f9l3z z{@MAB!@O6;VeZ)u2S4zJr*}32HlFbHb2J_sfYk=}Zx7(U1D?y)dcM`!#k(hF+m@gh zEg16(~a<%xEm$DXV!KkX=5Xtwvy<6kk#Ge!|y6n_?6)0f^z1AB+5CAoCAx> zD;tM&`?(>l8%{j&pEJvQ0JLHo+~r*Xd!*hMd^c>V7lEoAGeDg+Z1>^%9AmgHg{6rK zdsn4E5Z#*^NB0?05xSQ;Ar+zfO2u(QedQL`V!~^~lB3?uEcWo3Bx`i4?Y_!z7xJEn zn^=KPszjB+y;;R>Iw}7;;m%7^8uY+zq`21i zK3*O&#>-mXQY$OOouH?X|2jah)jlXiSrheEsR-jlEheI(a24B`%;YgF)!OO&qA^;Q zsGUw^AExZjvD^7MDT11W&qzfWC!v93@+XaP)9^9*FQs^E%6={t@#jq0@;#Po1HO2V zlZ&6i9PE^n@9;ciSsQ(Gt`{4Fp5pJ7v-jLyg*OH#Shq<5)EqotD#ADivpp~8{!{EX z8-sn0cQ-8D6@>Z?Qk*o@uak;6e^7V(YWT;`1JpHaqCP6cNkjc%sfhCjwLi%BkIw_t zHO{+#D8)%b{g_mQaj4hQsF`x$MOrPHV;0*L@gyf+$m3p>*U9>7ee2bxW|`}FNF|Nj zEA+9lrBe(Jr4ipkYZMr7o1j*MbJ00c5yq)nQuf_;EzLPW8GF(@<0Gcq7%@w=54S;; zzw1>pI1X%(VyIh8SSrFe4Gk=(Z!ktq9TVnWDT$eR2rTcZ%GxdKYV) zGfplVhubV)(-<;_eyBVt@>D5bsL`qfI;&GsWHmW?sR-lbjN7dFHOANxnl-;timj&S zOE2s#;NM)%_d#{2IMtv4@Jd+=iFnS^*J4yA9HZ# z5%4GX9b;^TLwQfBvO78QsarNcXKQpQ#qI)w6NGO`(bj~0T`IyjVSGA;u+0?ol1@Kx zwc|8GZ(6pcTxF`jMljL$jZV||nwn9o(4>fe)~vjH%8KG_@*Xh!lX=??%`-vCe5l;5$~skn36$J>5myj_kX zd8H$}iBLLHgiTjRO1e6RE8UH;iUBsSl~SSm#jB+vjI+7;NX}vJH(0!aQfhR8AuDZ+ zs3jpB)#&WY!&b1*J1Ip_lQ1Y1VVs1E_PS^AHYOEAG?D04&adHGG3P%v`CDULtO$9Q z0IUVg9M0_^`lV7lH6<^RiZD*e{Db7c$X7dko!8rq@qmB`m$I~dAoVum1EtcOQXa0L0a6jM#h zFQg)j)6#Mf-St8H3jybzU9Ik>Yx_EyBk$$~=ot+1K7y_?M$q;UtyrKTPlV|JCtrPnqfPK=TC7&2 ztYKixxl+o5Zb93nB8*engL9if2evzLv>?mx#D=KTS0yUlUHfQ^8l!PD1xYGvhpAbm zGaI69tcI=6k>al@yh$p;IEBqeoqRl21by82eB9h&jGGoLHdVIjOf)$1yhw_e2K@z6 z5ynBE>jwUb+Ih!^_yfiepXY_Q;XDJ~)SIMOX}Di66=581i%bJ(sa{MXR?R6pStK{B z)N5z*@V#5*u;P=(IJlTC0z5^+tO&MODisM3^D!x^nwXDBMHnY$*@3_nx+BWBv464h z6JwMt4`Gc|748XpE973e#wz7UQY1AQKah$rPR8QB@iX4E%&Vi-ldk#Knwk|a3H5XK zJm;#+KF}gsphJU`<9SjM#z|OKaes7YZ|~uH_eXn-5wSd!qbsb+H9K8lt%!sc9HzUZ zNa_~RDHUOyjCQsQ_S%GdwO3#0wXXIqZu!RW8;w!Zt}KPUm5Ji1A{o@-5B04MNI?$Ua1J< zz_;y&sf6-fP^&Nco-yR-hsa1n`7Ws24}DvTn}+_IQW3_XUrr%q-MEP8j;!5wGKy6n zdsX&zLNoW7n#o=mN<)XRiiU`hAdI~+twW#-ng&H0oX$>`iZD*cnuFynxO~OXtc>zu zSA>1TAT{fZQPW@scABw$5E;0tBU`=xTxzQW3@p>e*Y&VHC|#ysa$QKwN999lb;* zhjUY9qfe$k%9S#{!!YCI`~f`gF~-;?Se~$U6lu%VE?iblIqrb{ou^!wl?%48yQS!B z0{=!T!Z?A;%J+mFzS-}Zee(%oYaOZ@l19ePRzG94~9tw5gDaZX^1zThR9@Wa7%ZlYoQuH-}pO=a- zPT@Aisl-!KTs1Ynl8P`+&HBp5>Bzq5zSqIXc zdmXz#(Iji|wDLMr3!zT%RBqGdX2XnOkqWiK{YtI69pZ)1t--a?1yT{lNn2LYp4_>g zI}P_Ae(f?w(DKTLq#BOrf1<|W*A6L?x&>V!6=9r=`IRlGMk{fQk+8tmN)!vhd0Iq@ zoCd#7D#AGUD}vywi$+JtKGEMAhK&)ivIY@hzFw47Y6Mn4w@VS#q`Xin!Z;~i{!q3R zofXjNgrmYBX}-nkTZ|Fa4dbE8fW%d^0mPE29CzxqsQabJYx3SG6=9sbg)pq(8)1B3 zuhu5?r;X9Gh)gQ@1{gP=Yx3Mx5g3&_sByI1G$Jl4L6j|MCTqPA@oSYS9 z^L5YQN3jG$;{;Thz#XWHX;H@n?m=UuRGGlRZ@kYyF?t2sInUD+P`KXjn7CC)uens`|gco6udF`71n^g*FtF%^V8lZMT&cWIaP z3Uqoumx8W&`-xOUHE-jY!ny^~Av#oK8-WO|#ZsyFbvK;RAEzOl>bqywiR$w2b-oZs z*6BRK@pLwCw}Q7XFmJc6$stUOg9C1C(ns;`TlItwOw4Hkm3g$dMeC4PgS-E0SVUcL z+b2A(N+4{}W*>Y9Ou<(pY*CHSh5H&ubgxu|ZZEqPDfKG0G{6zrn&}}hsaC(}Xt9Mn zWc51Pg2P^S{Qjjg#t30sGd&i%`k;d)ec+?Y)u)#C>7*26-MPl4B8<~C0e6P~&KPME zc4zo5DgK(ml2n9o3eR#XQ}70<2;&rt=g#n_jBzpU zJHsEB0;)Ors8ob;PR@O2_@~BL7|WgEA4^fw!2eJx!Z`4w+!jpHzf#0>^8k`9H?klG$i}M~bW_=UY+{#>p9vjpnSwre@6Jx6zyqT^gJ|Pmzi+ zPRi&unwJ@)Vyrfr?Nan~t5_`+VVr=fEypq1XdW^~O4XL*cx*KHOF`B=T`Lt~oTnQO zXAALE{1)ZofK06hs!Xjq*bxjFUFBfOnYD8Ta++kPhj4M5E_yhYcf-SCr*?&aZ@rt1 zv3f%&Q=vZJ;`Y>s4`a3zQ=Su)_F>YVTr5F*-0x=X!$}IwLw4;I+6PPv(^3L;9XKf! zVZ08kKI$O)10p|LF#bH`oG!rD@~E91%xBWydBH35xG7c`%=o?jj2NTz5?rKI+JvD- zT4|G0+Jy7CI>x#L2>M$ox|*PuN<|nasOcCSS!yft+edC2xz)We^)6#XG_&7IoofK{ zw@dNTkpHVxgmK7My58^yoD8a54t_)ZcP@MulwUPQg5`P0FY8eiGa`76@=+OyzcNEeJXb{gH5 z12_7(_dvP?TFq~y*lK!yEfry$p4Q%i<75%2rly^?b;Q)_r!ACsGxmo4An+F<_XgKH z%cLTV1HaU}yKzneb{iw$0(CD9BlhS-TqQ+Ow}309B8(F;n}+Im%(0WcZm8cF@^g5Y zRyhVbo2V2g4fS)RB8)@5mY~Lsa|)Mo%1Jx#i?r}qEc7z^Y)%2R_D(@KqV zwS}KZ5!IypNGifODdjoRIq6t*y{Sp6o+FJ;#C&Mc;AFK$D#ADs3wyDqM(i{AHmAtU zZeXj*U5%~ANLUogBXdNbpv=ilah<4IJyP^^OX!k{Fiyb5N1an~nl>R4^BBTmKGPT* ztFYPUX%Zf5=*t_WxN2&SOGOx`X6ZFm+wbt3U{hW7joi;S#>fSs>Zz*deKRV6EYWF!vGjI%CAF<;8_o$Xi*ELSa&< zJgx_cx=)I-ChFBv5ypv{Pefgd)gN4`Q@E|_SP>sKM#%!EWKd;Y*I)1lrO0XU-!Bzm z9QoEeJwCP~ha(y)Wx}rHp zO-*E{h2pV|!Cq!2GHh_pI!!9VIOfX&<1(F<^Gzxr>S1H3FAt8) zfHhJP#>rR{?DE;gRQ1I+A2UacG1FdS9AIBl)fd}>G#!*;tZBMVD#AET+mEPG!JEP| zN8a?(+nZHRwa>|1-WYS2hujgwo8&TWo{HXPt#Y_MI+8waxE$$d2Hk z1T*AJI&bOJv7(bFKSVT=op53}LBW%+&wP(D?zZ{8@+IvE+z%-Yb)ujYWV^1@Wd#FF zzEetrX7XXF2;)qysO=D~W4GQjga&-`zX{4Ec_KIpqi5}NktgvWHd+We>BF! zm>jA9P70jH{wb*lm(anzUZ&1F*7;RC%@p2Er1 zt#8D*+8FE?hWepHIFroRa1kD)W0w?5-3WF_MHr`}%DTRK_FTsU@}MzpXh)zs$blmT zP;(HGiZIT>LXPT;)7{=^{3L=}uob+oExg1S-iuT?XEL`^nX?glkbq$+dYXXSr6P#e<4oRy}Z@x5ZzJ`{sGi z-QR!K8179Wid|mLG_Vr*v=l83_$Q%znwn;@8$Gd8JGseq<+l_Ki!*lf(DS*1pZ<2~I&cR{#WY0A} zpm~2hnJngU3$WnSu^Q>93?ft&mBTkO-(-xc!`{)Lt2eC`wBB`WOvV*(=owPVH0vj% zB8;=%TD_F;(e?skw6%ps-(J5}ikb%g7O4pHz*l#2KHy((5cuk=(800hwNlhH@P98A z(J1gQdt1jO{)ZT4kJO1;p#q^Tc)C--d^UqE&0Ng`^^X`MVPR;h1%O&pU2vI@`jKtIobQ=(CY`gEQM1QW3^Mzu-hpncbq7L9k+6l-yfPN9m$~ufOj! zhX1mVKT0u-2g`(BScn%eQcXj_>GzPfpY&hFGJ$z1zmJC__#Bxv@~&hAXFo5K;;gCqJE;icR4qCIuQ}E^ z*8uM|M$6(52Sp!!A@FV~ewu=JNJSW@VBJpt+`q$0XHKhw1>4GIl0)fCiep*E{n6*X zX^fY32wCl05`>|VI&QGTS|bFtt`W5QYf_*!S6`NjsOCx#$xsJFRN?XjBD5BlNCU^H zV2H(1OCb~OOBYjpHec;s^B9nwUyYEs0;NKX{Vf7A1a3Z&MPF(cA3AFX(Mf z;#cbFxAfMO__6x%hw8(}>8&L(nS@PF{9Jwaf9ks@=xs{kC-io0f&!%P-$y7j0HQ+CLQ!JH3Z#cyPRj>3BHnJG5gANjEvzn1PJu#uRB|ekj+{R+XHG5hs* zcDaR|hC-fP1X<7rKfOyOsgMt=Bo*aRl_c$dtCFPS%EkO=q~mUtBpnA-l5~8DN|KHb zswC<7d6gs`f31?FSt9FsuxQO+)~ zOWp%vHKU#7Pz_k8l2ii@sU+2aoJvv+c)3ba4fr>eL=8xM%}uiGmbEOdrLt!9xJ93Z zqJNuEn#Dz*vRozk?=JNoNT(ar1A%gjN)itDsU+d>A(bQ?zORym!<-9QAaKxZZg%sY zjl3iD{cO(r4wa-F{z)Y%hflak*61hR18KDKBK}j#`zkldKZgM(8fl88dll&M(A{YV#hi(BA1DDeK3 z5I=_toKZXuTm zxA#Cg4R{Zv(<{^imF~SNNd@|rN|L5iLoA0hy~Is&F|YL=NMzD`Ad#<84=7NvG$%?Y zaJZD3_6EawP=0_~!#q-67jBzdxF&r63zE>pgo>Yd z*6(B91L^madZ6+wTg?K|d*Soz)B_dnq?_c=@A4k#k016PsBn*Z52%{O(rjfI?ommqtnX4u zs;pmAN$Mg0sFKt}hMHL}^^n)7B=wLdRg!whyHpaLV&Y+yY(?@>m25-uw<PZ=D zZ<5mjsC!$GMGHfHS|th4SF0qI?E@-FW&5s5QrV`p@}FTO1`?ap0~Pc}m860Wxk>)X z1L}cF{c$(RpFiO}5LqkQSSaDK!%gy!GU|cAxkn`dPDOdV`AjRy^EmyWmCMsIpOaLc zZk43+9CMQ_`Hc5K+25}osHC4%Nh;}2Rgwy~Z~+UX!u7aGmhyD(fu!8(Jy4DNXZ64h zxeUKIpJ{`-(-%Ty8>_okB}v_bDoN_ztdgYE>nJ%NfAk?v@E?6&CFzf5((G(L&5+Tq z-06LPd_Yg{ORbpk7jll%7f+*8fwNpRIbKcNgr&`zjfnofmR>HSmn-RI2fZAimqYY& zgkFx)%Q1SnfnK8Y;?PTuUJCSb8@;@UUT&wCJL%;ty}X=W?xmMk(aUS-v$L5X$5pQE32PKYyCv(>nAFQexd^DCn}kK zq9W=iDyx2?LhC0gy?!DX^b_Hsp9mQJL(y`b>!a(-U(M)Mzqw~*ExEx(MVK*Z z(xKA4d_00>;=oSY*BsH$yPRa^G@hr!MP9mq#N3z@P1>+xp+lELiT)}0>%_gYNJdLM zf7Ge``fRDK4_=~^v7D3ciwyOp?UaK&OACFU#xhawi{?IT*`&+g0*qp$G&2)PAo4r4 zu)&@CC#lXVJ>cY>0vCNIhCO;Le?Keg-lU_i=}*vIm+#gg`zG%R4Wgg^tXP^F&1UJ> ziR({Hnp9d=4PQ1}1~Ac!U--dGoWMti5;xLoX*L6OICnS;u$mE|KAVKa&k-P&$`h{M zCp&WkcuqWn|Fiyh<**9aji(SVo_i5ebbn7-k5D8G&I2V3^B)mWw~kr61vs~_BF7{b2^;s_TFqe6li#%N9QVpKbMN0ewmt3`hsW=O&fW!STU*OKYG?(t> z(z+Hm7V#?&vl+N2=)lq%GE1`t9Jp<#T9bGPatC*nn%r8S%tUQC%t6~dmb24&R9}d} zUF0)EBV6wX2TC*X#}542kCdkGV;OgjMD83Z%>pM%CNzMHc<*U&SK=MW10BxbK;m6U zmF68}_N>ako{@(u`RL$C;)ASD^`Fe`<*%(H_JA{@^7Cbq_=qaTf+Iulah|rSa=`4L z`WB4<7uzj6hWz*9uM_WM5qKgMTs`nal3nqbED_eF=~zafK~vu`%{c`mxEfB*m!=IQ zi`2>CSCjMDdyA)_Z+@gS*G0Ejhy%9Py%>|J*E^n$-@GRtEhuQ)6E~t^(BMEV>h98> z=;p31TQ^5HZj1D6-4NZpxqEBZrmeQ!9qo6vz)NcPhDaD)2i8x(bosIolH$^vgZN{%RY>jM)M!LKDH%Geq zU9m+1;M;`xH|gbTc)1(HKrG1m@DjueCXbyb{z@WO7(h~m{lsi#Itd|z~ zzf0NKtXfr;mixb9aeWovFZF+KCzBOfR{gQkNbj1L7W@AgBV%C*4x>q<;8$g3=cb18 z1u8P;up^}<{Di9?c_+7CeYl>c6eFePgE0(L_;|PasW;4(T5w+@l}V@U+{r$)p`nqp z=s)^XUb+pq;2gA%J$mysX^hBnDP8S9fAaA(T;wJkrGtA>2IyM1FFRxmFq3Fp-<_x|#JD9!+ zjb|`EFqp*uR4Fy$!=<@#xSLHE(2BWEWAfKTl{FnHSKb9&dD9V8+??aqUR2yek=hlv zIf@OROscOxlY@@jPUhr5AJu?V4279T^ia>xchVWcoQ3s{I;e;>zLIgUz}_ zlh?DQCd}i}2Kut7^=laJ;yKvsX! z%JgHhlFDN7GKiH%Vi~e7`92@Sj2c_%^!Wm}ovNIurc<3N&Ew)jFA{ z$mZgxOogF7D4B!V9DcsG)T9(Z!BCjoNWvn~ZEo3UROm|r8+@^fzCaPV(2R}u%_a2B zLiZar)_Bvc#HIAzBKJG=xL~QT{AD|Rx!C=Z`qLPOF}HeRbDzF#if4yT$72IdA>W5N zZ=!=gban2}B&ioF9cj3-cy&cp1I+j4U<^Z1`=WVj{LSV1kG`)zHF6e>v5kXxsELeQ z$7m?5{M2i}Lo_5Uul$C%iJ=!!4V=Hfz!e2ru(?7RW7R`zIck}$X!X?BlYVOKKAU(3 eph!HM@R*BlJq@udk9kO95HGin9L3sb{r?BWbHqdd diff --git a/doc/LectureNotes/_build/.doctrees/exercisesweek37.doctree b/doc/LectureNotes/_build/.doctrees/exercisesweek37.doctree index bd54489041ae98c8ad73221d8c9004a89743b6d9..dc84301e53a2fae647134dd94da289e5ffcf318d 100644 GIT binary patch delta 3174 zcmds3YfN0n6=r6aN4Pv?g1u`B*9&fWvkQbE%mNF9Kx{81u?qo=eOWHqz5BTAgJ7^R zHY80}Y~#lvQrlEb-AJw6q_xe~R;|cRt+aA0wTkUn@()@ib<{d-Y9l+ToHUBkGqVrY zwW{`a|D2g~=A7@GZ|ovEHA7PJ5_N?7Co{&CI(c| zCVK+1PYl>)v1GcW>RHJ(Df?TO?5Vq!O6|F!Mi!(0&oOYKd?Q>?4dgSpTx{C(!*7x# zd#Sz5p0C`YNlEq!yJm6}VbWs2-2aOOL-;S{?-H7;fm5muu2uHIyH#eUjzxx3svgc) z<1G85@O)c>-v>vPJVQZqUkPXh7 zZ86WQHILJwL{Atn$_cLT87r8!x?l`9o7xC{iTPH-PinKtB9JZX;P zI)N;PZ`Qp+=$Qzq*1j$Dd;~qcK}b6jEgy0p7Qx(*O_pJMgPBScjE&a0V$rGwWc&(J zKy*p&(J@JM_(e&am=4%gkLZ-Ar&QlqwYUR6Q*=FQV}6tV=(qIA5~BFslFL;o`W>EU zTymAXcfu$8{SJf~pY+%QIPr^94#b@dV1{wk=azQ3M33Z_Ilf;OC7&#+;{n+t+7Jn` z4U$LoOuN-d{~CrtRDGf$T1*_d+K`SW(JZ#6hhJ>$AP=!0$(}p#|IKq7q1QFw9kQfd z{TvTKzH$w`O&j2?sp<@Fh@6NkXgo}o!(_`{LO5Eb%$u3Q{`j5de8_Dvqw$2f zI%3p8eQOb%L(CXzDTb$7ndhyR%9s&7Nei56X-4zWRuePhPmjt6t=%!7uC^jF z5cPSjtz^yT!-&F;BmTLN-p;^(U}=a2-m&P&bJ4)=_5|?nw%5jdFpzvd>a(Xq$IN(m z;PnoHE^DAqzZw46p{M7x{1|QS%qC~Uk92AYy`<%BfQ7#UM;>ct2B~8M8!$fOlO7st zDrcdjt1||6xJys(M!>FjWs_C7*>#5`f!C@B+#iw#7e}h#h((*J^~M2`l3{1Jh2p@F z6BbtCT6a5sob^5L6Hb>To>-%`VC4faJsh$zX^v2^5J&_Wnp1(h`_=A)CcWjgj|G) zp&oi68N{Z?jtvX+O*UH(!6p;z9PYxsIXLn)czak!{)`DtkhM!7H?V-btRMIL9N37) z9X>zy7&tnYU&U2Pj*kZuy<9bL(Q(?_3`WHi3fn{ z{zssM`{85u;qpFCj@%#iQmeh(%5SA%mMk;TlW!>0j4&cwMmjmgHj)Q_Qmf$hNOoOr zO8gS)D~sGe$)QgoMdB6e!G`L0xE(IZC)(68+yT`Gw`vP=9$@*K-4n2|yR)$Szs45) zTWmtPh7@+da}3dM=iLAwk(x%FN1lCn#j5bj#RlSr zdy9p{1Nk8XnM2D4=nIKx^@SeMx)API=wWgIE{4iew5Tax!ky5Dxfk&*raaHYK_+II zILX8tOjKc`P|BJ3yB5V?nb^!!0~7st*pwYiWHBKy(aXd(CjOa<;vS00&_nD+6_}oo z{p^{lu=GLYQG#+#`4JlwRz{GQ5aj8DJav$#4f2#ho-W8!1$mkvPZ6B7>lo;ln8|Kd z>JiRDWaRNAPQL}`9R>$YF%R;p;5mNjd$NztaQX)^Ugnkr@IJr^50 z>i_*263>0ld*1WB?{m&Q=Ul(_V|w{VRDE3i{fnoU)k7v?gk_U4XTh6wXu-Q-jJ20n}j~3AUcZSe6@pSDdB_i8rWQ`L2sSeCU+WO-mHhut8FC@sY2nc z@+v2sjF03lys&U$VP>JitM{|`ugqmhvPR1RYRMp5v2@t$quFqF4k%2q!Pp-UE)UF zi-ewIHj@~vf0AC5)7+}psLm-B3h4Q^0Qc9aAiqIN7U3HWHu?&I!G?Xg<6_Y15(S&k zl$dGkBwy=pAy942p}$c=&AcAMnrs-bn#j9w!s;N`85Y4mtw+dWV$}9;LhmY3J-G|d zG;PI(=rF*QCO!Q^uIOQthV^Nh&2nEcyksw>*($yua^G_EmSo?(=5mNO+t`2}`?jQ$ z4S)@H+xojqS89=p!7;oMHtyD96wW?-;OrmRU6&m9)7_=CEj^LC<*`Sihgu4f(Kv}C z>Cv>cKH1mbS`5E#!Lly2);>Z}u!rYPuESi40^@4hi}1k~7xLRQbWsTo!&Yc&Gtf6M zZX3MVrX}YSFSn@(y`)-QOe^6Wd7pJqg}1_6$=>?9a?f5IyxTEdom}RR{Dip*o96Fdarca@qu)vE)BhnbUNtlG9w+nU-Mn> z5n85}i|p1CW8&p*A0-wby`>DxdPilOT38I5_p80d%rWBF4EK8VbT1C9f%W?}r51JC zs;LDr{_0Bgn)TjnygO%P=f_mWE9(yDrPDX{8Bt&CU#4UJz_?#3x zQbY)kOR<;9T6mt;1Xk^5SDzQSQLIgVCo4++%Df}%qN8ecO)W3as~FsEynS!;gA#wnq5+? zBIb3+#7d!Bm>eE;hgc%FOW?8PLW>j(PsH2;mIli^bh8!R{dtmJ7#AbHDUs8o zj6R=h%w5S91*MRW$wN(;lI!YusvpmXH3yGRxypqD(gcf~7L~lF1aj*@y51wj;g9dt2-dKZ!TK&q}zIRm0x2byW5WF`y zm}^en+`PHooOT}EkqLT-v}6TFhFX*4H--%KKfG^iv)9zQJ71YR8o&IyJQv}_@GnT1 zVHm1K4atT(0R!}lWuyS?f$D<$jH7jpBOBfpyI7DtSPK@XhJ49VZ-PE&H9Pi>J4Xq5 zl8?g7eRZkKjfpashLR5Wxu=w2x5t^XS3I6HyAT3ny=m=@@LLNzu`2!Fzqf5@6uC?@7|LL$3YCd2M1ew+KT zq&P1p&P$2&GUB|1I8PtvspC9toTrTQba9?4&eOzsiui=L7&%d)*T&^}zb9+kg#t&o zxHr+{;{J{n7dMFtxwu8NxVZDEkc+#G78hqsiNk|Y7KcX{6UU$tnofVte%9>#?veij Do;o{N diff --git a/doc/LectureNotes/_build/.doctrees/exercisesweek38.doctree b/doc/LectureNotes/_build/.doctrees/exercisesweek38.doctree new file mode 100644 index 0000000000000000000000000000000000000000..bdc8a8a7e6893a3101b0b2b784a01dc61365a50a GIT binary patch literal 30573 zcmeHQd#oJSS@$D;+_ioENMhQiaVCya-+Oa+@4bE`wuKvN$Dy$kCw1J!#n)M8cV~BI z@9fNGX4dy^uWl8hrmgAx0nPF#f&xYSQ7VwAganl;q!#`v2wKV?h=SB0A@LAMEfSzg zh2MAPyk_#rEvXd3@(P-}jyGan5&M+V)GO)lKZbcyHLW9dE5*Rw|xfvBFNg zv*MRq4aoIB8s4ok~b!*im-P z5D?JwHqUG%Pdh8ErutDb%PWCZr82E}M`YIMQ^($GAA0J*QTyKb>4gu6RuC>U?7HQJ zmbY-#GJz+rw(u1D*Lu=21CM?K%!Ma`qgG&D_5sR*B``n;IDV?&G}~UOW8Y)%w%==S zZI$d@$vg-|+WW-sc<-}TGqM^bD=?N$=9f+ffRy_fAngcJKdAi};M z2!8;79>kxc_%n+^+7Ge8-HE~3^Qf0kfKfF-JEHS^(zz)b(LQmwOugM4kS5z5KFGN% zR%kYwn8%u7dKDwGE#sVHhWY2rz%jkDWn2x+ik0`PRWbf?swmVvF&=8}7N~i=3pI0r z1UV0o^8h(tB9L!#kn@oQSz%49B?p?SW$3lEi6VqbX!;;@MkAYPL5xPtH{DQRE&Ezw zrZgUA4qxU(Q%o=eZk__Cnzg`e+B)|?{DEo&+AaH-UMr#r2Bt#GkSsB8wEdQ`>bS14 z;`ytFjX#ku0DisI)xd($2i63Vk8}@gSE=O!_M+>AN@NOFC7CxxUy2UcR2z9B673fy zCqMR>6*xC(vs6f{rjfyT=gQLspyW+*$6He~LoAYe>dr2ouA;~1iUi94ij^I&n|Dz>vi(YE%-B|78BSz0%r>USXoXg_%v&~@5xmoao+JadM7nV*g zo-Qn&UV8ZSEO-}k!G(+p4cn>NZX4Z$^&vYdRz)%5Gm{2cOp)JG*dyh&` z`n!64e(j~thN|iMCdhTY_5K8E|9fl*%It?NNsA{0Ei%v&va@bN z`r`8tykQ6vAJWu}NThU zSwSG%7OXH8=VN;F$)128dH5uq1UpvkOGA1QgwY99>R*X-rm?aLhDgT>Sxt5$p;py zQr9+)sMX`j#r2N>yv|bb725@!`W0IL9l84GC-3O?dQ5#Q|DDhUdBU4>VvlOfSvVE*EaND z%dmsZ4P!WazYQRfBH~q`Xj~CdFQQWYXYrRN6-xiwPbys7fs)`vrX^HK{j6D}GeB_= zi*$OeZ-c{a=1S<+I@2Y4O+cbo2tyg=xsGsTKIs#ZD5Y6YZ&lxQ{Z-Okvf2n~zf3GC zOFUhDED)4vx^D^@COOJ!rdCOk8hKD#?l$%f)*p; zQ4uUnh96WM&xCsrAcc_;!Zk|jC7d5TJR$>K6KYLO*UUK6MRC=OFQ1Y^`9WS8LRo3! z=G>e)J2wZ{3_Qx@=hTJ)t_#>ZzZp4Xe^YD??qkHC)9c$Z*6H;`wJEMY{p7QxeupfC zh7U}x!C`7|SL1j%GY(kF+hN-KUQIWYwwSX*DAJ><{%EH9V5&V!r4L>F3^WqHJc;hv z^0YMe*fmH*OM=SNEOapW|jhY5l@R%E!}lG*}n)s(*3jg}+ug7)I1c7`JA@5c*i&ul3v4#K0F<(D$mM&h6^{k|Ne z=$3KNJp+cN^(pGSNxbrzv4+TbrQZ;@68aVTvO^ipBwE9YK+;U|g0!8VGlP-Xgj+4f z^1LylOz{`iuWlIX=j;pX`K1lTKyP$6lDU2l6EG3x`s%kN-6`BD=DJ2sNtjCHC_)gA zaTYdOMcPsZmmgx!s|adXRcvN}4R;-$rm9n+co>3ySkkb-tC)o46;_=Ur)gCjlZ7Ve z*TQ*Or6{V*$~AI~wXtC2K48?a3&T?@s&NDhsPcVx3xJJcon!#7S8B6i0=$~uBNxB`o0O$!PcrKlp9|C}U-&fXqdDX6m89lk1P1gRvOsun*cHZtUw@>*~e z+xuvbRv!9!URWb}W{u>zHA1DOHKJ^#3!*(TZ?p!*_K^|Io2-eYSsDE{lj_?7k4q!# zSYIP68*UbK^1AA(DLSxq411RuP_G@{BFu)pymHEbDo@;iVnZKlLK&HnWE8)Crqj_( zwf>@(WD7T)zWap4f#~-3O()TkkVR?Hk#e}CRzr>+K@h_Ws@Xq1*5J8H*l+N?&8O_B z6f^fTpS1QBfu%LWKhVr4jMu4MH#kZ(je$iLHeOg~nqSaZ=Xhe}Z=CI~A5tj8H5;Xh zDV4@2Hj3-b=0>79rd7-3$Zw{!%aqo_V34NT4;$DCp};Ks5a?tgn(LMqCF7{MRhn!3 z1*l@7#dk>XC??s+uWyo#WzUZgePY2-)y!~GVJ;h@E8_;mIS`li+Av5tRlMI=)Z(SP zb&xepz|8Pgnw>wM1!`OZp2XOM$as4Iqr{SvLET`P%*E1yUzMaKP|sxYl$^kkCuf&7 z(*j4ScM8~7*+N15s z*koW~M=H3;denF#L=Bi-*ghKgC74%_G7M$>nXhKhl39%Vd0~DnQ*H3X^Tqh^pUR0B z-Li>)#+dX#)wGL_MxPuF%AS*B%*~YtIK+B8yS|jcR|d^x(RYpgc)}KOHS`GNSl!NH{aJkM^fE#B;jPbf5*Z6Bt6;ghpo_c_B zgL#j#)s%P1SY9_Wt!7n(K@Bf6D5S>l4jy4bj!-VX0cxCxa&d0Y)`6DE$Mu!kNjyT* zOC^{0BM6%zdWB;?4jf{h6EV+HvJGDMPm@0>5u37&Y!rxTf2zHYV*9`HI^8=E8jV}O zli}91BC&(7RbrizzZJXxOQwOr?9P|dD<>o}1ptiQfAV(FGxDeobZgOKfW1Dcv3`G& zW1W^`O{PwDBTzEBIMeW{#F<*t<72`Tkw4bYvk}ma^Ss(|-cf#x(N5ab9{Yj-N5uDLQ_RQ1>&7i0Qe(=SC;emYSYJcZ}=RC|#H2eh;q< za#vyDm8;}F#ujSkNrwI-5sqibib`^dkOhqS(I9N(=HcQVXGw8pmZ~MkjMy<7c?u^H z5%4s=seLNs&5wmV3U~%D4@Ta{2zh;_WRF-m9a5_LEu7tgU!MirosSCQVtdLGub$ZS ze$z3Tdq4E^I5y=%UoHCxcZhxTLP3r9X5i4o(V5Jk(?WJwK$>qT8CFc;_*|yDkwkK& z&@8}ZBKc`{sB41wQgDiXMwfFMkcyGgYTK{m&ZlDD*EZ(6#XZ8_%hkXvuP<$^)9Hm) zqqy#zU4(k%A$m8{E;@O+`?ToHzHscUvZ$9`s~TNT_0itQr}~ilqET}>z+sx`hE%%V zw(uqM0+s1!Kt2;unO^?FK+e7PTZklbf3Awu)5&m(Ee6(VfaBFhVi3|vK`UsmLyTZY zJo0k|w1wStJhB@tc@|SdJO z9lEOGgeB`J3R-xNvn*^IA{Ej$qw;WI(mCsu9m8e)u<}s+AV`v!q6CB2@_b~?Ferrt zI>LTj3>+~BY~D}gxe23~9#iLjljd!!p`EMuEC?zTG@B6pYs|c6kPESe<3Bi|Beo&& zjr|B9RSoREE#wC7C#q-UOU5wl=L~-_Q*E$V68nh$TGEi{mWid0Ylh)lMlza+7r+Tw zmW}oG`2HV1S_C%$S?Tp~b5&^Lcv7pyr^g~6ZfJa_5BcoN(!SPFcB<0Q?chuzV3fZV zq`xsqNKAZa-uVQ*KeK^)mNWS91qNAwP*di zw+PKqdBd$k9hXheB|hkh=sHC^c9(9Dw) z=v-xIsxhTX;W-XS4|RfMA0W2PBx#XC>ATc-u1{@CyIL}uX1_?!zD%`|G|lgRS&mY4 z%QVeL>14v~j7zboXjdqS!xvo*C?$(kwfLOy6y=ZSnU@HTj6CqG?MPz#mgrT=QPeY4 z*?G~L0jHa2j+qVc`tc1;gZxe`SzDLmPkFyaar`|b9e*dz7)h!l;hZ6zfEiHF@qBAS z5{^|zO~>(Z$b@ZA+VYyDkwsoP<;eTSISXz%i={V1?hKaEimXFsq3PLz$U-$?!~2^%KtH)f;PVTM~G0*!~*R^fRz&6MmqJ z(R`P}wdj_K=07*y5!h8^C`LE>0->QsBSR#h;*K#SkPsPPMf$?Jk2w&BR(K}9sd$eI zZ(Crx-5E6g&}lS%B=bT(ie?+)$MBlDoQH!$ON)y+DKx_5#bPd}%b>F;TtI%Zir--W z%&7A7hBs@>p%Fx^mlktEQVpq*%QfI1J8KjQg&d>re>s-1& z&OtpX7t09UM4e0L<2|e-PDq+8tI!6^h|&#;#uh5#hJD2BgxGl-NK6E%eJ zX|hTinJ8jk#)8d{yfE5Bh9fMe6^p$WT8bdWo>KF394~^wGg7mkLi@X!Y9mR_W6#SK zN_5Ml<_#9#|}x|vqo^xd}SHyr53*eWaZW2DZca0Zg&S^0{CtaOwb7e!=11Hs%x8_IO(Z$5X<2^meGD=^O$t2S?(0JgKLje}qAw|9eqR7(zWr`F1 z7ZFiK-;|1n_W@uXuZYC=Q87eP_Y4BqT>_#gw=R)qpRNQr_+tzQZy1IX4uamddHcxP zWc`yM|Gs+U6{sR;?+09HW%bDVr}3e`{#gG7eAYBZ)>@)5e!pLfoxv@B7N66K;*qqX zxObqUs6lXha^k-rCk7ZrYp_{h3V}UYBEOkD(mdmTZ#LIXzQ8*$lV6*gJ2p%9P~jv@ zBa~-vXV&t^uza#F%A88_Vo}b`#lq5Tj#)!6>CYP1Z{);+&ws*X;!n8;XxmtmwiQ6S z)U27MdE?~lncM>g{!|y;hc)d_|3vR3pl$6HQTbYsJJXgT67tP)+^<50EXQd;%CXp}~B;G;f@_fgBn38eLZ z-@X`ABNh;kPA=MT6}c!Xi|0hlF@GX2?ROT!WHJEMBmD&0k%WgtEdx>s4!gwMgsR6WKG#%#Y>aINQE@MX_X5#6{T02%Z*NRRJ$`|~u)3o(Kdts5|G*%7B z%g?egQFgl+yMSwJFA1OWzluxG?D_aE2tqGRZU894qGY!NuuIRjMSf(uol8)KMH#Dt zpDifc_U3rH+(O{TgKd^v_;xwoE#8p9h#b>9LyI{}a97-ZdHEZRSg;OuoW$FoV>vKn zEFs?Rw<7WtU$*xmK{YNJKW^mlv(msAF-b0ttK~fag$V%>fDxRi< z^-Xw)>DGcMM1%JvZ@B+DEYeP?qPQ3i{qYL?rZ}JlmO5e`Mc2BC8*(mfiF3B;%Aeb4 zZef{YbWtl5g|@}pDpsjgLwD1w$#rlj0VBPEH&d97vPDB8_?Umah%`t(k{Z@J@l-3S z=1=9Zg{>36H~lfOy^HYFR$K&eSjIj?c02Lj1oK)e+gK}|_}LI)tx@!aXckRwurjN-_XY5@r&@`rtrH6>l~3 z4=$bnKBt3fInzA7QO&-WRs9KYy*rZWN(|5FK3INQo!o-C+Yajuc;P(QS!9)Sc{i^0SB} z{w7E5y|@<%@L&Z`_zhTQC`9d*7UjvOyDHfQlw1tl0`>T19IpfoaJg`R1Vz{?Wr|@} zd<}Eqn+Y@0osugEana3-6>xAH5ZIO#p^v-ED78|w8m8kGajPcng<(+d=l5cgL#&#} zv|I63%ezT;Q^q?JX~L5D?{mD0L&5hVH%jSV%KZQeloi202C6~2wB!oQy{q6L7U#;- z6H*aQ#XpfPN~cFrONwNBTfFt;=|vIyWY5XMP5z%9z5PFNS6BK{RsTOya9}%NEc+Eq z4O}HclUCn^R)wQ9-h5nijVIah{(c?zXs=J#t@uD{?8)_=bk%1U@bw>%i0jdGeIMXx z(S%e*b#@UE-4(ilyF%$@@;p=jWwcTMMH=~e`={(>`y+KECE8TKkN(=d6@N|9U+=>1 zy87MpR~{ay`W*cgQ_WlWs})aCi&47?Gyzi9<}LgDEqnPEYwZ?m=@x6{7Hi=at9`3w z-%sUP_q)-3-u{UFIP2RkCLL4is-$B&>X~#*d&5}Ab<(j~@gAKe*$DTPp)fNl0}7Hi zBzZ+SCdKyaZ%ACQw_stqAVzXIdwEypr7S&_YH>>R$|@a5wU0$KKpb~-e0R1j{sp=W z)&<4}v;giZBPBfe2Q&}OV8?k11LOUx!0#)aE0A$G0|;q}dtO7Zb<&|+9#XG&$=8fC pcOkIrwh+;F{TeZ44gp;s+6*z*Wm3;@wMd)%+HR{xm}`{^{|`ljyn6ru literal 0 HcmV?d00001 diff --git a/doc/LectureNotes/_build/.doctrees/intro.doctree b/doc/LectureNotes/_build/.doctrees/intro.doctree index 79d28ce2ca4dad6e97d83479726fac999307a876..8d9d7609bd75f8a105215ba147bcefceb9f8722b 100644 GIT binary patch delta 175 zcmezIlxguZCe{YlsS=wvvIZ~X^y^_OPfg7>HlOUUOoY*5a_BOB_N+Tx3=CQKCO0l~ zV0<`v;e4sdcb0hrSsUi1PVQYU&iHlm(&Z|QKcF(wj4vjw&}aOy*p*+|CF9E#5{A delta 213 zcmZ4djOoWyCe{Ylsc$!JWDQ<6xpJA#ZCm&d*!+3Y{!ue8@`Bz9!R$6WVDlK*8e4ABBY{uUs$Bm_7OAdO60N$uHI`vDU(6lr|^=Wn4CxN@anJ5G>Cx zN=eQy%`4H(&CDw;NiCj|RWo_=1}%0bpmf$>uE|F>q%$&2*4b#s^p|UM&c^MG0GO0k A_y7O^ diff --git a/doc/LectureNotes/_build/.doctrees/week37.doctree b/doc/LectureNotes/_build/.doctrees/week37.doctree new file mode 100644 index 0000000000000000000000000000000000000000..4f406706701d1d3ddef6127e73550c2a2706b4fb GIT binary patch literal 271058 zcmeFa3z%HjaULj;AV@StQhW%aDN(!xn?lcEdR`zY5+q3CO@IJM0iXy81CH->-V>z_s&%3ra ziLAAgZ=GcSs&nd`b8o+HPd7-(hCZ9eQ*KQ&p#`&Z{?kqP}h0o9KV7+rnC* zSXrO+^Z80OAB0z0H|49jW;v)du6+E;*mqpnypn0{tov(WwOP*vS6Xkv7r8>Ql&=Ss zE0^%{R{FkCEQOQ0gX{6NX-DIsUX*A)`K%uz&xcFrC8gjEMV|b zu@YQ4eaD`{uEO=Lo%8ry5nkc?=0f3SMSLKj=a_)<)$;!HC(fTby5D;dYwPWM_kQnO z19TKJDS5?8Bd7y*dQ`21o?pp(XM@l$*D%bIcRFYks`-%CUoHRbqwiO(8D=wq#VWiF zD^&P45Zc@E|M%km@5BG!j};Q^7t)0X3-4+c_7qZuOku1re)=v(MPV-^U}x*b5Xi%1 z9BwM?All7PT`)p~pgya-N)>O;*x0|XqRITSKy3gc&*Jy1b&e;V`NeTEUR zt8l=~eJ2RtU#k1Hf;+`$$Hz}s>y4n|J+q8|3)z!Qg5%>8-jM(;8YH;9>Z132VIwT& z!U^w%N^up`T5N34U1^j6c-{p-7Un&ME8lqpQhUEQ zK7KB!HG=X&Q1>3q&dg4Dv(vK=kBTRYTuS1b&ni-f}aO^d9cX<4fD=Ww+D* zn0-(%%fJw3T{4J3*dI%#Gr^LTT}N4VHLCflhb`8FC|dCvRd2PJ52{K=xsTuLSG*UC zK^+1MbnR70Xu1+>M7=4jflWWQMLD(S3>wR1pO6I<*>q4IwC;bteKI(q*=(rOT(O9oE_s@ss1@3Pjp1pgREde#v8i8->8Y zTCHesJ%Es?vl^LZCIhr0J~eIVh1VY;x``$ZSP{+#lVV2>vj=M+6#+P$8AQ*)s| zubA%I7vo*$f$~MaT!dN(SqwFceT3)V@~fw*Xfr0zU->YUPmiuom|Z=~!La^r z!TQ&zT|e#NkFZXr#UH8`g|CVLQfuNMJ#(U5t5;VELv7^6Xe?JrgJ_loWyu@Z3IF%O zCj6L8_^*>r*tF7!85U}xr6eb|Y4uUX53fkKigN~kOIgxPApckswTB83U>?CEo$jT*BJ z9-bDIjHow@uO4=d&NJ>^d@pG^gm!HNUNYwVMzz^!PA&vf)$*xwZGL5? z7)*~jw)yR?;W|;sF6>%91%z!;$$7E~C8vFM-z~Fyu>cXjQ1$D1rKs82InIj1)pAYS zB5mgJLaDkiRR%u>bvBisffDj!z1V5G-!x}U%8taF-=wJA=-{J5(jCeUf` zIvYSQsVph&ok=|0FN@LnrDgr*K&(&7>!BZ2%Ik97uzuky(fuvcW5fH;X9M`mu&3pN z{C@9vwVGcHN~P+W7c@M-M7Cs51A8ioI6+Pn-Yv-MKz2%elbrCv0M0q(mI%FvCii6@ znta&v;B|XKTy<+}Ym<3&E9%QawYoA{tuIYMu>ol}!YS6F$p1GAw!ZWyp8?(#=s>kL=IPKeV0zC8HY#IXwB)}vd+ zno!)+$4RZKE7P>`;AJ+^vhoqcql$2$2U{KC-QD&#$Zq=Fr^1Q&G( z5j81`HR4|F^8L%i%O^NIDm48?gO2X}M2*OVF^v-8@T69(I#`c!vB`?oLcj+nYknhF zcxv@Pcx3gV#fQS#bB7*PzHkzg_x9}5;k@6EVI<0VdtJ`!Gq^0?KyB>Od^HA0cPuN; z8fX}Yla9-R%FL6vIZUH}GXNEsMv1841+!x3y4!Va`)H>%Sv_5te);+H(~VOnEJ0+F zcJI)}b=O5r!2xN*fg>?(xNR(Lq+);!O&jho4sDze+K`zip$*gMzYjngrjezM?(*C* z=$4>*3e$ZxA*+MZi?*0s8ug>{l$Wxw?mt8L#UuUUAj*Zc84 zreKQ}+s$cQus-8Gr^O9RZsVx8*mb`Og(UvoQpcl-se^DtTKL(2p|$XbbyIg^Z0!-x zFXrkJ~&c?RTE^PQ~>PuSk7B8^N{<4^u;RM|4Y$ zCg~^@b6|VB7$Zdt+49eWxZ_Oh!skX8t`!2J~)33btwVVbiFGkupvvQNq4p>Wa4zc<`gt5P6e# z@K)L|*chaVo&9G9p8bl<{wK^t`p@1pN;LZt%|2I!I9qI1H~@v@9K1KH!Me9rC_?Zo zVV}SeRRBTMC_wlbTnWGN*L5=ONE(J^qN$)#Z7vm}$6BFmF0b#Pb_ITM;1wv#3jF*4 zD_|Pg73j|61F3I@vO8{xOD_hp--B-Dlo0z2PFZ0}!^O8o>$etQ$1&T{VzG|JcSXP? zVy^JTDo#}_QOJP`9~|f-#@B5;yMn(v@CsIC1s#;ZK}Y^c9LWZC$3dq~5SPD-F~qy1 z4;6Q|ZYkja02}l?9QnBwr%~(Cb?<;TJ6@?xf=dv%#{Xo>6EkC}4ZL}1@4oT%z0tQ7 zel6Uv^$rwXaPslz0J-PM$4&65gO71M+}d8LA`(wWqb>g_A(R8sPS$VV_9pc8CmO}B z)=L{xE0u#cdA?YIV1%aX2Dzmw3-duK;6T#WTllja0!-(16d~DAe?+~Gt1)P}UCUqV zHqLhHpw|dH%?K8#e6W7y*&VG^upZ>*bJZroD4*Tdx`n@lV&dae&{y%TxoWB8*PxJH z`3QI~Uu=-Jb>(}SouPE7#zvBSnbpkn`h3xwgI0HOdSd31muA6Ps3J-dr;S7j2yIXY z^uE`u6d@bSqz$T|PI72@2lE&nzc@IS*^-&gIl8S>i_lqfs#iaktC9}CM6B%NAPNz% zrAU`vdJa3fclpd_1~84KC%qKO_cmXeSH|*`2;e0|%wfkf0dlPycy5j|^w#U6c0R?@ z!Ul?j>fIVeg-~|thhOa|_i{KHMndpUWGY^nhQ^PkVdi=Zv>Lp##HjZ*94~nj<9HIV@36 z>ARy+T|;y$1kMf|har0M${CTr?@^@vv3Wz<&YZ2p*~um#sb4}c&J&C5+#tk43iip* z!2923W-p&bKZanlsVGVz^|~N+zBp;nI$$e<%nuBY%+LSB4s)_}*E%X1Y*uTq?C!Fj zk}OMEcuH{a3@1Z2RGyM7c}lY6DdB@@$H8MdG~sJ8*Nb#W`Y^OD-C^AI(&kAM4@eUC z&)vu4?jIc`CIr*{14}=xf#?U$vXfu@9fRwS=*E$t%TJub;d6S9Sa)Gj@ndFVer94W zm#;R$iH-Tn*h}Zr6ZVrRtr7obqK|Cj*mLfQU6X0W^4wN4|AgUv8jo>8j+E^Lo+A@~ zB!+;f=)SMdJOtH!zq1(sOl-hJ8{wBM(gk4N2(Oz+K9^%MmnaqoMYv3PRNYz3{Ol+( za|6@G01M4z)0Kc^*7x-xG-tqH8QFBKBGI6&11Wae5hO zyekxvLf>#kPY%9tk2)VS40=y@P#xs{+Zb}AY*oTucFd!3Ubf zRp^VsQn7M*L6PTXYQ}q)w?04Pd2@5AxmDVw@Ie2dXX-uC`X*lRSBSVW?X3oMRmV{H zf%5t0?L#Y{U;h7k@8M(TwU2PBR1L$adNEwFjv>c}^fE$o1GKOU{SM4zpxEM*ukF50 zl-F(x{khcPjFMYy8!XoS+~t|g%azSnUeUat73r{nXY-e5A;%!RpyAqY=fY-r{&Mla zjFNGibEROhaS@6MsJr3EEQ|T#*h{ls)|>SWB9bGpS26z6>U@lS(`mEX?2bVW1%(O*dF8q% zjqk2bJvmZ$e~b#FTgeVJq|mwiX^Hqyta+kM;kpl)*~{>}DctPLEHQ{X4GIKG=ZOXM z{3rpvl|;RRY&(q1C%N=?kYSnH3jNmIZPuF@Iwn$qA`al) zwNfT_GY_xeLZL^>TZRN9l5~oz2u6kLPA47EaSDq{BPC@ek~gpzTpwhiD1hPQ9Y18q zsi_-d9@d%ggA}NaBS1J3({q$aOL=gHYP>1WJQM+!)@D_bk7uAD!exmAh_HazV|8W( zMz!Zg`izYfPRj1})wvKcXUan6(EE4AFc4+QcNu7VPC=F+>F0IHVTm}ApfBv!} z7qQp?Sil~ep&wbtyrT(Vrce{&6H$N7DLHqUr!%r;PTQ=N0si2BmAx}tUCh!G+B>skW@IHE!SSuftHq-Vqw-L_-AU7edb^tA1U zsJRs-LPuxRr~S?oJKvqLP7)F2VLN33f{F6Yq(z#U*g@o@HXh|=pC)bk*372zD-7GoU~tSR(Azv3 zZ5xnx!8kdtp|Onu21BEwa_nFCgOi5bzs^3?Ta+};7rP z0bnA7b*w6hU+u+BLlN;Vz%D4Ex=u)pTdPu>F|42vr3)b0u@}OjY`4L0>pHq4lxAY` zWJ^Nnu~;XGgwn!AgFFF(h0-3HTLOe6p2Q7|IoVG1Usl*XJ4)bpCs-dlw9GrXoOO_9 z<*c*G-vNrr7SukP)%&Qi(p&Tv?*7Ow+qQAr{V~bLed{@*YYv`;Cq!3|5{2Rt^p#rN_Y&%n8 zhms+%D_0^CG zLdAyQl}o{3ta#~HUg^39-Yc(=L*fI_SZiUi1RxK1Gt)0!&OW+{-p0qNj)Zl*Q7k6l zp-KSxDz)0!y8D^jggx^NPZKtL7}d>;irKv7N*3 zk8%!giBnk*xVx{1lWubZQ)!Q0qO*TwYve3g+G=x)&A%KaHm{p>BW;9OL`FN6;0}*~ z6$J+)>B}PIrX5iSKu;E-5|t;r@kb=AG>#0b4tDe7I^Eh0%Q>&uJP5%Fb3($T>HL5a zbdbL2p;9sG2kV3;lFsH(kmHz1yh-YlSU%!m@I+iA#0)z5g)zuaY=)$sTfb`BO2LUi zz;O7&i{UxS7jlHCy)5#lU8YQ|dCz5%Q0Y%vNhZUyB-^dD$X;ma0eM_5NY_I5(V6X5 za=I6Gg|d@;YBBQUC@~VD-S~w1)2@SPOS`sN>sDmnz#z$L3&3$^>NjE=;7v4-`lK~9 zl2|@Ayz_`Ip?FkP2^1W6|G;9_9|W^(4SnXX%$E6*ZXB_O-u4W|iZPce%S;YoHNAIz zZ-J@~1QZ1||70hiI^^|qdh};VOOep$3pIFTAB5dnD^(k%;sUfLN~K7pQV)11$~6=v zrZd!4ANmrCmY}5e3Kcj>aYnUD0Up{5hi2lw2SPh>m$KOmM7jy+mpN2(cJD;lN z{6@7t;T_}R>gO7D^#dBAKp1X>Gi{q^nVY|e{~}c>%w*^V?2h$I7&+>dPd$ULGqaP^Q9WSphc<8 ztamR&-=nQJv8P_$qZ;|^#V~^`FVNY<^n^F_@Z>aRMBS~Id>B@l`kwT9T7SG@9&N}Y zuHKH1j&}*s*DWbtBBMfI3%6gBJ=bK95?t40j}k=JWY5)?Jxc3>*X=u@boKPQ*^J`l zzotxHb-RU|PyG3JZl`?WSB+X2wOp3pgkbaTdBn>%;I)&J%e|@O&&_TnQ%Xu4Tq*`=r}1*P%`NbSsF&9$MvD zY52ET~$7e_)otvD06Ln^4saD1}w0 z@C^e8>su=M=~M@7>>fpWm?%j)b)=+xI&Z_*O%yGl*j-fHLFTlv+NoLk)ckCk6QDU3 zLK&Lr2{v`;xTE;GLE>1;zZ64g)Ed4XAd4x+zwJtVDb_H`CvK)a@qy`rG zMREWTt$|G8l3!lP`*5@?;(Fzkm!K!kndfs$@V-x*IEdEkI;N4*{|^?XrE0#iHe8xRhSV7yO*O&oLqLna9SVG9X~?nCVqaoiF{XLS+N+%qFqD@hU%IP`+aN2 zF&r}aZ!rRlvew<*fDwM{{()t}9XAf(4HYJQ?c)Y{0wOcv-a)WnVURw=pVprn900m- z!v(4dC{eJAmc4HH@?cx3$fa-gnApAEtme?Rz+JW%k&`nAW>vpu=3?=*vq;Oa**$Rh zb`GiCD?&i^GiJE`8cyd&h*KlY-U${QQks4JpBjd={NSi4KM2Kujt`KAPdCqCivVe` zd(wnx3?k)>P}7A3_&8XOCfW`lkY!Q@dt^nUWi{9`u|Z$vD-+&C_6zH2?MEPA zG)iugBM`1mBVJk+ZN?Spl>Kqm*dL-6B>vvgS#c0f-~*D~(*|pcx^d)yvas6KOw2@ggD9T7e&Bmv-uF15KTdVIJr-^$3eyID0J$TXO0|SxFv+z1QkMw&c-? zO(R>bVRx9|?C#h#LiNML3)R^ddy9v^=BR0$(%E2b>`8Hf5sAN+(CZ1r)>4IG`W_LZ zTxbr}pZTDUgPK!p&Na)Esc4Xjl0Dsuf-VGf=m{4M^4!S#(panNaPgFci-(gQq#E*Q zaW&UO_wl_YT#AI6#~KXoj3U&f8yk9Y@hW1$QKdc+6LIl)hYs&=5*%)+8z-FV*l5;Z zSTG(D#-B}&UT92i5 zsSL7^-niPI1i9)e8E$Ca2=TX4m9Y z-XFgvr|fmRcvMaqi}kgB7Hj{oj)i{(?QYldSBz#CRR{+R>)_s0Ou_GW>v7<) zjt8S*9oG*K*5SZobyvq?*ssqCvKr%TF328l9G|3?XW1g?Y(z*e=8gu6)s$YM%02he z)2^`Ub`rZ{?+biqeA{69_MQn8i!Y-lK6wkweH{a}v6jQFHw|#4s&Z@p!dMPB>&9@F z?rhz>&@3Ty4hlrraPD4Ew=?v(68FM>_!q~=kH_OKGslmNaSgc;5hzqKpUqF7JP_n# z6&n~iaNaq-r3tde!;v|ALgKrhw6i}U)hMvmHaZ81RaK$xhh5Zk zY*C$Dr$@JnXE9i}-hqX0Zw$VubczMja8^^p5}izr532s#-}1$`I2E)cKV5ThdcR@h z-b;a-n@&2--f5b3zq{$eoMP`I#E!UkLl>X`DO2qUs%i(`Q&((+#LLfBaQX|mhQhDx z871!}7+bMq#cwp3xrouxAJ>y??}rLtfbS=9n|2Ln$Hak@Bg`d{k*%O^q6;VR5Y;M) z{poq;kk!rjF4WO{U%q*oYw^29bapR_Na{9JIg)d6$v-0kJ{3EEo_s@Gb%k z#sTXJfKxFBa|FPH3Uhr{LbyHQ*Om;=#YXN;Pn58@YFS&rCf2^N4|H>l93brnBnP_| zg3jb3!C8wFhPQh!A&CU1St60}zakjf?)V5L)fC>;Cm4Y;=X z(s{mZL6Tb6H@QS{`f)Yk>Jm@JQ8D$A4c~g<1@$^W!6fPd99lWZh=mQ7nz=f~?go_x zK-hq!aWP3!)o7TA1LUY+z{y8OkZs;La(TzSbS3Rf8sh2*lE`qo5eY5GOS%$sgGe-N zpi2Bg06~Qnu&NFTu8zx!o0Td+8nU-IG0z0u00Qb82!%xK25+e2Q`9|nVRM0RMbujb zFA-gu2t4d2-OpQ~i?)Msx0bbZmnY5M&*uqMK?QY?8YE4TQdY0#n>m`Gz2K?XfS9iU z7SaNZcG*$xHe%Axk-Nu5@fwocMKX`9KPxXT$Dr@c_$0g6t?ZVmvs(5)d=s;za||Xh zi-3%VWj#3@@5(|A+l^hH?;B)&B>c78620<9ik2R0088?wpj-ow7DLC3Y+VnA7L@XD zipPP0VH4lWZRWryt*-Yg@kH?9pV!VI9@ z2PS@ycY$JkvqnvySODR<^S}9ArLMQGcO8|d92mvhX z&9Ms&v}U)!CKi#eL)B<1l2ZB-9`!mSKkF>;BpT%8V%Y44W}@qjM8wNg!Fz1Z$0IWe?ms zL8DO3DaVB!KI`v>AMM4Sgf&=)2po!KtKEW&C<^^GDF#WK9u(J|0DOeQfC7Xlc6 zMZm5tDA;lk%}5F}NML!+eUp% z+931*4RyLoOqZct%Na_7vSCtVd(-&TO3`*;(0~dWxOyK zRN$qAgi;IevjZJij@n}m~7XAB731gii8AM)q;zN_iX9FH7Y*X4KX&gemov-Laz zp}hsl7P+$3=9cl_8iTF>CiIHkgk)m82{pi}Tj00XUj5-~90#rJ8plC;ZHeP7F$+D) zzTAHobBOt7%?U|%F{fFgU2F{sn%2Tu$U-K(%CxQs2eHNAsZFPL*&%0V_BhHSXXw#`ZrD29+&hIX? zX5RoD>NK+)+Fc&BcBl}EE3IwgR9|HI9i-DnPE>6~h3%#_&UMGLqJNkO799?fb)!LS z65n1`tz24x^++peXvGus9&HvcY-li&Bw!_TqD<-w>0fQ*v0N$RSO9!n7M&t_1G}u3 z23l4shVsFCc1Skl`^~aCnZ|$qB94)YVXlco5>^sJfC*uRvsO|mefk%Yk6%IliSWRo zF=_Nr9`z~1o^k+>#>O6X@{1y9P^O_y@@|V$=DPX5LQxwe%XC9QOB>e6OUCy&K!iaC z;}{F0k&+7qo6jsr=Y~}P5nEAaijvJ8wP=X z3j(5odt4h+m6}v_D^nga0ICbiL_cuUNkCN!xoP%sW*t=sch&<~P8Dx`d>p~)+|C+4 zw2gM@#eNj+#v7*~ckZW6T$lqkhC^P@Yw49SBa!=1E~c9B5ORT7stK<$Pt_Dr!xMJn ze2Ab6f*z$X*`Om6TFx26)K2pgpXcC^q}Y23Udy1?q57Aa;((G$zOzqM6dTr?YzJtk z#=p3K!kfN?Y;#(gOev^j)bp`1%vk@XZ)Psx?AL8FMkCdmu%KN#Q_nfIlTM$D#O_QH z88u0o8CqGZQgDUG@)M9*EM;WE7>#hkj$XzT{+4%WVSZ*U8Uerb$&CXldq-G-4x$I&m+B4OX>z|PpMbE`_?MVwiM5pa8V1dL zRg`93YgH3WwVtByG7pV`%hauNYWdi|Q>@@rJf%ngaA_fcTAW$b4h4arcqKA)OhY1z zMzsNa+8UxMMFwcSZ#w!V5kGdZmxGjH1AH|-IfKu1=wL@egHyw==L)H|X`{8Q{?-{0 zgTt3aJ1b9KBclmN5;jF_iP%vAB3|YwN|Nf-@^{3QNOq#qsG4I?-}91iJPLl!T-u9v zV7y5bUvAB}CORe@zZQ@HQA8p&leANt`JFSLGh2l#-XKY^iEH z0c(V=XbPzlZbnOF5QjB}Jc_h4?}U~nygifIJre*Jo~r|UP*oN2;J^_Dj!|dStMm}9 zI(ls*k^Sh}MsjT4m*~1j^eiEw7Hpz!5BD7hm%oRE!^L>%N0)TjU$yJ8&rZ zTgM%JZHsJjlt9*tZG6|ZNWV1pH`_~oZHqL%_P<5$b>kKJ0<6#>y&|9g0rraglCfup z@``Y8$}94V-Fh6@D>5GSirg@OSHywGTI`NjWSXLuv*dWlBHAts$3r&a3~BR~Snmkk z!qejR1350DclH>Eg7T6ixwB{Aa|kM1E#hvxRmAWj*|S17e#4mJV2a~Xo#cp)qRE^? zc@lA#Y^kmFnEa_;Qi&*xl1e-Wi99A>{CVRs`4dAAPGHM#TzpYSsPve_rA(pGsD=Bd zrqoih=KWWOS7+P6$lt8sJ@DF;Oq1=-l|y}-^9 zib^dm6_ByPMKTsCmLDg_I80WtZ^-FKNE%L@L45d4p;sywoJdR=lrX8*3PmJ$g*BYk zBMYDgu~Xq3Hd9>(=zL=axA^u_(dho(B7(w}e$^7ry}EIdpkt@mJ*HXr>MpH!JX5U& zi>UmB1N_x$G4Gw?INs9;l_~qegS|hpvwZi!v;1{C%ZCiOy^%vXoKCYuvwZ()4)F6{ z2yuDve($X0j37La@(L*U4`jo)opY5Y6@=i1Ln;YK9wQJa@=c_Cg@g{buI3OH!)FG@ z@NZfSKRW=1on{up-K7L_z7amt*r)|pmgzF2)(%8bE*96X6m~41#NW%$;y-jH-ycLt z+lms~cWVZTG`y8(c`MIWD_NWo>#e*^^~i=xd8>Z3_DN%_2iV4K2PR$yt;gEyNZSoM z$vMs4hg5+IPJE(#>8!3;Aq>?01G}`%ftHrEh%f&;qeaxs(mGniODEb(<5C`gqHgUb zt|CoLLI)XsTzqZOMB6?|U0XJVGYP2Hh@t$eJ&Z-scMgOilA^!U8$}nMLp~C)#QAJl zLGT(^YNFzND5ip~MVA}7c;eK#_ntjW4tmrkfsa0_fI(g^1=&i_n5@>9rp!m- z6t|i}P$Hy9N>c0;MkyJO>$Xrf1784#jjfFQ1s)u~C5Rbm6I4x$3f@szS{bVV#g+s& zXQoNa6vPwBa;&hss>BiO3|6Ac!aym6h)Oh9&xzxhQ|8R(YjY>nkp|L9Qs|K-W*LA3 zVf8B2)@=_33#nlg0wxQsP^C%@Jj!7?Nv`2E^48c<3NLhyOj5#fl7>GTqwuKS{FXQcihOB*VVUNKV_kONU6n?>*VTyo z5Vfo6{+?wunFZI9xq%74#~kA~KuBvx8$M(fOBV+koNcxQfOB`m)5EgeumD&bY?*C` z1>x`r*nU_T!Z$!{3=8w|5s95);dpy5v1KJjoB^$B*3CF{c;9Lwwpr_LE#0#a$=Qg4 zWb>Wmn|39tEsDk}K1sB*Tb@T1HSQH#gsHb1dunI@|0b4~q|cHb_S*IvB~9t;x-lf* z0H*PupCjXSzXzWb?KF$DwbPCbvN6;y>0c<(rmEGVBmou6Kg+EUogn+J6UeAQCe4&6A3Gr(zfy=m6n8E|16jI_hnd$QjRp>aT_=QzN+LRir zMI=#bZa-}0G(;Q<&WSD9;f-B_mj+q_Qf|J!*C;m^%@Pcv+^n55zB{|0$T6JSPqf10 zig!-g)KZ*Sk_z;bk*>}Ij5#P$o7@;N78QKFVdGn@*9XEn`K`WTv0gP;?=L!?W+l_C z`-b28cC*3Dg7E%M6``O!#@Azl^gGBd);p8QNm+!eKaj=tM+V0BUs`N`mqA;9Y&*>? zwyh`_O3XMbY1z*9-MZaboUG!1Yf8BY4>3H(nC>Ms)DLT&a8B!#LM(ihKqjUKTpBH4Q3|FZD-%Hgy-l$aF7Aii|f$%7ItJu1rzm}dO9PNH#0rl(b&!EkRW~THt!&OE!if}{H!zWN2*^=q{iv1;xkaryl-gw( zPr~Hr`B5ySL%x63N0xMW|J0CVu%*!*G4j-qWHj{nLz3aJ;~yb+EfpZuj+!lz@z6kw zf(?cRVroz}vSKz)*Zsw8s<)VJ{oDyIV1;56bZp7;LzNSwb<$DLQHecHXH`NRm!RO*I#p2+m#X9nl_H|GAYJ5)T(XHc zT1!{vwqsixl_blNQGo9(==htXF4ee_(&7D)QA~bIAUM)x?;yyt^c0^?e|oy36Fu2x zt(%mXfr4+%2=`g6yqs0slUNV;_i_P5B5@OZX|ZrFw&=YrHnKPFc-hz+&+EoGsdC!;8FY|9HG?&sL9k3ASV201WXp2`h=eQ7EuGGvc zU*lS#3I{qLYm?Dn_ui3!;?Sob!&aO{I=p{;n2FeRiuY{EuA7p1SMPPYumhLYpQq~2 zaJJDUY_trplIX^u4y2Y6=LVCSm*JSE5FT4Zicn!yIz&SeMl4l)!N+tE2<*0cP=ZLL zy9OZBFe0QR1;3EvH{cU7xkIuBGZ{w^SW|Tdv5NX^C~f^R%DHhSzbi-Couai_6_v$e zmvlm}Orb|?Kp6ipMoXErT_QyTFU4uiOi!C=7bdq@IWqbN^q7+KLHeGjKj|H%0jFORy>+~G0-A_dgWOeJFDNrkg#a!@k&3Ltmy#$d-1(c{l^h}lNNRdyE zKI0p@Xjp7ze9a#UrO}{V4;%=#2gYW?V$P!y*d*?T(3NfZ8jjQGh^C6GOHM;|QSm#+ z&u+;x*2+%e9C&*Niwzfl^J$%&Hun6+ zVrthdQ7=$$Op!xi_ot0R;7;8*;t=?52e`Ot;3;#GvJA|X-Y{qC0X%Rz4??NY)x{K< zIVk8?Z@|jKkpyeZ^vu{LvTAB19UD0!!^OuVUW;S1{q$i9E=GJHRII^dN)R-n-V`=b z{PZ%A*|T8C4wS%vW97<6T02yk_$y}$4;J2qL-FPJV4B!TW`C{2PGV8ldb{07+xA?? z=k9#gw%@j=soL!;$xeIsVa^j&;tm2)HZ&7D*!=@bcb^=D?l?W-(0?$P`M7Qz*<+t) zhYh?(d`AUoU@8EBeS-oc%i4tiE>yaYp7uqES9{lex|lXtRnwu(a_;~=I}GrXu_?xP z#18M@)&;H}y+-6!YgMh6B5`%ZB3wXaag}mVWQUMXB;C%^Y_|>A>EAAB zl-)0;<|@bx@aLJ=y{^1pe5v9^z8pXZT-7m*X!uuHfgwf1*It>e< zY3hDy3Pr>JLpe$Dr|LpE4pOG(;;QCA6VskRN}o=1&Sjx1HbEDCf6B@v_r>9Pi!PPKYqNGPg#hf=nX**v{vvQO;+ZoBkA^O8;%r zAyg~&ZL`+hN@rkoEL&Z4HfI}x;eJCltjHuvEQy6=ST!q&E_gGax2h}h1g(*07N-k? z;FLAKdfo`2dEMC6_%3_}8{R?aZ^+qD5FoOB&_Q};IvugD(ynb_whB*7 z1Q-mLXQnsD(jIC&TS(CH2<)7Amx7g*hCxrlinG% zVN$fnA|IS1oQr0XK7C*&O%I&-66{MVT3sD;vo8~;_AcbSo3nod{tisJE&(hPDLR#> z5LP=tC5@wZx)_NaC$#Tb)X;$jYXY=Mu%>`=B$EN5V1;ZsJLwf?`@XNwLOar;79+H% zEVyoFS_qYXTKZg%HA}QdhL%KSSl%PMn)LEj=Pl%2?y7dDPMJW`&+R1t-Y6#-*@)s3 zgN)wQnS~sj#~VeL>U!{+OqGAzd3vsM6D6u^=@E+A;3jLQRI>|fQcRQJLr&pHXZn|v?#!k0wDx#WN zX{Ky1t7CTa;5glOQqWre>^!W0J~x4_PuC7Xi{dh0>nI4N%z2}Vk|mEsxRReEodGh` zc;!-YO+=|D<~@JxTAc6!awQY=VbqQwOrh4-3VHk*Qp*9~u_7YcusM*V^cY?Z^8Qiw z%J7qLNXLt3m6@D%-Vr4#T` z2F$0Jw&yP|@15Bs+l@}-=9h69>D-cEF4NCGY~!`DP5iXZ9M-))gBen94o~Wz`8Pe6 z|3U`-3l##Mh65X0WSe;9oeouo{Za}k3Jd;1u>`-q3Oq<(oFNzYr8JU25eJRvcYPu! ziCp?3YJ`pGU69NhFM~E?G#K1Q$UKn9d{klL-C4O)YABOSVdkJt4jfNOrA!JKS8fE~ zGyQwGb30d{mbhYDtb2Z@!BR=>7LsgopXtBI2{7M8y4lX|V_B-BIDFAP0x0G74T87KkT?;w8KK zadNX8aW|dc;Q_m+*=DWV0h#eZqu@3)Y3qlkmv*<)FJlg9yJ;rb%kMnL86ct>u*eW< z@A%Hn{k1_TpZD`G?lJuT72Oy|R9{8$XG#52_ZM_}^k>dj7rqOUA8;d(j|5^3Me|`F z)x1=yf?bpnsve;K7zb(SrB36;!=C3o(D^VmRmOW9@G9y4fECS-lucFUwVC3;%y|>yVFj-)_S!idfoiGIsg;sX2v$uk z1QlGu&cyh>UX)F*8i0bwSlEwWERS*9%76rlB~8ZOStke7?2I}=YT5=nQi z)z>uflsPP<8vvjnBaVd~S`x{tnppkz(C`c9C>L*gOVnE!*ReXhKQfBt-Vz9onukM_ zp~+=f7@ne4mF<>(4@hpqj{jz7|IhU3EUNPFP;@Oiko;-Yo~L3Y5tWFsYsSR#d97(Q zA0ZjGe8!WrPmdXNr%Hz;rUzxe9s@>Eww?G?5T^dH=z8BM(RGKVArS^WpiG38?c^Hn z(Me>gT?53zQO@$sj^^sdz6j-r?Q5-uZpMZ~c6Nia&02Scto@6>kgpna%nkiIY8R#!lHI9VXq?}kE49^ zO)y3S_ZO+AJY1HjVNucEoD?F8v}Hj%9Qc%Wa#fJKQmvbsPo@(L!do?0976aGRSAHC zfRGVXF~JE9IzgB_>JOj~lt9C+HL5-lhvoq|)ZLEe+DOkS^!##Ahw>6q)dAc?=9j`M z&)Zch5GO>c0X|yeJ6}{RYyd`ThDuSv=1vMluB2msv{Ev)RDu^y>n6>z0~i{p`(tRp zXmyQ9sVg#dcw`;9`!MLUliA2jaCu;E+Nnat#~a3hXEf=v75C zP*H@m60WIFaQGFDZ>I@R2V8!^q>9C$0jkvv&-gIS{@XMfp={m~>PK>OD&mSExfiRZ z93BDjIa2F?j4?=D^mTZDWE7;kB@moV>BuNZe`p{^!8*eNacFqf8x{nmG!DnQ!@}@n zq=H0@+4{5ou)`l5T67Ex!Nat73@mzjj2cm3S~swal08@dWjESCjmeQj%I!DoUMq5j zmD?Lw67~SCyOeT7#I7!q)B-JoCfAD`h^=6>i6nq3R%&O_G)`xbf?(O=pW zBCC5OQHb7tu0*HgI7rGD7g27B>KMyiZn922zp;qeb$irW?YVlaCSCU>qbA*{8%NZn z&yku0lbhmI1t?MupyQONLJ{q+R11|Z;)6#rE$5a98V+g!Hb)ilnz9a6t>|FyRW^>c*9;<5hO0qgrze)9GD)gq&LYMnFRIH3X`WISHbo^A?VW>- z>Y88Ahh%Gpe66m!`&l%qtUi*4g%o6^3$7H-mlM!aaTMhO3O`>ANGAo^De$t7a$uuet*a-9WuuF4g(&!x}kT!l~722?eoc_f5@CXJ=*~ zepLRQtmPL`SnXU8Fj`Vi$c1xVW)uaUTu=EXO0e-Xd|hWUHba#~A7-I90}nyXzMjoJ zy70uNrPmd6(USqiH?ZCEpn$;Q^VTR2^?)=kN2MDPhLSu zG71hJ7KoXFB|wi6qqjo7t&6B&dLgpBqx}P?^Y_L`C%&c~-ft(e@6m8Rp(q*^Tlef$^4@9b(!}Hm);u`;pFji(GX77Asnm#1tNGe_?sK8C#InKJw+vaoHx2+xCsziflQ#}#DwFV0 z9BCmqZR?!J1=U8dSz82JQcW5+?Lcp}Rx9mLFCK#Z_2U zIf+((+j&w_agqn^mc_KLg3|Wf;^CJP6B1nl!F81X)M)R&s2fML_fKQ7V|ibswWkA$ zQ<#{Un4Q?Clf6k_hX$=YR&sHg3SuBh%dHyo({Lan&+bVNNr6Z;r&CT`E{}LKrCiVf zXR{8KwodhOCv+=}6{y@huk7vhiaOQYOP4ujk5p@`n?#O?FmSqfU|RCg%h{PUYwIew zO={wJX=8O{8jk}&1tr%tSws=w%XqSRxv>|JZ{k~TsS0BNsy+KT>8v&afHqI~NR_W6 zhvSe++rr3zgAnm15hUtuQ2;RX6vm|~7vVo+b$!lz2vOmX#!Wlet76 z+F5*cl(WbrCGNy?xxEqR0+~>UH+EKkIm%hxsM5k+tx48S?1U&uZ7U0q_+^2*>5c)M z4E+g)oCy&QjEewg=-%+&8kH3SeEhMU$vvZ-$y+;$ro_OHi+R~!hRMVRX<{cmF|tWV zspZ^N5UeuOtJ1ec$^Ma{1X{Qa-S4s)BqoWwyX_i+mhK4LPUA?)hzSX`O{4UmK4_JrH#+_6cY=VcBP#hx z?<^dYh*#3da;hU0zeJ^UQglFzb6TT7%K4%SULujDqptv}ir}IQJb3$xOBF2&)&kC% zW6&3Il}jmdLdZwJO8B`)gyj38-XrA??YZH@TSAYsX!#gHM7g-n@jxloIFYij%PI>p zI4cX=>8!Q~WO0*L8q02l_eV#v)?0$XF{_V`gjR-y;x@{DacHeae*>-gCr7#F82|_e zJt3?&>Gy!XJ6$+yvQx|hS75~{sK3B+Xx9p?PT}btg4iPw>*sNh(lKH^AK*G&7!&jA z2qn{)XiQHuBucy}2W=i0u8q-i@XC|jp4Jx-XGo3=5 zgGy98-~q_S%-@O_cuLbVaqz6#Qv%m`d2i7?wP@+IxaKnzM4v)-8in}pmR9P}>p^5} za%R|m@4=;pq)6YRbtMcD+u zsCSg+LR5bkFq?pTQ`rPR->t`ivk4AHvk7h-Ae+E}$LX)9 zUn%#H@)owm6x?Bys3hLU$-}sSR9sbOn&kx?-$SoFEeS)|e3eIvDyG{z>j!OSO@Tid zL`h)ZTXf|J|GIH47)B9FM{=T>|qVx_ohIu-fS877F%tsx-u19enO zuUE@rjA&&;nI?E2;Nr7(1w0kbYa8^7GU{rjhS4`t>)7vS$D?U!naTuI zk`26hXz#x9^}W%z74p z#dPb%_0%&MA~QQVJyjVS_Xv{I(c;p3>9(zK!lR#jV^$aeBq%Oro{|0+cjK#|Vk1Ol zfYUs{{>!GnO~9mPr+Rae`zo1PLL(-QSJ17dfTps)o-u95QfEwhIst>+Mkrn!H9^lQ zjuqV@&CvNwrxVH%E;Zwcjw>W)G8ceFM$cFlSfrFe{>#h4UElD>O+!Jjddn161+JI! zQ6yt_>l7HxOO1*j{zm5F#pz2E7iTYB8cSU;uUsmiX%iLT<)zdH?r_f+bEHkt*OMqg zn^EwYSF>&JG^LPQY;!Xoo*qfIC)99{wcLj%Jambx?iNi9(7c4dWEvXL?TWfWSUi(c zFLtLSQJDgIe84*lDy0HYi^U~G10nGX90|8Ht~98#X9Ut51%}yu(X6pZq{zj3aiK|( z-LS22rL>PD2bPRa@%mEqdY6n-MUC@C+}1#RemchID*eLEc{q7%oPhT?M zZqTA^H6u|C4V6(W$Js<>uLS*hIzChj~c~BDZoc(;aI^=pikvI-2lk}YZ+rOT-#@^ z?K9W*nQQyZwS8t91%0mVGb7(;jN>46=g^LWL*LGhgB#IPVQ=C3Asq+Yn=(DOck6Lr z$HC#Ku1%bPLblyL3x&~dE7$G*1WA3!!5-RV4b;6 z%cviq&Fx^V^dxSFTXCj_0+di!`Bp!%#^aBzo&AVLzZ~gA{Xwx@YIYl({chtn*k{(x zaT|PAyAAlhHg+E1>M&i*oNh{1B{ESp0a+HXo1!{27y2^J1k~?B1eXa)PhE`0+J^xy zb8f(ztRwKhu;K&P<*7yg15^2GE}Wt&d^-Gnvd}1(#<;>aZq5M^$OyFudQS`kuyaas zB!;V%Euq_Hty?uEXOUZh$UVoIC~t%d+nCz& zP!guT{WMZ?Dvf4YUtWIvh%9A?H+IG!O1!QT&A9M0hPA(=6Kj8&_z~GyIaF9d{eN4I zMXElkiy(0xAH56^tAU#cAU$yr;CNY2W}L87tORv%u)6u44<_jzLdq4w|H&2Y`aC{g ztiu~b$tgJ2=VYPkD*5LiMC?uRA-r@57KG#g6!I}JEaJ`jS>HroWPP}RfhyM18 ztV6@k&7sli*0!**>t#`{S`8y0a~X4gLgO12}#%Z&z4A zV8(l5n8n!AyhTs}Gfxk@n8N~aqY#6|u3#s*xNuNwnI~Fp)HkI;S8+>6WAlItIWu*3 z`Du1$bMGAaVDW)e`&I6eqnWt{R2K+0%J}E<96#KgZ;YXgZ&j%lDIu8GVI+ks947q= z2(jm}RR22z>%ZGc@Z4+|V(%08XS3c>lo4hAXTJ-}h-y6#?=+4i;G{-9fo%3kgMxwD zn92NJ-^sv<2j}&bLR#N@SV-|9%J|6-{xv4CYuPS-!A{`rC-TKr)Zjj_d){w0sx!Nv zd}0buEVSJ}Fra;H>!5vS9JDB1UZyIzOU24%D%46C-t_RKozxJ+J+nzga@$`uysX@N zqG%hJR`N1kRukUz=1|??^2}yx4zXW$AhTR=C=3S2-{_0uzV}~#m~gFZ8;2C$BeZ;> zi8vHh0amnPQ;7^46EbM^UT^m_9bJ!$=4cO1rKjggfs!2f;mzPVWeAXSd7&PxQh8nU zoUd1FHEeZ|5_CRQD&~;g3pGV?P@|fHtGSp%`Wk;Bj$fTwOsNI_3bL$`&P%mYYz&@N zSdr$NVh1;cSMVKOl-7&hc>7Xe1g_;0$30rrg@t;}`&)kfFn~Ce+Y1%0VTim-N6i-yR5zEjLl21aNXax;≪QYWyH@c$= z8j&`IaE=I+l!`%}i`4>SMVx=}Fo@U$0_f&tf`x!>){qztg{o*(oyoz-&)HqK-xZBK zkA|`BP#BdxNd8pJm4Rq{bF6u9l|^SdeC&ang`Z@2R#|o^cIW|JTV8G1OzYC`rB9l~+Pf(TA3p zPK}b7ZXiMK_W-mzZ#cy9@Ro$@P@_f>cw0B5ZlU7 zV@ENsYbzWud0+N{#r5}$64#J_tUC<06Am?Hp!=aM?W7h3@jb=*hq}3yoc+w|hi&@! zk)VWVlunv9E8#ew7u&J>m!y+>gPD>Ou0Gzq=c^5a!Y?$2&w;{*v@Jh#(&!Hs-;Te! z5Y>hdWCEuY@?Ph9O~0EJlOs-wHnqhsxt||Mt2Oe4LDU_Mgo?qWv#rY70XO&DG<+F*4H6Ig6fiGFIlqo4G;9lY`Zp znq!DHIBVJQmv6i@yNQY+Aa;Dw>AeRP|YQr=#>`Qh)Z`BIZiMjniTbnq<*C(^*_Iy$D%oV2Ig z`1(@TwQ^7sHN z!hvKG!3KgcLS_$Cb0jgWa;6#G{fwIfz@X~=;szxySsO$L#E5w^5w%Fn`YRPIz-4K!^xT0hRN0Il*RYx5=wGg4ynF-!-%$ zF#A)zh58K;>U6O%-}y^|qXHp|qUC%^Bc9SD5u8>8le9-{0Nt3002>zgEY!OSO*pS8 z+;KHPAOjk%LNHU|LXMZ5Ohw8(YRiMGB`SXyoK+iBs(gD?SeJ_BBJRJ#HFhZ##-MAt zQ-A|URK+7BygA;+*S+!-++|jYg2T*;@R6Xy;h6ieij-y`1RfVy9p3L}w3CCd(+<=V znOhoIBqcyQH2h)=3&O6>i`k>w=*E?a9kg~`brJi?8(6V*Ft)0RHoZf?ZMacZUAh0< zvO@Ak=*23h-A>`0ZmmGHhSP1IkQp@~$jMEY1n4*+BoFI5Af>ku)zrv6q?`*-5ko{Zu{YcX zVuRVC0u+kyl7~&)w?H}eu=lFC4xv=yfUhux4hp#CB8$?pt9-wK1BTvK z9~%uJZV3oSOrliW2ncytC=LzJ2*ZMKbR>*4EEK43fU?oUG2*Z=92o^84-Ld91i`RC z+)IL~yCAWfXq>@*^@E34xUh|$kuV`(AdB1L**Ub z-zBqmSE5IkQFG&6$(BrvjijGj`u*M5j1rj|FIW-+n&pdHz+bJF8@hs5k&9QDJ!9F+?r zN)nV6F^8^3=yb>)d?P_n*u|0lZEqXelXvz%8{dJ&wv&}NP>P@qXd0-Enaq8CC(|{p zaN)jND6O#L1Y%i(v;EQvoevCXlUoNZ7Y8j$$yXz#Z!(fri0BkfE1Z2@(+U&AU~qh( zFOK`B6$XTBl~(AGf{D&-eTXiT){1r6Uq&*ZpI?RJhALLlb5a!Gpg<$>W_XROw(7%_Kq5YT`#n0*$jPodyPbd4xN> ze|SrC=`m(>CzSLyEf?fZwdR!Vb}0sOA|0fD%q+7^somWvZ<4z^oo+4lSf##ohbXaG zqt7`(&*rT?&cNGP)U!PcPjX0q?>XgbQ{xKZiZ(7ce^_VL7igs+B#Vc9!bw z8s$JntfBAR`IU#~7&YYKd_5nB12QAUXGY4Zr!m-L3$xrhb;VV$wPmCN=L7Obv)yG(M{lj#Eaklw za=kW?v%5mbvHopZ7|LRXi@FYw05%-;IsiM)Q&Tr8mKcf`Bu$!LfmBb+sTxo zU*(S$U$N)s*o{*R3s?t|f(E1cJh2qVG+J^+-0wd{_g+5)BG%VPdUNP>C; z{t1=JgFop#Q(c4qrLNZ&1L6-1$yc(ltA&upHm!1SL)@p}xUQoP3Z5&xXkWE=LZO&R zS0T1k1oGeqwD2O#GXAklw6`PZwGceg0iH#qulXwJ*~XQTbFFZ0&sW)l5toV zju}VHP?B|6Fbaj=hGBtth~)2}JIk3Q*K(d{6*wJ^|o!M&Uamj9tj&7kD&X&|sGk*Xl2}hc^u16I( z!66br(y z1@?^Sm8 zD)!unk>!xdJ$LuE$~(M2IucZG2?j@X$h*sWwU%lqAY%nQY0*XX+s=Pj7JDSN3f&by zRlxJUKD6Uj>bzq0Ls`sre}ueA?vHS~wLES`L+f4@ilZ|MsManPEGxX_xF+<6+VXDm zBeZ!btl6K$OEGqsY{dW;?SAiYp$ZKPqQUo$@s-=BkXpmW`&szNWNij$VHfcIi5E~R zb$;xM@m{>B8@t|%R{Eg67o5EaTq+A?tzZwUr0qH)!5TIM|0em`q1ZwUrl>KpovaU; zv--nMrxt74ogS$&r7rXIaXTX!nh9+ec`ZA@Zikc4lT)8FmV>>oNQt(>4zjT_g|9{* zSm#j7KS54Upnx&ks8*FNtzv(r;ug6>D2)=gK0v;cz@g%4NLV zw(&~k@siemgft$f+4q=cJ(mAm+w!y9h%UpW3y^pL2|w(O36ZPdWy*d9!K4Uf;Y}TW zcyg4$y@FtoC5qQnw}Nz7@P86pwE-6V3$dn&7W|WT!DVP(@RPFOBbcgPVi_X&5{vdDzn7oIe@Gbr zL6m$PabjG_t^Wh=sT$Q3gm;TW39wC}2n8LwfDjP3TFOGP%La&v#U*id;~++SE8kT| zn~s(?oR&@TS7Ss$s8mBGBiE|+6?V`q_;9(Q{Bi0SdhtEuGv+8+<#4kuOdF(YpFzWf zAFr~6K;orR72V0WWfT^K8xXZtZLF=`Kgj;xSd_q7<^4nXpM9&6f@Fl6ibAxrWy zG?V-uRsOOsx_f{g_hU)_LhpEga&(K;1CRl5c;mN4mHMG@xJy@bkCToS0dbUcXG-EX zy6MkZFNfa-*uD>+C0l{!9kGE*!U4ky}NbeBzvCI>^{@1$DViWSjlfpNr1nLI#l1mDGZ$;`$`ScxPyRF zDG#_FO}06Mwo!FwG55s4nEO$SxyK9&dQv-YB~G(MnEM3B^cBOf$vF~0g@noC42*L! z@(}1apy9MIyrv^BofCyT6%Pt#e|I2s7pJ0RR8!8;5h>4Tnzn%_lx%20vYt4gBTM1KeWqzF*aWB zb$cJg#*f?O7GPP-Ke9yxT3Dnzk?Q?Lm;=l61CXB6%#vRB(FW3D9D!h^s_na#!GWK$ zg|mqKU{e^E;Xo%si8MKqha1pVexl#1IyFH%3nz%_U_1&kDW{+c0x*mdp zTHsx)QSzh*QD5`U;cU`Wwn*}+UE9_`bVfx1KJ(9vZT=CnwuJ$ z^`7qFIQ;*q7@h`DI6fC^+LM(|cG}lz&jG&Yo~}=nvO-vrK{Z*fM0vp{Xq)dybsca9PlvPWAmH%u7q5<456c$Fp2JY6Z zl49D^Dy#m@37lQkzZ!T|f6A`vHwIW$r&*#^{eg27MFww=3E)7WoLrZMb%HCC0`RV_dZ zvNTSve>fxGtiev=F>sn$s_@S`F0`fu(5Qih)*}FfPdB4x>SaY$l z`t#4v+DQwb#6O)UAV>d}q~hc%&PAtN2bTj_Z|8*$NQ-H(CgUS_Q{7U8-H$js)NvlN zD~6pcwTyML$(`}7tJAq%hK+%ift;~FZ|mj=iq^Xd5i-$E-|`r`RwLP`LsicH#@yJ=>N?a`lAGSqaIFZ&-`uC z_OD|d^qjwt**~>pCIB#*-Q8_ki6Pe4BdPfd*wKd;_OFc+_UnLb+qFz9IW;*r`LzM4 z$!V5|nx+mmJof3CXaWxc4qh+5=#3?yF}mRmwWqXqv^R9-yW{Oq^*2IC%RfyM(iQz( z=D#)8DiQJgk|iD)m5Jw`9+R*OBTbSoj5FT#8r~%OuG7peOm_uhAZNR?{aGy|nmwCN z6|mdQJ$_*Baj!t7wr8jmJ8dv?lqN;L>}@^CK2}IXYT`biVaRgnU)*`CLjmRR#fcb&NBE+3OJqGnT*aPR+I>%~vqbFjFd42X+0F5(Kv%I#oK5b@aU19n``9Cu8`Ja^KzEpp&)i{()WMACC=@h$nx? z@}z*sJo)$#Xh}Tn8#Vr9mga9esed%eNk!;Zrkk8@9bA3f5MWO%vzF~NvvfP6?bqQ6 zyLs8ZTMw|`lD#_;dk!BG?K@fHB6a*t&gZg6>Cdf4wcX)ly*5c4PVW@A!V5~zJBw;i zo^?VAE%f`$lE}XwghV)H{9%j*;rtvzQ( zU&U#p79=EixEz-Ct7!$-wd#vb9r(~ zNEQ0)f2TJ;{hX?@ORS)0pwbQC0TyQ5K;obEefM;rO!t-ZNJ zf$tE&bd0(pG}%hoc?wIpk1S_B8tWzzkzV)r4YCCc=Bx)fuDu^h6Hj8spf@S*Wv*Lj zX;E@=lqlKe=Kc$^WWPtq>v(cvNs71SHIkOv!Qd+3-#*HT-P@aREjr_q?k}kv^6Dn3 z9@CNvwy1Z2gckkC&MhJ{;4;ao&C7G)2kMQ>#JqU5xv33Tdx63bJs7#Q280eNx9<4g z3@h2Wx_SWUL(%~#FxAgfT&juzLWWGp^~90TPz#HtY6UV)MWK=9j|?fF9HH!-Z?qDu$3%UAh+Hr zegpmpd8Gcb7L?utDWCcnGK6e%7=CIg! zY_*&ipSSlwfRIAZZ!AsB#zyHW?F$qgch1s;fXJf$fdq(8Hobd$u8vHw&sTPeM@BZq zckmRu(S;pR@RD5kI7r=N7_IkOP^^yAENzCY`%;J+#p^)r-_ZO^^=5G+rZ6DwtEBZK8k;z# z;27~r*7mKpAG%}+RtfGy1K(-&jkD%r3Y?VdRK^el^ltxqe1iM9vajEQ5?9r84p5WbjQQ%dLv-pZ00r3J&_-FEMq|?m9K# z936W8RE+$htZI}|?bI@V+lBvJtb?8_C@lM;Wmy4$S@ttZIsK+ax}Ca!o}A{LdH>h| zH19M^MDxF@@>{rq%YGbLtBNi-*G4jFx%e_`)|~Wk9A|-iOpc&aWXr9=g@uIcT%m|e z3ffkh3)vDb>%`9D)`a0|>X~422{9|rWoERzJHLtJ^m>3gFb!`3Hz?(CWu3XEj`}4P zc7ZA`cSxM|NXyN$UQ3_p^l?Mr#n){@4r}~+jIyJwaZ6<8BIsjJEv@`^tg}RH@;=KZ z0t~asbEC$yg;T~!PSXyMzcK(#JIyRjTN5&VmNrmanxpDDrkE{d^X~t@y?23=>#Xhs z{gNzQ%fe6DV1%#LZL7M~RqAd@*p@BZdfLL0EF{@(kmRPi>ULL^>e19g-EF0_#028b z6f?nTSdvM|;AIl>m|@omgxMjR855FB_P3eDn+G8YWE?VK1KDBu@tX|!!OZ^8`JVUI zt*Tp(mJO7Da8-RD-{XAeobS9poX)5%T~Jw2tgx1nN*322AZ!Ifs=6_-&Y~cA+6YFm zs0~=?^bY|H@S~HcJ|@|$_`@v9{|I~<33<*M|-2Q}Ca7J~^ zIP#VP99YpPoKYESCQ7VTd&OZ8MISk7j;nV-;Ry3-owmQ&=p{eqam zdZf*Yc@2u&fNxYXMASaM10^0MaT3mxR0b_c=3qV@z{P_=!9fjSgl-I(;ifi* z%q{0y0ky)2aRR=zyRlg8BOQPblm(w(JmmS~{hWzesbEhA6&2Fisf?79l?q!azx(b% z=c0)0oPqu|J&Dkmi?F7P7;`!cT-+YP{g_O*Iu=_%+$W-_P1JdBjffc{l;&B0x<{9E z`PW-`sjVMn5p_Zliu(`G911uv^7 z(U;q5_7CN-qR1!KXTPsMYa!DgJLRPyQ)@dFl{|(afjvY|_?E7qTUAY!>8o?N#64D9deWYxKP(T|xWACezOp(pDz z1N-KJP)G8^NYEk8(nkYmBITkwXqh4ut)_n}g z|E1!XJ2^fv3Eo~S&evx05QDw6RARAWYtXV;n*$xM&5BtY0|*1;`km!Mg=TK5ZzyEt zx_@>B4p+XqS5Ptu=;~dz>xb0eFzQ7VooF*ujDq`wFw1&n#g&4$4LHtr1P-dI@xr z^%1jPAJl+vzh2g7k=ARTSuSkN*yVuzd*Ogp1#SwZi?k%ta>Fc$UXi<{;6_NqhR-Yu zE?F62z9f!#Q75;=8f-nf=*{Kt*sR7ccf1-uZC2wG9ju1+S)|n{z`}*9B9wefPIh^` z6dBSA9@XW{Rth8`rb^{91Qg*y%tC+xXToj)he2@7VySX11*1N_0ly{0OkH3?o=yRF zb(woL>+o{N>+myX9e%Teb+A4&>tKYSpvkQxNz>wxp%TS=Mn_M1m0G?!I?7@~f#RG{ z54=`^`X`;s5J9maaoCYLYlIC|;C@gF`8;6}izrT+fC~tAXmCr^6hHmwAveY^D+ytQ zmRlujXwA_sqV=T!qGRr{O3x2>5H7wI;uraESo@ZxI(5HlAwoalPy zN6lQ!avM~tZ%#tmi_6dg$gtXpK=xemR0SUJ)4R^o$y}KSe6pz!mUGo>uDIfPKtTG?S%7ZQmfxzhT|vIhiYT;={Jz2JL;#d6`H%{=`nW-C z1!`tRH&OtpXNTRlm}>Jyv%q$HQ_M0`P(z5$eys6Zgn6Cj2Rk0TD;{@|0VH(Kd~(Li zLwsNgNz{W!1k;6z8;?KaDA;%OjTk%}JSPDD)(#%po|3RF4rlJp+(S&x5ob-m z?WNP;E?aV;6S)0BWC5Zw?Y?c(6!^bu+d~iT#2qu*<}c`>o$r2VzZx8x-@{gL;18P1 z7B8XRIYW5aQHV|~23CJz%AY<5epOAnhhyhvO8InUsepejiJQwKu5*qT=8~8q?Elr| z+@)P;%PXd>rqu#n=3X@%^KIc+EG?9i^Q$ZM^*C8;1{>N5xz@S2UyVIy^lhIac*Z~$ z_Hz4SdQ^juc!j%dwCe7OW>Ns@*TSV)NBtWGJT_~Z{(ce548c%7>_`;S9L5k8tk zfEH?-a@+gh2SqJEKJATMJ&-bRsFFHpl)1hAMENoGX z4GLxj#%^S+;Ybl2Lqtgy&?oI;ZWI>7O`s-J)xqm2njoslm!PpyVjyh>7#aScR9S%7 z2LRTIIP8$j?~XaVPMki$9~9AoDlVg>Mf#5Bh?2*!L8x>qkqqJ>b}dQw_`oVmV4j-^ zSp9VTd5xWK6m_RgymI_Qr2Tw0kI|daY2B*cSQq1GfW&rT6u*45jxj?qc=?wgfEak? z#{(Znl5Jl7U5)dqbCzxH>lU{y5lA(tB$c2D6!QGdttg{K(|$m0)Fh;UW!*6V`#<^# zu&c)G1mED){Nr0sIWES7s{S-1`-6Ut?B)>W(E^msSIH7oZ!_!UV?Hg~^@HGA$)(f+ z>AwvS{;r=uSTDF?3RHJS3)QtT*{T%x`UceSZkF;cm`oZjb9mJ9TY!hOdV|wE3{zLnZ8@t{4Sf*N?uk78qGg)5DUQCqAb32nWm7Np2 zCnghbz5l*(;pWJ_fRb%mHQ(%SL-C*BZ3Cp!cljlV=0ZU3W9X=UXPZHhp)vPCya~PR zoBu`23Qx#x+J$V8``r5MnEuQXtA8J@?-yr%A&^3sRV1cIGM^s;R6f*U$hL2hlrhsvvJY1Z7xo*@X$wkq<)?ClEB^ zxAsap$eKhlGYt5UsM(Gr&+9~)<5}b}l7EbNLy=E)Capr)_URBM2RJLv21PB!Ql4FI zICl*`Ps-2)i`@ya)7L(dZ2-i{0`q52u1XmnEQweQFKNQ%9VA8v;Es5r_i0_78qpUY3baLL_E^*F?$YGAnvAM=)|%K581iO4pwO4 zB|48#Tr{ZPZSKE#W^%fE9+ny8y3NBI55G}<3AHPTUpd-23pKvFW&|8U&f8N4GrWe%4iq7m{2j@?Ij$>4ZKdOwKNXGY@PRhyA$5~X4a zvzbF|iET&1UJFxMxE%@cC_D~k{p5$cziaHU2;nyZ8%9vu3VHsP*0|MG;}*YPFL1L( zGk#_J?np?iHq3}`8^ZqM0DzkBs3c5o6-NH=nypd+khjWf2oYM$tU2@*fc$fh;3Qfq4)Zqv1J z3h$P#=-rpq72Tp&4acKx;n>y{y^i21T+wE^{V+YDUC|^Wk~g1IW+$|36}KA_Y9~hY zW8ryi@TvB@Sq+tU^bVDq9%>7f;~FZKgMNBa&$`h?KRwyrN#EyIHB7R-!(^M+7A8mh zj(WtoWM*bB6_!b}xMYiHu5iSw?CXhVs-Uq34Cz;M8VXCjLt(PsB@`+I1=oOqv4+8* zpfK496gqECLt&+NC>;3Jwov#Gk=k-q+{U7em%3^IAwDzu0y%!pa;H5I1w>O;M$gJ?*3r>je$x&vZYuRB&ki z9rON%fH;3}+a4)V>}e`d^w5Hy-|w`YUt8(Gqj?6U1LN7Kf8LH3nx0@Qk&N$|#uE z&{$fLdaCpjzhyv|7u*^_a0)^5N67OrpYf>;y0qxZ53LsU zLarO~W5HT2Vz_~)WyO%K{T!Mv=-vQc4GT+iQzNlO25`&rb-VgGj}~;I=Hg#_bJU3> z6?4Oo^5K3C@l9NXH=G@rF3lR8$c&z-&9wI@Sggb7bB1G2UAXd!D~?f}IvAZL{|lb# zp^HpYTw&6ZpOM<_OI>zr8U6p|=NTdk$JepMD`bLCl3#jMEfTT-TNV%Qw>2;~tz z;W30dqK<6^+Q<{Zh?O+8|~Me zsJ7PK2q;0~{bj6n>lCQAYR7X=X!+Mb%>cW8bv8X|BHAd2GTP+oAm`RYJ%Rb9fX4pxtdu?V0?Tsst#!@8(m3SfB5NLO2)og^LRi%Mo_r@p~Y@U*|?=#_Q3kwU0G}2vFaM(<#bb+G* zk!3&`36-68k?};PTFA=^739)B+?SDC4C4O|0M{Q%wWEV6s>SZ;8?(xP9vDb-4lY6$ zA5fu-T>#V$Yg{$BqF}+BfUjVjdJ>(tSD28Pg7ODlOe{sf>#Y|i}sv_ST$6#V4hM^Av*E9BYYjJ|>hy8lIpA+0* zIDr_SBKO7zX4P*EG-ad5*^(*b^4)jdmp$EcI|9NGCq^;M*5X7{ z1M_<8Ss@59|K!hxIQThGn+xa}Q<uW!1bzaSv@SQzHMxfM9n;9I|BMtgRxM|(9ENDYdAo4#{v)Ax;a$6$oT z6BRJkzHc#uYY1`FD8(OUn9FN3OoN>0P@*h&y!gZPUSE^m&EDHAV3#paI7C9(%FWDH zU?sDdih^M1GchG_sgGesUGlk41R08%M_r2wdUK5PmK>)oIXNqZ2;r?AUk|6(B$Qk9bWn^ zwK6$1t0=OA_1S0S$fL>*Qqt^Mc8J!Frl~J7}vJ=Vke+wO<; zsfJ;-lpQR1T?W~qX>W=HhR6=hzcI}IA0yLA3)$h<1?2u~80NMX3*@r4e#wk>>#PSYuUkUHBn@T94Fvt0q3#^4tKQZ(~=sp0bJ{Y z)bQ>=lg;@{f31iwe@#mbYF1IC2J5p~IdY%W@L)@eLm`^T&|4##j^5ur9sL$TXG<=0 zip>OqsPe*xx*yi(8-``%g?EH%{|xbd3k$KRT`5T+`Uhs?{X}HKXtDABjsV!l!%(%g z1kj63xiDWUSLt|g#I}~XA*%D1sm;zJ znu*-HradyT1{+McPvE8U0_t->gu?4Oo=X+ZkW~k81UbxGRg{pcm9sdfM{n_pr*jy*5CU+S}p@0HAThS4PR)+-z@mVG^Ml#>tFl9dbj?p z`I#G2_g#I(6{DkPKnE2Ji(tPY1BnW^f)cxR)$-Q1GaF0y?J3kIB4l(^UbIDUC!L2w8GAcR=?;x-p>{13-xVyGD}KR;W`Bk)57 zOcJX|CE?FU$MPh2KH{C@I2F=+$qJw2MJLFkEmU&AsxX&Pr(t;IGVY{GbohI%hzZk~ zQq=Zi6cLI^qj~PieuLOw4u=NFCt@m8L;^tq+lGA#R@na176kV1mr8JC>2U`04;Gr@Hf=Wv`ex=7VRvHAjqCA!R(C>#f`UL*K^lZ_8HCEBzoimTiJ&8UqlFA* z$EO2~&|=58hEkbgyiJ}O7V^$O%dL3zU<@3$!b1Zmr<&#v$FXT6Um}azcVa77(U(-~k;aARRC0P0dkU!q z)zUnjM=}ao1!)|p#|Om$_aV5H(IA9#agq)Jnf9?lvWSn6x`A&hp&P`d02{Y9hFPJSv4V<@-oxr^TMs!*vq=2%_N$0r?pJ3F+QHSbP$1Ns zQgJ)QT(YHrBf%+x!zJ0ZOj-nUL*l0LAms`YyRu*d{3;3U#DO4Q!fGQUxb;ZPxe;!}bnW^iD%IX@A_C8mKCJ^YQv@=I|Xbq3%NND4R&nbZY>IJXW^iG(@{ z5eloG*GCtGbLvk8$jai>hx&mMJ1$fy@Xcb!vMgf8c5K?H zZ4;5TOJD0cGQr(tnmLo=8b9W)i1Y9VWB?#FYo5G!i$ z?qjoLU+ajr$>#p*{o3aKRlQ`!o<3CvjUN^Jgty451DUTl>520aq}>B2G{JO&9ikF8 zd(?jRC7Os+B}x9rcE-wqt_@Aw2_+VL3^v+CT-5Rg7tfIiW`fiZ>}k+5RK~M0vBSJb zCP-hEX%hUg*`O4626jQp{-Ext6J?mE!rBLDFRYeJX&h;a$4ytwxtOJMX2{SpQ>G)& zMn~ygYHD;8rXrD{Q-MKg0Ve$UJd9OrVj3NN+x{tjLu8QQghD_N9)-p1Og2wOC*X=y z4CpkOWFEOYRDc_Mz0a6u_M-a!o>#@%6TTh58$=8H>KKGPf4Ijr?W%Vx`}9EXIMM46 z4)p`4dL6_KL7Z&W;5(JytiC}Zu3|mrhG8dnZSTZ|M=If03_DRc!;YIfT2M!{0bdOg zQe)-cW|+JBHq4D|?oFMU1e-P6eKWt= z*9>|Z=;Xcv-A@sQa43h}`nvVj)IYuEiYtcPC%GWD+QdooN+t88UBP^$_ZPNXJ#p-` z!>(N6Qa>0yTU0#(WpzGu?5=HZ3FE;6rfDOyd0Ji|UYk7FIMBPHgS=ofuDpRbDU z)R(G58kK_sZfB8xT&ZGIF{(p3ve$9(U)h6euIR=l65~6IBcsLnM5R`U#g^hz?jxij zB}Ya_Cr2Eoh%zA)dTkKnzScztOE zvI6$<*tRF2ZQ});oB?a(Hg(%>Y{VJTelb81t}f>>bP$^q(|4(9=WurJsbs$9!P`YY zA%v9572hzt>|E?I)&A@@yyX>Pk;Eq?4=KmnaqeL6CEeaK#w6w@TC-qSj7<^{F>O5J z#h6bNl8Z6@?TB;2oVHHyjAJ#1;0v3DyUQotI7)4vR``h5>1Rx?KL$ZgL~4$N5R78Z zi&?Ep)KFj3)EkH&nb>Q6!w^kD?cFq9TVpQ!GAa4Oasm& z8RUR7DdMNyZNQm`vAq_YNl0vn>@Go0Y#BVc0pUV(HQ0n#bKilurH*|58vUklPMj0= zF4!x!@%>31l_54v)!d^w-p)Y)!<8W}CIR zv2}}^@wo1!P~T-bi-T+zS0Df3y5lW+scAlbW%com_3Qa5jQueDSCgyDtFhe8_-ZA0 z3$D-C!pCFeQzO>acPsNsJX?&DY{ym-81H@Nl{g>N!p~_3wh~hH_q5mR?}K$f)Il%G zIMtArYK%rvEMnuQafBJ{7$|Z#1CtLKM|@rhEh=*j9aJMxYF_6b8r|TZ2UL>QR_;fD z3s*g_Eny$gxDciYgs2NX-C%OvKsSh@8w?GU0~wXKrpm#tap(1|8_?z~ls3{r#J9^M_qI;%0&GhPwgJ^oGX1Fm_l%~$47lfIF|7=^XKPt)z&&3d+*9jVpM>n% zqmFgcCt1h(6)gq(h35`+EN)HcSijh;#g28X_xN=zrvn|!8jq3lEgcJmQFj@Yi3_9t zXT%!#3ZoibK`21ED*OS@!_)Pk-;WT>>{mDHyN(u9c?5*P!F;+}!Ul`uH&gDTP|)}k z=R@|v%KMe-C36zH`qn5zUzBiHZY2MD+S)QDV39Jk?N78a^rw2!Ojtnu-?8@@p|Gsh zAx8ok8JL`qBM=v>rl*7-rP@(oRk?)piiYE$A+f=0!l4E1vPYYA`e}9+!CLHB$C=i( z0<=TQ$bdJ>K?x@1%SUEh_M&i?Gq-}TN@Q#xe2fmq76+Hw&U)j@dZgExSbngt2H+g@ zgM~bQBoMHW=oAiJb#FN;Cpv~>s~$UaFkcbt)WJM#5K}>u9n8Kutm=uvcSGIH9D`P~ z2H(3;{ui!|uKWwML`Wj^Gq~aqQpo`}uOmkx=~19csSU8-1|F7+oUp1!YdaH{#-_$t zJM()K;mxr;iQXMa36CgnUjyS zbw7@gKebVC9;0-?7L+>!85cI1vkl)$t%4>^E5 z#(Tz{>I5}3Mokjq zQ3*QJ6{Pd0Yw(g*{j8@tzB5+cQH49bwX>a(kOyqs;A#eFamYQ0)KUsQl!b07crmHM z#*q-fv5z9Pha&jcoPvt7s0d#ZDg6iz;VJWUA7M*$N)9*D9a zNEHNz)7y(#yfbyy{%EiL$ByIYxkGw)iQZj`_O6oNXbqX@j*pCn5M37Tcl=-Qb=?st z4xcqH5CeI%up?{6&tKwY@ozj*>_8}hc&k;P>>sW{oY!XiD)=ai=2)Evj=G8H5Yt7jnf$(R9O zw}BpS>&S#ew#@^>x2Q5>k7`Wl#@6%F`hl$nQodi|8VH)-OM)hah73$SWG(uhla{1W zwnf-V8=D4M0|q7tf&%_)5`06COb8526h1p*ivS$aCLI$3Gm#F$p}W$^0i&W0?9n!p zAvY`-hrZIQZG9CU4h6%AKjNv0nPjkg4a_9p<4I40-Ag4n40bQ1t^ME{(0_n-FKuCX z2D)317KT%gLlfMRJFB%oe_%|9fll0-urN$DYq4Vs!;ktc44XQzFj(U;dah+*K%moZ zf1p!+9Mu6kOV#j&MH$UHdRWxt0nQ0Q-YT4e*pX3uiK@6#IV%sL2W+4?;Vo2Ad#fTV zajE=}-H|^Z94cdy~I{n)B`?Uii4TBV3(>cf^r~D zvu@O^dTPT=?js#d?j-CY>9EOlV;6Ce)})f-154NM*cJ{+QTD6XEXjS(=;G+2(<-YS z9@5dq)OO0L?^kBK)mEg$c3bb4QFo?Hbh?D>^&)Bsq`i46JcoLoAWggc8+GR(zn(JY zVe+w4@{tTrPsIb_hUU@$#n>)NYqyfi@yZ?;AYQY)bsDDdLi2!;waznoL}#Q`STpM= z>496)8SVo8IZ9HiVce@qcNowLu*Y8kVn8d!^ng|b(Ft17GT*OYQ|nRYEB_SBe7~r- zZ~v~}q0Gmv37PNbo3+@n%=do3%y)eUGM_abBcNC^-z373;;47;Q7LGaw&IYsP#2A* zE#Xaqg$dGDyb>4XNsWkDr?076ek)l`d~Ia42A`6Vp`!Sd9yyI-q$pSF#ws{wfwm$( z2SuH3K-t72ROA!KF%yP798Itns89*(%jRaU?3b`-H%6`tTbM(7 z#M;K|4!MU!p&_dpF!>R}kV|H@tU)m7kozcNk}K@Ns1+}O4_F6E%c>_!FCiaz0xy}o9Ysod;#Bf0d9jpD>oZ|cOW|UcN>0(yBF}o|lE^4_ zF^CB-eqRKa%%fb3Nw{^$^8fwqq7k#*A&OE_B2>ky8b#Bm|3o%4K&ZkWDx;9xcg8sX znW_j>-d(Q7xT%p~R4IgdnM@t-E|F`hrE^9ZT}^4gZc%S)O;SDv?AD8x9x-6I19oc% z@M|6je%NtP{_B<9l5G8<+By`Q#(&GJFv@=&VsekF-V1-rs^0&j1@4|yJ#I~?djGju ziyfND8E4Hht~5(%$qR9*SjL6cyda}3(}0$*tkxs4rOuVqq;4w^d1(~JC|dHt zQ^!wZtn(}^qD^tUh)<>!o|G0cx>A(;(w14bdpcS-(v@G@qt|UnevD&aL#l3t-0~IT zdZ^?0Oa;b)vfL(Nw-f1Y$nCt)SGWz>R(pk3XZsy`Dg`{yq1RN0Hj1_p=E|9pga2b8 z#}oJ8KTxzw1(EOBG!L13y#_nbU?GI z&Yj8fV)kO9RG!;8pPt=0v3p`N@z(qA8&BXb;iibe4715@z$(x+D;HFSIObk}iA|TJ zyM}sFdcXL?DDK6;r+yr>qa$-4L!C6qg%EYY;cv*6D@J<{0FRMk)OT|SykK#@Zy6u7_8){Y zIa*rONCP>?ALo7{TZ$t)2-%kp0Mye7%~lCe>FxnPjxV z1jHf%MBVO;sqhGEEF&Uu)r>QmjfA9yCI*thW;!sCL_KwHU~0eIruM;RI@nCfY;W&c z*u(~#sj6ih^l!XIo2j-Dtw6Kw(MI&@FR_j2{aQ!%>+c=rIdE&jM)clhEp}`p`hed? zbVCO=B5ORx3Sf)}^&4O$jZ1?H*@z03!H1q{V~czih`umJhA7sXtvJ6473ks6B59o* zXE?$W>Pm&;fGf(r(&jB=c=?nBu(H|s7tS`ksA~P+{6lR@c}cHM=F7^?BhECAWxnFS zVZ{nPad^=yr?SYs#VPbWiv`cS;O@GY^{=vrD!i!sOlg1>=`)ORSdcwW%HI`V?NQ;g z^}e@jp!6R)f)d38eak@Uw>6Yn1NX1}pM6e$)&iC{K3ytbAW*?ZnWNaM355Y6Pt1J& zs^j_mS2Lf#)B|fjAL}zSA457pyIXyOhic5#rFopm_!!yw*24`rn_XPZtk1m-f9D>@ zg%I&eepp$Ec%8z#`iTecipSki_v};Iv{&LX&<61Yxkwzk*BC%ib-7wg%y>Iz9^bwH zaOzAgUP6uNc>Ezp0k+@GYCilOz_leGUJr^#Y}{&QSe{jxapXaj|3Is=C1=2p+q%J+ zUIj_Ez@nc!0#?Fj$jbvFh?Nk)PG!?`6r6{EJq8$4s5*9>@(%D0j-)B$5KWv}efDX* z<>lRr2>E+D3nqo*osmdyfNQMFY)IKNet%;8{)xBVKLQID&L_#^WRHrQ$9Sn9~2P4&E5(pgGM5%XTxXlLnS; z*8aAkN;MHapyJMU05z;YeWx~n=EO;KN36~2bhDgXuv}(v-yqgzeRf@h&vGX>g5KYb z3z_lX40QIlVQ}4yxV{zS!J8wyYp^-Sv1uUWQf`h(#c#}u;k#%<9--OvqU_OkpRzVF zeO{{G&P1xC-m4?NZIf-U8_)PC8iR3Y&oG=}~p>15VRR%>7%3HJ<&DYXc<4hupoe`{IBKy}MmIQ}M<2N3fJq_h zsGNNkNAr}52_#Z+_c$DvWerNwLMtpYAZcwOi~cDd8AC_KG&N-Y7RFyGcTf#|M~;#v z%H=f*9@yJBb3*kYJaZSb{H;4-F{}PS)Z!&i2Nphfd> z+V#|0;X|(j8}AsO4B-dY*Kj_?*zB6R#H+7qP5AZSear*xfgot5YKYb+l4T4hy+z@U zh2Pll`DBSyDPJoT-I$!s2%RiZs+Gw{ot-Q0RFjmU{KMyFO8InUsepejO)U>6oHInZ z=(kdv!66S2Tzwr=UL&30yu`Odh5@=2y#*y{v=qA8KXoNK z1aF%~{y?C)<^lt4j9bwc*4Z)ZtcKHgQk8n5w)!CZ2zh?8@9ShH8tkq;He0g{-NwF* zZA|lhKHf^4@wVVOisvWum6Dr6>L)ZzLb)nV6M`1%L(rw=VdC~o2E*gS3HKnsPky7| zdrpk#gegK(0)59h!4RpjZMwR?4P{DaL2gMXzO>DO+)%3t`;#|cb?|S2k2P`W+;-Y$ zE1B$U6-AsXwG<`EBn6G=FX$ByN3mlYzUV<<9(NLM46|1|^BA^NqU&0oLSDszdDywE z@5K%&21(OEx{iYiJYp-)R>!!+0hj9`LdC9|rvvgplmdQ$M`FY|m?@Q@$`2R!O^lBZ zi^0#A&Vj%!7j;Qg$ZE<$SscB@1lPj-S@nF|SngCjT-tciRrXNunoKaCLxaklz-&sXeHR_>~4&Tim)Iw=GKZ(|1UzrP8Ol|2L20UePc z0`0`6+_)=-Gve;VFLX?C078tj#XfYQXXwUC$YiBdA&6lmBkvf2DFYdWa+pS(B{kQF z@l!}c5Y+a$4@=ZygUSG=&=30oQ$Z!LZ@&Z7*wRA2A)hP)dFJ3g_|8?}qIYNRAtmW+ z^qU-tqi?&(-9nb~rzBkm4VN8yD!Db`sXX1R#g09dKjHUO-q?Ys(i)H93C7M>-+l;( zR8!g=uoXBiFqaA9Vls~kM`X{((VGH|Ny;9I@PW0HJR4w_{(bMtCB=F1%do!Z0v|=% z@3uXPbrg_SoeDAOVXI~JM2WoZnX1)n7yzfL;FbHKm67rY{R`eHINnlZUoOs#`79#@ zNqgsiX(5H%xR{2abKY}`u;Z3_uT<7qdf9J4gRvBxvATp|&VxYT72t8~NS`C*G`krW55_z^) zY7a4fnKA!bKgaA-n$&C*5rV5+L-=fU9(={WKHKc|2}gXjMp0fhbN;=4&Uuenrw(YZ z1rUPf3Iild`HXi&yDfV2gZzz6iCjUt#Ty2|U+E`+PxP0)w17n<$ZztoCo^ZWOhR>U zzvoz#Z}uGXS^aJm5)((ERP9ZSODB~qoLaGBj`RoQRMQ`5ua z5w=GXn6uay!23_9vKO+|_%Sb8E-E}}^Ok|z8^$`&dNY)qHCfamz$W=IIOf}x<$dZb zq`uKndr+gJOr#Z$LKfhEloNnkFq6gh1m|LD!L7k|Tf*+gI}dG}4jE*}q?eN;6R17` zS%Wd>0vTr!+ejh-zM;Yebel{uEKQNZRH{{+B+4Sd&#{D6fR`2CP`E=q7sHbq{%P9h z${3&;6txl@y<_%gSc8Kq*iJ|z>1?VB*9h5btF^RnpRiiS@*PaZjutgAGo#KymLY3# zj35L1u5j+qJn)j=rQ-;f&Ir$dtklw1O0o>UT~02{BvTjg2gV4MQJcTT;Q2n8`=1P%EJ7u1hDe(@zY6y-H*fhbcMX?uU^YCTYh46Pvcl#@zrfGLh4m zMn{ili^r3T8pyoferPs0q)~SuRmE~+ap5&Cy6^cKrda5 zz}gqLgwtEAX3bJ<_7kOAtl(|4Tu1u0T-W>PKiY8pNF>eU{+ni~Q~erhINDoF&;(7a zHxBg9&RRj}#(Mi9Jkl#W>~#Q#`oT_n9mFHOvg2L{aBC<#Z`G=yRzIen1l#nmOQwR8u`CjXpc2rBmSDt>RS~}v3wrS z;*bR)pL6jFOI`54NODx*#X!D-k-MX)2{#?5s6wBiOrd|+BLk6dbOLWy2o8KmD+GTp zKlUjE{}#qO2yPGuPpF7RlDJvWmli4#T^tTy2!vfo1C@Ib@fMIzN!m@iUM-bJoDxiI zd9OH!Ql>EaL5$|$Ll#rr1=RUOWONpR6(T^NKH~rn(FqhZ5>QjwGDSbf&~jy-BQB)u zok}9M0BJO9DX9d1A24!;(h>%&6u zue}c7fgb3+*S@c_`GukEAj!1fYAnh&TaEdweiJ25qIka*31BH#{4}8cG&o+2;u1xp zy6XrZxm6_NG)vu*Lu%8Z&Mb@Yrh&}=))CJm2h6F@YJKZp%a48f)=_N2$CJe+CCEef zp*@h*1y;J!7(e+@z^i1=m{2OPDI?mvT)GIwQRHY*%o(dyL6-=2E~p^*-yEXg`5Y|T zty=k2pt;E`S12sW8FYe%Ik64<+LD_Q`!NXy3uP2oa*+{?t!M!$WzO*=8i&$CLO_0r zwruLVc(Ihfh<#S2ve1lZ5bSp$Ce${Y1!EA|2)rf3&oR~@SM232i$Zcv_8X)YrTu0A z2kiZ`6aU5U1lno8f1WY8qz1wJ=a%pVjyC3?#a^0qg0~F~{8`_|eNa=Q+cm)X8vFymQd#+4!WD=6-f_rWj zPS#82S=>EG#D%C$@6zJ(bZ+_5Ipn7mk}@dRs8mI1o>nSml6$d;2iA`fBsFwI2M-Df32M_WQNcoEAkr$rkLeA?n zB^6hNi9jRi8a$)H~N;E|LB z&opk3wN7)^^^1 z3kpJ1Q~U&h8@LIkS56|B1Tq=y&7zC}bChI(ABiu8*hyj0Fol%xCGi%|0n=dV4+ffI zwFbivomZmUR~;a{Rx~f;Z$$Y4s!t?_#ym-iifBCPo?NPC&@`Qr#x)fdI7d@tYNBgO zv}EfF$)`e=C%LcsT7ba9@+SlI6U6e6=Rf+o0HDFk;1Jv6Wz8^OgO}YT*?bFo^|o!> zoT1~K6ReOD1ujBz??tc{4140hnkgii@CPdE1XWf^xroWzlw%Y6T;3itsW|)^CF6(> zI8VlWs`|kTvT3|v_f1(eXRse*(5Mr_p;;n*m{0W~y5h!&9EeOD{>w|F?l2`n?{h1q zf+zk+;QN>wpPmQW7w|Ep#1P_y1vVstfY7LFU6$k2jvEt zIlDto!o|kAnlN@McEJ-^nx>ZUt#2kYaI{2uK=QHtvBM87EV$f^rUvW(1G@f#`#>Rh zH~42ug`j=%(sL6K1-Imik`wosyu_`E6q*aoTI@J^>3{YoFWuBZ@{%W!YE&hV6$6L@kOz2&EhG_%jQw+oe9M-tMo0^&1EGvGMi&y9Pw+@kWe9P! zJ2duDR$)YH4cOUef^-!}6hTNK-kI4FR}q>mrO5wYgt~;PN5v#YX9}jFS(uwBv{pLw z+2G{LGp&s3aj*|_v-av2+U(UK&tp*`)1sT)zp$##)i?V=sKoN%ZL>YU7#MYH_2}nB zNw62Dw0fjwCDbEcu@-|3$}j3`ktq2172L|nzOT5MuIVsprhqlqPzo8Ly*X8?RCne} z75FA)d^>9KVp}vw33I7hj1Imq1An=bb>fn7A2y3*2JW}$tT+c_GM8+8%$2_;#@s|A z;p)FH&BrH}N3b0WJI}=OrQ$N@p@{IS8hK`NETL^eucHy^P2>*%`j(qWd#lom|C1&dvlbSc{pcx=%-ez= z3Qfx+;fgtCg&b-Sh?r?gFSEA@AxM;mq-;aw$%V%nseQ=)qrf1QJxGYCW0^z%60#?C zjJ*`wNbxDMw-`04cFGbxf=?Tkd9-iiw3x(S1yDQ4Bto7))Y~k(>fCa+-U=UE2$T%9 z28Y~U>3QkrTVlx6Cg?IxvkES=IK&3Wsx$d>;J(M87+;1J2@Z2gjNApCG>W>SoB%Xs zWn*LkTwxbS6}YkErw@xX6R6`{R*W|Fa(2-jo*H&It%NsKSR*-1jx2_#3Q&c*5A>mo zBm|dHZ6(~LVr8&5J}s+`Rb@sj22##mg!T_llQ@`E7(hwBFhdu6J=Gc@)x!D&Zuw@e$7-?s1h{^zMUF?v-l*02#46WqsxL6EMEO3o~1b5)bzUC9UzBUc##YndjCh z?y&GPh|(?E_Cd66&`iSB+%R~zub<$>9uK3!LJJ}@9h-Sa6`U+=H+!C6{@CFNnc^HL zq`}sA5WXVyTZswF5Ix3?*Qld!#Q34UJqft=xQD;|f3@V$^,X%ioY!-l;qhMN(b zDff<93URFs>SYT(yujM}&?C2ShatF}^;IG8RU}UI%D41fRIkj5){B8#&8x?X$M^;n z4QGG2pCjC$L1}O^IL7w7tu_B2;v_=Q=44gCu+GLR&oOArlD?ko;711rst_@PPbEv>(^ zaQo*2i0Q}etI>fH0rX>QnI(rhP5A}{By!J?KbzlJMXSlPD zPjh1o1yoT>WSkT{*4Ly~hi6iu3*2p{)@kH5SbMp;(`fnIy`$w-IlW82O-%%|><_{s=M*(1WKEtx*-R21~QP@=qRxWFj=!jg13ELUw6$a8j{KZziT@B!-5D{91ce%UwrIY6I(IyktX>>l#`4S|L4?^-zHU1%)L*5c+cR5ZLRIJx$` z%j(Ebou79dc8W?fi_WXa;zZl9U~&_NHT9`A&3U!~ygdq-+diWa-@w4)Je!|}j?a4? zQ@`4qg15~wOm?#5Y%`qva>x+;%kU0WeTsJs4*hC037XAA0=}8;kai%kMrLW&um`tCSlyg*B>)&nw_S&PW;Yq0-N?_huK#gKVeV87PZZGqzYZv(t~$H05xKZJDZ2i|TU zBLbo?8SleHIN}=~)7{4!tUuj5SYP}`$P6v8o;)hjlyRIB7fxbxD;t;i&?rlkgueil za@5|4s8f)E`g204NIu&`qO+DQG31v5>liL^>8j>R;;x*bE`P(yH#?WH4Y{zfwJQ$yBA4Q;pO<}b8y+8GkMj6G_)`-|S` z?&XIxE*V%_baywTEOM!pJ&}q++Xtcvp|rU;Nw^%+k1N&Gxg_-RNch$KIFu)p`ijtYyB{uc8YnEv;ngCQr>WIujjeQ@o7d z{MC>F`Ij+V)XKG&*K$@?MQ@GhCAb|E>`|Y&>d!-FY%RfK0a;C%#61y-%k?$r6wJOH>D$etvGI${3ebWp|xy}1z_x7wyQC6 zwNm`sEY8WmhmjWN7PC02V_qE8eafO9P2_Ri9SN>#Y70-fJVn@ zYc&i``}t~}!kjE6Scu9T(-gR$&8NLfKxWn?_>ZaOurHrE)8&Uoe0qHiN~{1JrOyIu zdgS}tMl|y>D`N5e{lYMUtP(1W9u@@BxK?0AGVTQ|-uQa%Y}1rQi$5M92S1OvIb@X; z<%K>oDEmU7;Yd7W@}!258W{7C>;3aIy{NaM6Nc4=Sq490oORA2q{wF=lC(jN&Dc3q zJpnyTEkAQc{~USd5^Q$*PdxE0mLeb{G_GkM8f|~6ceFhK54$|wz`)Bw+kYw~MxXp4 z;XoV%?lTJ#OUy!=?v?Uz0R>+Ui$n_|K9ZPEP(QmMLM;J$ymKLEAIjF4LHWN2Fw&3e z2kmioIRQl)eQC=e`OATUMZuI_r7!dqN_J9Z5CrWdN(=a*M6-AVK`{;IQ-GJN<4}^QUSy89uqa3}q9B7IXSFyg zQm6*D07(KU9W3eFB8H=kz}hFi5rD2A4aRkm{)j_WYv1{YX2JhkU`Ub3;z>gmYD!EN z+q9$FCw%*0XS!COD!OTp5mvtjBmCRWT<6=4y91##;tZ6d5w{>CYF)Pe1l%A+-7wj+ z6gBgC&DIPYt?lRx$Gvj+qsYQRa^k}%)lLN=mnfMZM22!@D4#}oG8C6auRZL_IDver zbiqXi4`m0c)EHUh3%qe{8aHU=0zR1?nX46AwZv#PpOnAe5*$-C#7Cc@(lC_ai(_0% z3sm)!114mEMjWX^Fyc`TB|5-9G8V=9J2uMWmAOe+qGS<;%P>Co zb01l{NO{+%Q91)@R~0Pc=~U)(pIP#XqlnOcjLJ5UdaVi(*)=`^1`ZF$AXJ7C^-ohY zzA?(5$}c5QniJ(4M4qRNFPD`dDmZ&qhF??RP)w2YKxJY#hDQ0A31)Dln*ZAOBKR%F zp9tWjKe1|87we>We^=icVtH#|U@eIyCsVYlhF;XfnEfB?C+@U>9uUG8y{!hJC?3xr z?iKS4h=Q%R-=-}181~vtlKCf)5{K-^QkjeT&Levq<$C8Gc?*WD`Fsfl7-tgmOZ1B# z&*!U-!->z83tSjFLA7#{Wtp-qKRw3xPI+@O(Z_Kd2%Z7){;y^dt!kwb_)2zed_zCA3uis^oe{iHf06?>=(9IJt65d(1T$57_&OT(y19X z{n}+3MbAx|!4tv&2lNcc&7nSP46Wi1?8)RXj4gs(#^61i@$&n&9l|C`4dFR40(z*Y zsb2yp1Mm2!tY1_x4P8exK}dpgBq={jq6ed+0Z=Hev_f{6B_?RPp*YMBL-9B^527PB zMTBA@tu?Y<5SP%?hqps6)s|4_Gm4~1Q8IXnZP-g9Egki#kikb%k##^~1;r&wv$G@= zGS@`LMi~d2#;}N{oJZIT;EU>oFscP~rewpw2>hkG8F9>laE~OOffvg@%x8)o07s3s z$s(1<(G7q+_3viX>Bh)KL5YI;)e$1=W)!R6!b^mR`fsLLH4=r3Hhnkgi5lrM&7(A< z4-QR;6^vzv9)1}0t6AumnlvP-K9r7u>iY9BcUwHZO%p6A+l>_hV;yNj?7kM%00Sh? z6d>D~D`(S^2jx9P4$&WiBS9gG2{&5i#;JkKGLTvL1h*)ALMXqNLf}AVp~L?MGD{ma zCMBRI3{1$LXks9s zvj_#ZAv-o%lY@m#%Go4N2612j7d`}f$9HZ#*P9~e5kFDIk5Vv}sFPHNmxkk{9Jw)) zB$BW?dpJ>9o~Z=|h63BI!{+bnc951aayeiz4H_duoj6kBxdb06k@)H>&G$@=O8 zfdRKR@yc={FYMLmrzXTE-k~VqHnvmst+LFvx1CxYhvC`O=irMuNnA)d9l(q@5FS=b z7=chg4ss`aPct}d1`Y`*q$mMPX9=)4$_MrE7s-k^#RuM;T0V_ZGWy&eUM%=y3W%^E zZwXQ z`7!QtkT@#Ga+M_*5BZ2U>d8(#GU|Q8g|%CnViZ`+5Tr2%3iKFzXoxaJR}M;u(Lv4G zXGQ2aB!mK-JIDaG3@AF?V_a6AFoy{VS;7H%D=#}TLWIK*(^hq(%=JJ@41p+AS*gI)b|+kwS}J ze7rbX?UFwVAZjGW{2{}b6(TcZ-Xf~27-AEI)?&oX>B2(Ft3li?+VIc%S{X*6e3SZb zvmyMy{hV_wGPf}UP(X{mBUum++mY%sXG%!uk#)@JV6`Ni_%t&5-N0Y5SHVikPc_nm zbvMK@p^&&a14jY$a&%=twSHHpffPS$kkf$5AD?sHQL#4XU;wZ3#TVTuW3ycIczW`g zN!ZNwcQVC@>O)fHNAUqp51BN9p=;cWfDwfKTL7XSBbr}KYDvq$%o42cpMgbP$s}Co zPSxVm_#??cr;yEf@IZJv9l3`V3^ELNj0`V5A2PAELIza^)F6Wi(dr*Eg!ZD5VX}8* zc=?kdFtd>1%|{?U9!pkG*%-V_OfSIC>P>y+mS=h_cmtNkJy)W;p-wlDES>CB6l$)bfOG@YPFgm{*XMW2dE1%P! zy^gfg$RY%?wkl3wZXKS*8WZ6R8gNVVRZ5l+Isi^-gHm1q$H}HL4)!2Y1se1a&3dS0 ztVfEBR81(m730CdZ3vVTY7UBQgcu^=zDX{Ofq%Bzi;v^xMnv+<|cU=rSv!<15tg!~gUXKd2h^5$^sWF! z1x2io=ZE{UP+g6|67vrAW1)K;#F73WO0UD%*At@k+PjUYhE)d7{&7A*ewKqF-cK?NdaYcb?q?@brr{K5zT7q=dATS8A(P*+0%l^Daf*9 zuA`#;eCUl}D^MeXwi7n2(5nkxJ|7o_`oTqt*?isv&>BO*iT)1QTBi>OsrH>CKeD>K z8q2J!uj5%M@mv07mgS>?FI#GG89HSC+h$o*x2(RM>^t1dC{;(%RUT`U26?&(U@Wx^ zV4R0-7a@xjwU}-j)aZ+87GI>eOp2#U z>=8kXd^ffjpB$kl4o*v5oG;~A{+qC<+C_y!=r+Q zJ}x4X@HzNPHV!5tq8R-RnjkRo9tSjzj*343A@tS|Sk95;LQe9X1f9r5M1W*s&AamQQScksJG{7zB)S;5zDbs$s0_s>l8BpkO01rUvjU{Uqz2))<-CDSm5l`qYSc-Z*v zF&7cDrY%CbEO8-0lLhCkgQ>WYB~H6kKmk1Dq!DAMZZf3A%O);W4O$>$_}kpHdQnZ( zgsci+j`&(`^)|3ryxfRsSX>Mp_Yp40#9;-ua+CT){!8L#)32;ggadh-z!qVm6mkGU zbH!l&f&+Zc1bOk94}$iC#`f&PjoUY@fyv>6zLo(5MXy91r6}VJ1!${QI~y&U4}cv^ zh-hX3(f`oah`GvA7>(h>93Cu~z{Ge=5;LOM8^=86ZZB__A%?()?F0!k{WCzH?N-Eb zc$4-x(cWCJJ+b7nEgtM5$MeLtgGuS}jmVCcWQ_H4#qu>1Hi<98UO~{&2!uItVAU#$zl@mM0|3Dfe2Abva2s$%&K77YcD4ZiUfN zTX-HF-CmL6zApU+sIAkXxqHzYG{aGsbkf$zB&_LG|A>Cd$7F zwA0>@dQ(N)x78rnkb3yl_xx1vD9~z(f#h@5yRxpr=kx4qhd^Bo+;Rjxz?DC`L%?49 zzQ20}HkcG#j`p4s3zG~h>c-ezxvo{V{_DWl?K7dn_5@!q4(sF!CDJy2x=JSba{4q( zYF>Fwt>dq;ZTz)0jmLGibI{;D5wM}}b%##S2Bxx`rLCNdVw4~!K95gGOSsDVCwmYv zp6f-<7qZF?>UfFHLhVecik=p#=^R}v5qZW_iz z{hi7-sARIUR6_&7WrQaJPf3K7ZVIaPg(Ms%9=t{cVOX&q+K)PWrZvdh@&g1w0_DCS9BWCiIN~Cdhf;-8Lvl@`@MhiHGJG#?lWZ!yDg5{*6vOQlThf4>{Os+ab3%x2%n;XC@vG zI%jB+d=462mLi(7r?JJ;#~K^#o5N=ye&+?|nd(`Hly6pjYao7y#+wYpZ^Zo##BaT6 zvLFPRSk>YTY@P%0n@iW}K%r>SNCWZPr@41jZ0{d>;I=FHzvWfOl7Uw1yJS5~rm84(h{$Z2v_*McxSg#yNG44p zIGKAaH|8@p){CH_X%$;=Ge;d6igV`Sa-}z@o+E{=yP_j_Jo9|?e95FY+NCK_u0yz7 zU0>3cur%BGYZ!D9&QC5;&Ut3ayzFShl{^3po{pWcRm-E_7%ajrYz+^F5x7XwsJ?2P zq=);mGM2OKL5Y%<2hmsFI^?-2n9%LIcGbV-C_K~;yy$5V{@MfPJlM>&=U3r`H8+fN za4G=lmTu;pEQK@q@3p5{tpt0T@9!Fwdzv<;_Hd7+&~yL2l9E{w1yP0j?cKH3av0ue zKC9m~TSg*0C^^3r2j5D^$vFW|6ITHtmDObC4xq@^^Q-pDlz{=;w|H3}97r$UiI?v> zzgo*|&pdz>u2Y}gA~WkU`^4U4^}ct(^De}9-5ak|k@!}rX3<`pxgjjfbPI<(#KC#u z2+&R<&WZ*ms}zZ0#p;`@-r{5~xj3DjL;Y}XwZ0h)D`Va8c<2vMPwuXJ_qtYBg7KQJ z*i~k_f(@7pA~j6v__h;+fUg12#vj*}OQkA?>dj^sSL;{qT&-UxhA@qOFW?17{#h&M@z?d$Qfiv=LFp-m zvJPV|FRj*Znng3njP|UD&)2W7ARP_4T+@q)O)9KDU*BA*mb3HI*tk)WtXe^XcZfUc z7}9Clou*THWb|V)=udv2Z)}2>9!YD{dGDeJa9&k+GReIDb6x#LZxQ({gaNgRdI@QZ zoEC=mHZMTX1r+fH>XH<2XCsy)<(XJVT`uffO2Z4|4 zH_#k_dzB18go8yvDA(r^>QqBfZ9qJYzpuh+e)XHnsD@EaPp3V^RHe}k#0lG+zM(`(0G4ksVdH=TYVZwwhmoCD-`tw7 zc(rtCda*zVbpry#N)=CJAjvNoOH$S8i`hz6pzIr_aXKj#$HPSAD6dtd0z4>WS38YVW{Aeu z{w7GiAhdVaK-Coh>Sq31{5hR3fsYH&Zbd*H#)AL?L2rOEgAiX|sLi8`%@`f(7y$l) z4XglrN;vm+7C0dJ!cD{|==2=_YuN(fZoZTh#CQ#}B%n-T#!G{OmoS2i=T*_ittq^9 zVVY_udB$c>#9JZ+_ZwO72jdZIQ{-ypaN$QPK11Awxi4IID0 z4nfA78!;$gRu$|PZQy*43Ul)}EP#SirCPBXr(dWnv$1|#@KscCpszGbM*8$Md+xpW zUKRT%{t#O~l*E4o@rRYpsW)_H@4&hLMj{+kIn3rpGb&VHs?oCLzKzE^d;6-%`&H-k zQ+Ms`Y0`B_i%mMO)o=36znG+gJGApP3_te|#O6Yp&TYgvYP2Y`>ht<~+L-|0<<5aIYq07Id%Zs?w z$2nnHM3naH-Sfh8uCYo-UuJ%dgVq z)thknGF`@R#wAXdAEwJCy8PuWxO|Z=pS%^9U!lvhuyW)U=yK+ET;5KX_rDRB_tItm zn{auEE`NRpE?=O_4O?-!o-QZfg3EEbJb5QBr*Oe$lTH>?{S~6h+|j%6>NovY9S_Lzh?S@}cdxyg-*jF*?}$bopDl{Bj(ZPtfIkgo4lEQmbzy{H!j`3%q9xU1988 zVa!}%tXyG?T*(aq)mIqrRv7127~fVH*H#$MRv5=t7{68+w^nk){1su!3S-F%W5^0) z#|mS{3S-3zW5fz$!wO@<3S+?vV*uX4aSE+r z)D>RU6<$%agN0n-m0aOfT*=+dBcxSZ;T2oq)xx~6NGrTbE4)H0ygDnqGAq0)E4(5r zyc*~oi?G70u)-^_!q8t~$geQeS8ACzg1;~m$n0YdkolM(|1xm{CjULe1Bm={C+Ju4 z8eCTBPbT&n{mI1s4f@5zet~HJ=hx%%3|*M)f0=$U+0PQ~Gui*2^oz;a*qChyR__T(!oEYI9^ZxdRlwnSD)+^Blxk8bj{4+W@P zU4=M%Yp^Z(1bQfBdKO{8XY1D?j8@hTfmV0zK@x3n|0l8U=6(-TfBc%t`&LguPKCi0 z5-;}l3Pdx}u@>=tx3AN;S-QGTT4W*3&5=Nkv(BJ}NP(SH?7CA}qB14<8dTin)!Izr F{|Cxd-@E_- literal 0 HcmV?d00001 diff --git a/doc/LectureNotes/_build/html/_panels_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css b/doc/LectureNotes/_build/html/_panels_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css new file mode 100644 index 000000000..fc14abc85 --- /dev/null +++ b/doc/LectureNotes/_build/html/_panels_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css @@ -0,0 +1 @@ +details.dropdown .summary-title{padding-right:3em !important;-moz-user-select:none;-ms-user-select:none;-webkit-user-select:none;user-select:none}details.dropdown:hover{cursor:pointer}details.dropdown .summary-content{cursor:default}details.dropdown summary{list-style:none;padding:1em}details.dropdown summary .octicon.no-title{vertical-align:middle}details.dropdown[open] summary .octicon.no-title{visibility:hidden}details.dropdown summary::-webkit-details-marker{display:none}details.dropdown summary:focus{outline:none}details.dropdown summary:hover .summary-up svg,details.dropdown summary:hover .summary-down svg{opacity:1}details.dropdown .summary-up svg,details.dropdown .summary-down svg{display:block;opacity:.6}details.dropdown .summary-up,details.dropdown .summary-down{pointer-events:none;position:absolute;right:1em;top:.75em}details.dropdown[open] .summary-down{visibility:hidden}details.dropdown:not([open]) .summary-up{visibility:hidden}details.dropdown.fade-in[open] summary~*{-moz-animation:panels-fade-in .5s ease-in-out;-webkit-animation:panels-fade-in .5s ease-in-out;animation:panels-fade-in .5s ease-in-out}details.dropdown.fade-in-slide-down[open] summary~*{-moz-animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out;-webkit-animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out;animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out}@keyframes panels-fade-in{0%{opacity:0}100%{opacity:1}}@keyframes panels-slide-down{0%{transform:translate(0, -10px)}100%{transform:translate(0, 0)}}.octicon{display:inline-block;fill:currentColor;vertical-align:text-top}.tabbed-content{box-shadow:0 -.0625rem var(--tabs-color-overline),0 .0625rem var(--tabs-color-underline);display:none;order:99;padding-bottom:.75rem;padding-top:.75rem;width:100%}.tabbed-content>:first-child{margin-top:0 !important}.tabbed-content>:last-child{margin-bottom:0 !important}.tabbed-content>.tabbed-set{margin:0}.tabbed-set{border-radius:.125rem;display:flex;flex-wrap:wrap;margin:1em 0;position:relative}.tabbed-set>input{opacity:0;position:absolute}.tabbed-set>input:checked+label{border-color:var(--tabs-color-label-active);color:var(--tabs-color-label-active)}.tabbed-set>input:checked+label+.tabbed-content{display:block}.tabbed-set>input:focus+label{outline-style:auto}.tabbed-set>input:not(.focus-visible)+label{outline:none;-webkit-tap-highlight-color:transparent}.tabbed-set>label{border-bottom:.125rem solid transparent;color:var(--tabs-color-label-inactive);cursor:pointer;font-size:var(--tabs-size-label);font-weight:700;padding:1em 1.25em .5em;transition:color 250ms;width:auto;z-index:1}html .tabbed-set>label:hover{color:var(--tabs-color-label-active)} diff --git a/doc/LectureNotes/_build/html/_panels_static/panels-variables.06eb56fa6e07937060861dad626602ad.css b/doc/LectureNotes/_build/html/_panels_static/panels-variables.06eb56fa6e07937060861dad626602ad.css new file mode 100644 index 000000000..adc616622 --- /dev/null +++ b/doc/LectureNotes/_build/html/_panels_static/panels-variables.06eb56fa6e07937060861dad626602ad.css @@ -0,0 +1,7 @@ +:root { +--tabs-color-label-active: hsla(231, 99%, 66%, 1); +--tabs-color-label-inactive: rgba(178, 206, 245, 0.62); +--tabs-color-overline: rgb(207, 236, 238); +--tabs-color-underline: rgb(207, 236, 238); +--tabs-size-label: 1rem; +} \ No newline at end of file diff --git a/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb b/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb index 25296c4e0..b3c1f79cc 100644 --- a/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb +++ b/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "1b941c35", + "id": "8e6632a0", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "dc05b096", + "id": "82705c4f", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "2cf07405", + "id": "921bf331", "metadata": { "editable": true }, @@ -46,7 +46,7 @@ }, { "cell_type": "markdown", - "id": "3c139edb", + "id": "adff65d5", "metadata": { "editable": true }, @@ -58,7 +58,7 @@ }, { "cell_type": "markdown", - "id": "aad4cfac", + "id": "70418b3d", "metadata": { "editable": true }, @@ -70,7 +70,7 @@ }, { "cell_type": "markdown", - "id": "6682282f", + "id": "11a3cf73", "metadata": { "editable": true }, @@ -83,7 +83,7 @@ }, { "cell_type": "markdown", - "id": "89e2f4c4", + "id": "04a06b51", "metadata": { "editable": true }, @@ -99,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "b06d4e53", + "id": "408db3d9", "metadata": { "editable": true }, @@ -120,7 +120,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "63796480", + "id": "37fb732c", "metadata": { "collapsed": false, "editable": true @@ -140,12 +140,12 @@ }, { "cell_type": "markdown", - "id": "80748600", + "id": "d861e1e3", "metadata": { "editable": true }, "source": [ - "Fill in the necessary details.\n", + "Fill in the necessary details. Do we need to center the $y$-values? \n", "\n", "After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n", "and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This makes the optimization landscape\n", @@ -156,7 +156,7 @@ }, { "cell_type": "markdown", - "id": "92751e5f", + "id": "b3e774d0", "metadata": { "editable": true }, @@ -168,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "aedfbd7a", + "id": "d5dc7708", "metadata": { "editable": true }, @@ -179,7 +179,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "5d1288fa", + "id": "4c9c86ac", "metadata": { "collapsed": false, "editable": true @@ -187,7 +187,9 @@ "outputs": [], "source": [ "# Set regularization parameter, either a single value or a vector of values\n", - "lambda = ?\n", + "# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called \"anonymous functions\" or \"lambda functions.\"\n", + "lam = ?\n", + "\n", "\n", "# Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y\n", "I = np.eye(n_features)\n", @@ -200,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "628f5e89", + "id": "eeae00fd", "metadata": { "editable": true }, @@ -214,7 +216,7 @@ }, { "cell_type": "markdown", - "id": "f115ba4e", + "id": "e1c215d5", "metadata": { "editable": true }, @@ -226,7 +228,7 @@ }, { "cell_type": "markdown", - "id": "a9b5189c", + "id": "587dd3dc", "metadata": { "editable": true }, @@ -238,7 +240,7 @@ }, { "cell_type": "markdown", - "id": "a3969ff6", + "id": "bfa34697", "metadata": { "editable": true }, @@ -258,7 +260,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "34d87303", + "id": "49245f55", "metadata": { "collapsed": false, "editable": true @@ -273,19 +275,8 @@ "# Initialize weights for gradient descent\n", "theta = np.zeros(n_features)\n", "\n", - "# Arrays to store history for plotting\n", - "cost_history = np.zeros(num_iters)\n", - "\n", "# Gradient descent loop\n", - "m = n_samples # number of data points\n", "for t in range(num_iters):\n", - " # Compute prediction error\n", - " error = X_norm.dot(theta) - y_centered \n", - " # Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring\n", - " cost_OLS = ?\n", - " cost_Ridge = ?\n", - " # You could add a history for both methods (optional)\n", - " cost_history[t] = ?\n", " # Compute gradients for OSL and Ridge\n", " grad_OLS = ?\n", " grad_Ridge = ?\n", @@ -302,31 +293,33 @@ }, { "cell_type": "markdown", - "id": "989f70bb", + "id": "f3f43f2c", "metadata": { "editable": true }, "source": [ "### 4a)\n", "\n", - "Discuss the results as function of the learning rate parameters and the number of iterations." + "Write first a gradient descent code for OLS only using the above template.\n", + "Discuss the results as function of the learning rate parameters and the number of iterations" ] }, { "cell_type": "markdown", - "id": "370b2dad", + "id": "9ba303be", "metadata": { "editable": true }, "source": [ "### 4b)\n", "\n", + "Write then a similar code for Ridge regression using the above template.\n", "Try to add a stopping parameter as function of the number iterations and the difference between the new and old $\\theta$ values. How would you define a stopping criterion?" ] }, { "cell_type": "markdown", - "id": "ef197cd7", + "id": "78362c6c", "metadata": { "editable": true }, @@ -352,7 +345,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "4ccc2f65", + "id": "8be1cebe", "metadata": { "collapsed": false, "editable": true @@ -381,7 +374,7 @@ }, { "cell_type": "markdown", - "id": "00e279ef", + "id": "e2693666", "metadata": { "editable": true }, @@ -395,7 +388,7 @@ }, { "cell_type": "markdown", - "id": "c910b3f4", + "id": "bc954d12", "metadata": { "editable": true }, @@ -407,7 +400,7 @@ }, { "cell_type": "markdown", - "id": "89e6e040", + "id": "6534b610", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/html/_sources/exercisesweek38.ipynb b/doc/LectureNotes/_build/html/_sources/exercisesweek38.ipynb new file mode 100644 index 000000000..355eef5c9 --- /dev/null +++ b/doc/LectureNotes/_build/html/_sources/exercisesweek38.ipynb @@ -0,0 +1,429 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1da77599", + "metadata": {}, + "source": [ + "# Exercises week 37\n", + "## September 15-19\n", + "\n", + "## Resampling and the Bias-Variance Trade-off" + ] + }, + { + "cell_type": "markdown", + "id": "e9f27b0e", + "metadata": {}, + "source": [ + "## Learning goals\n", + "\n", + "After completing these exercises, you will know how to\n", + "- Derive expectation and variances values related to linear regression\n", + "- Compute expectation and variances values related to linear regression\n", + "- Compute and evaluate the trade-off between bias and variance of a model" + ] + }, + { + "cell_type": "markdown", + "id": "984af8e3", + "metadata": {}, + "source": [ + "This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercises are also a part of project 1 and can be reused in the theory part of the project.\n", + "\n", + "For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "\n", + "We assume that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$ which describes our data" + ] + }, + { + "cell_type": "markdown", + "id": "c16f7d0e", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9fcf981a", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "source": [ + "We further assume that this continous function can be modeled with a linear model $\\mathbf{\\tilde{y}}$ of some features $\\mathbf{X}$." + ] + }, + { + "cell_type": "markdown", + "id": "d4189366", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{\\tilde{y}} + \\boldsymbol{\\varepsilon} = \\boldsymbol{X}\\boldsymbol{\\beta} +\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f4fca21b", + "metadata": {}, + "source": [ + "We therefore get that our data $\\boldsymbol{y}$ has an expectation value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$, that is $\\boldsymbol{y}$ follows a normal distribution with mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$." + ] + }, + { + "cell_type": "markdown", + "id": "5de0c7e6", + "metadata": {}, + "source": [ + "## Exercise 1: Expectation values for ordinary least squares expressions" + ] + }, + { + "cell_type": "markdown", + "id": "d878c699", + "metadata": {}, + "source": [ + "**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ show that" + ] + }, + { + "cell_type": "markdown", + "id": "08b7007d", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "46e93394", + "metadata": {}, + "source": [ + "**b)** Show that the variance of $\\boldsymbol{\\hat{\\beta}_{OLS}}$ is" + ] + }, + { + "cell_type": "markdown", + "id": "be1b65be", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{Var}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d2143684", + "metadata": {}, + "source": [ + "We can use the last expression when we define a [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval) for the parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$. \n", + "A given parameter ${\\boldsymbol{\\hat{\\beta}_{OLS}}}_j$ is given by the diagonal matrix element of the above matrix." + ] + }, + { + "cell_type": "markdown", + "id": "f5c2dc22", + "metadata": {}, + "source": [ + "## Exercise 2: Expectation values for Ridge regression" + ] + }, + { + "cell_type": "markdown", + "id": "3893e3e7", + "metadata": {}, + "source": [ + "**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{Ridge}}$ show that" + ] + }, + { + "cell_type": "markdown", + "id": "79dc571f", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "028209a1", + "metadata": {}, + "source": [ + "We see that $\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big] \\not= \\mathbb{E} \\big[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}\\big ]$ for any $\\lambda > 0$." + ] + }, + { + "cell_type": "markdown", + "id": "65f6f914", + "metadata": {}, + "source": [ + "**b)** Why do we say that Ridge regression gives a biased estimate? Is this a problem?" + ] + }, + { + "cell_type": "markdown", + "id": "b4e721fc", + "metadata": {}, + "source": [ + "**c)** Show that the variance is" + ] + }, + { + "cell_type": "markdown", + "id": "090eb1e1", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6b8e8697", + "metadata": {}, + "source": [ + "We see that if the parameter $\\lambda$ goes to infinity then the variance of the Ridge parameters $\\boldsymbol{\\beta}$ goes to zero." + ] + }, + { + "cell_type": "markdown", + "id": "74bc300b", + "metadata": {}, + "source": [ + "## Exercise 3: Deriving the expression for the Bias-Variance Trade-off" + ] + }, + { + "cell_type": "markdown", + "id": "eeb86010", + "metadata": {}, + "source": [ + "The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.\n", + "\n", + "The parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ are found by optimizing the mean squared error via the so-called cost function" + ] + }, + { + "cell_type": "markdown", + "id": "522a0d1d", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "831db06c", + "metadata": {}, + "source": [ + "**a)** Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a\n", + "term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function $f$ with the model ${\\bf y}$.\n", + "show that" + ] + }, + { + "cell_type": "markdown", + "id": "8cc52b3c", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8cb50416", + "metadata": {}, + "source": [ + "with" + ] + }, + { + "cell_type": "markdown", + "id": "e49bdbb4", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "eca5554a", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "id": "b1054343", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{var}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\tilde{\\boldsymbol{y}}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "70fbfcd7", + "metadata": {}, + "source": [ + "**b)** Explain what the terms mean and discuss their interpretations." + ] + }, + { + "cell_type": "markdown", + "id": "b8f8b9d1", + "metadata": {}, + "source": [ + "## Exercise 4: Computing the Bias and Variance" + ] + }, + { + "cell_type": "markdown", + "id": "9e012430", + "metadata": {}, + "source": [ + "Before you compute the bias and variance of a real model for different complexities, let's for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.\n", + "\n", + "**a)** Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "b5bf581c", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "n = 100\n", + "bootstraps = 1000\n", + "\n", + "predictions = np.random.rand(bootstraps, n) * 10 + 10\n", + "targets = np.random.rand(bootstraps, n)\n", + "\n", + "mse = ...\n", + "bias = ...\n", + "variance = ..." + ] + }, + { + "cell_type": "markdown", + "id": "7b1dc621", + "metadata": {}, + "source": [ + "**b)** Change the prediction values in some way to increase the bias while decreasing the variance.\n", + "\n", + "**c)** Change the prediction values in some way to increase the variance while decreasing the bias." + ] + }, + { + "cell_type": "markdown", + "id": "8da63362", + "metadata": {}, + "source": [ + "**d)** Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "dd5855e4", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.preprocessing import PolynomialFeatures # use the fit_transform method of the created object!\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.utils import resample" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e35fa37", + "metadata": {}, + "outputs": [], + "source": [ + "n = 100\n", + "bootstraps = 1000\n", + "\n", + "x = np.linspace(-3, 3, n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1)\n", + "\n", + "biases = []\n", + "variances = []\n", + "mses = []\n", + "\n", + "#for p in range(1, 5):\n", + "# predictions = ...\n", + "# targets = ...\n", + "# for b in range(bootstraps):\n", + "# x_sample, y_sample = ...\n", + "# X = ...\n", + "# X_train, X_test, y_train, y_test = ...\n", + "#\n", + "# predictions[b, :] = \n", + "# targets[b, :] = \n", + "# \n", + "# biases.append(...)\n", + "# variances.append(...)\n", + "# mses.append(...)" + ] + }, + { + "cell_type": "markdown", + "id": "253b8461", + "metadata": {}, + "source": [ + "**e)** Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).\n", + "\n", + "**f)** Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/LectureNotes/_build/html/_sources/week37.ipynb b/doc/LectureNotes/_build/html/_sources/week37.ipynb new file mode 100644 index 000000000..acd6b4e52 --- /dev/null +++ b/doc/LectureNotes/_build/html/_sources/week37.ipynb @@ -0,0 +1,3736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c543a71a", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "4ff6b050", + "metadata": { + "editable": true + }, + "source": [ + "# Week 37: Statistical interpretations and Resampling Methods\n", + "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo, Norway\n", + "\n", + "Date: **September 8-12, 2025**\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "a7079d3e", + "metadata": { + "editable": true + }, + "source": [ + "## Plans for week 37, lecture Monday\n", + "\n", + "**Plans and material for the lecture on Monday September 8.**\n", + "\n", + "The family of gradient descent methods\n", + "1. Plain gradient descent (constant learning rate), reminder from last week with examples using OLS and Ridge\n", + "\n", + "2. Improving gradient descent with momentum\n", + "\n", + "3. Introducing stochastic gradient descent\n", + "\n", + "4. More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "db3af8cc", + "metadata": { + "editable": true + }, + "source": [ + "## Readings and Videos:\n", + "1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at and chapter 8.3-8.5 at \n", + "\n", + "2. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.\n", + "\n", + "3. Video on gradient descent at \n", + "\n", + "4. Video on Stochastic gradient descent at " + ] + }, + { + "cell_type": "markdown", + "id": "79d6d5be", + "metadata": { + "editable": true + }, + "source": [ + "## Material for lecture Monday September 8" + ] + }, + { + "cell_type": "markdown", + "id": "fa427fc3", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and revisiting Ordinary Least Squares from last week\n", + "\n", + "Last week we started with linear regression as a case study for the gradient descent\n", + "methods. Linear regression is a great test case for the gradient\n", + "descent methods discussed in the lectures since it has several\n", + "desirable properties such as:\n", + "\n", + "1. An analytical solution (recall homework sets for week 35).\n", + "\n", + "2. The gradient can be computed analytically.\n", + "\n", + "3. The cost function is convex which guarantees that gradient descent converges for small enough learning rates\n", + "\n", + "We revisit an example similar to what we had in the first homework set. We have a function of the type" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1185629e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "x = 2*np.random.rand(m,1)\n", + "y = 4+3*x+np.random.randn(m,1)" + ] + }, + { + "cell_type": "markdown", + "id": "42dfe293", + "metadata": { + "editable": true + }, + "source": [ + "with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n", + "The linear regression model is given by" + ] + }, + { + "cell_type": "markdown", + "id": "a726cc6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_\\theta(x) = \\boldsymbol{y} = \\theta_0 + \\theta_1 x,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "991c8919", + "metadata": { + "editable": true + }, + "source": [ + "such that" + ] + }, + { + "cell_type": "markdown", + "id": "ba895122", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y}_i = \\theta_0 + \\theta_1 x_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5c2b9317", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent example\n", + "\n", + "Let $\\mathbf{y} = (y_1,\\cdots,y_n)^T$, $\\mathbf{\\boldsymbol{y}} = (\\boldsymbol{y}_1,\\cdots,\\boldsymbol{y}_n)^T$ and $\\theta = (\\theta_0, \\theta_1)^T$\n", + "\n", + "It is convenient to write $\\mathbf{\\boldsymbol{y}} = X\\theta$ where $X \\in \\mathbb{R}^{100 \\times 2} $ is the design matrix given by (we keep the intercept here)" + ] + }, + { + "cell_type": "markdown", + "id": "264eef67", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "X \\equiv \\begin{bmatrix}\n", + "1 & x_1 \\\\\n", + "\\vdots & \\vdots \\\\\n", + "1 & x_{100} & \\\\\n", + "\\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a451a9ce", + "metadata": { + "editable": true + }, + "source": [ + "The cost/loss/risk function is given by (" + ] + }, + { + "cell_type": "markdown", + "id": "e7fa569b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta) = \\frac{1}{n}||X\\theta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\theta_0 + \\theta_1 x_i)^2 - 2 y_i (\\theta_0 + \\theta_1 x_i) + y_i^2\\right]\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a2655bc6", + "metadata": { + "editable": true + }, + "source": [ + "and we want to find $\\theta$ such that $C(\\theta)$ is minimized." + ] + }, + { + "cell_type": "markdown", + "id": "277ace2e", + "metadata": { + "editable": true + }, + "source": [ + "## The derivative of the cost/loss function\n", + "\n", + "Computing $\\partial C(\\theta) / \\partial \\theta_0$ and $\\partial C(\\theta) / \\partial \\theta_1$ we can show that the gradient can be written as" + ] + }, + { + "cell_type": "markdown", + "id": "75bbf03c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\theta} C(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T(X\\theta - \\mathbf{y}),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5bb2db2", + "metadata": { + "editable": true + }, + "source": [ + "where $X$ is the design matrix defined above." + ] + }, + { + "cell_type": "markdown", + "id": "53430d47", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix\n", + "The Hessian matrix of $C(\\theta)$ is given by" + ] + }, + { + "cell_type": "markdown", + "id": "89dce328", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "542b8278", + "metadata": { + "editable": true + }, + "source": [ + "This result implies that $C(\\theta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite." + ] + }, + { + "cell_type": "markdown", + "id": "2b94baca", + "metadata": { + "editable": true + }, + "source": [ + "## Simple program\n", + "\n", + "We can now write a program that minimizes $C(\\theta)$ using the gradient descent method with a constant learning rate $\\gamma$ according to" + ] + }, + { + "cell_type": "markdown", + "id": "bfc6b93f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{k+1} = \\theta_k - \\gamma \\nabla_\\theta C(\\theta_k), \\ k=0,1,\\cdots\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "703de92e", + "metadata": { + "editable": true + }, + "source": [ + "We can use the expression we computed for the gradient and let use a\n", + "$\\theta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n", + "when $||\\nabla_\\theta C(\\theta_k) || \\leq \\epsilon = 10^{-8}$. **Note that the code below does not include the latter stop criterion**.\n", + "\n", + "And finally we can compare our solution for $\\theta$ with the analytic result given by \n", + "$\\theta= (X^TX)^{-1} X^T \\mathbf{y}$." + ] + }, + { + "cell_type": "markdown", + "id": "f843aae1", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient Descent Example\n", + "\n", + "Here our simple example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b76443c9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "\n", + "# Importing various packages\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "# Hessian matrix\n", + "H = (2.0/n)* X.T @ X\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y\n", + "print(theta_linreg)\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "for iter in range(Niterations):\n", + " gradient = (2.0/n)*X.T @ (X @ theta-y)\n", + " theta -= eta*gradient\n", + "\n", + "print(theta)\n", + "xnew = np.array([[0],[2]])\n", + "xbnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = xbnew.dot(theta)\n", + "ypredict2 = xbnew.dot(theta_linreg)\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7b75f4ec", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and Ridge\n", + "\n", + "We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\\theta$," + ] + }, + { + "cell_type": "markdown", + "id": "47ecdee7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C_{\\text{ridge}}(\\theta) = \\frac{1}{n}||X\\theta -\\mathbf{y}||^2 + \\lambda ||\\theta||^2, \\ \\lambda \\geq 0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff25299b", + "metadata": { + "editable": true + }, + "source": [ + "In order to minimize $C_{\\text{ridge}}(\\theta)$ using GD we adjust the gradient as follows" + ] + }, + { + "cell_type": "markdown", + "id": "21a88a84", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\theta C_{\\text{ridge}}(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} + 2\\lambda\\begin{bmatrix} \\theta_0 \\\\ \\theta_1\\end{bmatrix} = 2 (\\frac{1}{n}X^T(X\\theta - \\mathbf{y})+\\lambda \\theta).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67491c15", + "metadata": { + "editable": true + }, + "source": [ + "We can easily extend our program to minimize $C_{\\text{ridge}}(\\theta)$ using gradient descent and compare with the analytical solution given by" + ] + }, + { + "cell_type": "markdown", + "id": "f1ac13f1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67d83802", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix for Ridge Regression\n", + "The Hessian matrix of Ridge Regression for our simple example is given by" + ] + }, + { + "cell_type": "markdown", + "id": "75935d70", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X+2\\lambda\\boldsymbol{I}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4707a65d", + "metadata": { + "editable": true + }, + "source": [ + "This implies that the Hessian matrix is positive definite, hence the stationary point is a\n", + "minimum.\n", + "Note that the Ridge cost function is convex being a sum of two convex\n", + "functions. Therefore, the stationary point is a global\n", + "minimum of this function." + ] + }, + { + "cell_type": "markdown", + "id": "ab16b163", + "metadata": { + "editable": true + }, + "source": [ + "## Program example for gradient descent with Ridge Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fe7ee1f7", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "\n", + "#Ridge parameter lambda\n", + "lmbda = 0.001\n", + "Id = n*lmbda* np.eye(XT_X.shape[0])\n", + "\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X+2*lmbda* np.eye(XT_X.shape[0])\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "\n", + "theta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y\n", + "print(theta_linreg)\n", + "# Start plain gradient descent\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ (X @ (theta)-y)+2*lmbda*theta\n", + " theta -= eta*gradients\n", + "\n", + "print(theta)\n", + "ypredict = X @ theta\n", + "ypredict2 = X @ theta_linreg\n", + "plt.plot(x, ypredict, \"r-\")\n", + "plt.plot(x, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example for Ridge')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "823fdaf7", + "metadata": { + "editable": true + }, + "source": [ + "## Using gradient descent methods, limitations\n", + "\n", + "* **Gradient descent (GD) finds local minima of our function**. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.\n", + "\n", + "* **GD is sensitive to initial conditions**. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.\n", + "\n", + "* **Gradients are computationally expensive to calculate for large datasets**. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, $E \\propto \\sum_{i=1}^n (y_i - \\mathbf{w}^T\\cdot\\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called \"mini batches\". This has the added benefit of introducing stochasticity into our algorithm.\n", + "\n", + "* **GD is very sensitive to choices of learning rates**. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.\n", + "\n", + "* **GD treats all directions in parameter space uniformly.** Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. \n", + "\n", + "* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points." + ] + }, + { + "cell_type": "markdown", + "id": "21ea3f2a", + "metadata": { + "editable": true + }, + "source": [ + "## Improving gradient descent with momentum\n", + "\n", + "We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5b729a7e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# take a step\n", + "\t\tsolution = solution - step_size * gradient\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# perform the gradient descent search\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "209fc69c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "984c4202", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# keep track of the change\n", + "\tchange = 0.0\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# calculate update\n", + "\t\tnew_change = step_size * gradient + momentum * change\n", + "\t\t# take a step\n", + "\t\tsolution = solution - new_change\n", + "\t\t# save the change\n", + "\t\tchange = new_change\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# define momentum\n", + "momentum = 0.3\n", + "# perform the gradient descent search with momentum\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "8577b238", + "metadata": { + "editable": true + }, + "source": [ + "## Overview video on Stochastic Gradient Descent (SGD)\n", + "\n", + "[What is Stochastic Gradient Descent](https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer)\n", + "There are several reasons for using stochastic gradient descent. Some of these are:\n", + "\n", + "1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.\n", + "\n", + "2. Hopefully avoid Local Minima\n", + "\n", + "3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c11d0874", + "metadata": { + "editable": true + }, + "source": [ + "## Batches and mini-batches\n", + "\n", + "In gradient descent we compute the cost function and its gradient for all data points we have.\n", + "\n", + "In large-scale applications such as the [ILSVRC challenge](https://www.image-net.org/challenges/LSVRC/), the\n", + "training data can have on order of millions of examples. Hence, it\n", + "seems wasteful to compute the full cost function over the entire\n", + "training set in order to perform only a single parameter update. A\n", + "very common approach to addressing this challenge is to compute the\n", + "gradient over batches of the training data. For example, a typical batch could contain some thousand examples from\n", + "an entire training set of several millions. This batch is then used to\n", + "perform a parameter update." + ] + }, + { + "cell_type": "markdown", + "id": "64ad52d0", + "metadata": { + "editable": true + }, + "source": [ + "## Pros and cons\n", + "\n", + "1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.\n", + "\n", + "2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.\n", + "\n", + "3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient." + ] + }, + { + "cell_type": "markdown", + "id": "5dd4c467", + "metadata": { + "editable": true + }, + "source": [ + "## Convergence rates\n", + "\n", + "1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.\n", + "\n", + "2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration." + ] + }, + { + "cell_type": "markdown", + "id": "42904823", + "metadata": { + "editable": true + }, + "source": [ + "## Accuracy\n", + "\n", + "In general, stochastic Gradient Descent is Less accurate than gradient\n", + "descent, as it calculates the gradient on single examples, which may\n", + "not accurately represent the overall dataset. Gradient Descent is\n", + "more accurate because it uses the average gradient calculated over the\n", + "entire dataset.\n", + "\n", + "There are other disadvantages to using SGD. The main drawback is that\n", + "its convergence behaviour can be more erratic due to the random\n", + "sampling of individual training examples. This can lead to less\n", + "accurate results, as the algorithm may not converge to the true\n", + "minimum of the cost function. Additionally, the learning rate, which\n", + "determines the step size of each update to the model’s parameters,\n", + "must be carefully chosen to ensure convergence.\n", + "\n", + "It is however the method of choice in deep learning algorithms where\n", + "SGD is often used in combination with other optimization techniques,\n", + "such as momentum or adaptive learning rates" + ] + }, + { + "cell_type": "markdown", + "id": "49dfaf2e", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent (SGD)\n", + "\n", + "In stochastic gradient descent, the extreme case is the case where we\n", + "have only one batch, that is we include the whole data set.\n", + "\n", + "This process is called Stochastic Gradient\n", + "Descent (SGD) (or also sometimes on-line gradient descent). This is\n", + "relatively less common to see because in practice due to vectorized\n", + "code optimizations it can be computationally much more efficient to\n", + "evaluate the gradient for 100 examples, than the gradient for one\n", + "example 100 times. Even though SGD technically refers to using a\n", + "single example at a time to evaluate the gradient, you will hear\n", + "people use the term SGD even when referring to mini-batch gradient\n", + "descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD\n", + "for “Batch gradient descent” are rare to see), where it is usually\n", + "assumed that mini-batches are used. The size of the mini-batch is a\n", + "hyperparameter but it is not very common to cross-validate or bootstrap it. It is\n", + "usually based on memory constraints (if any), or set to some value,\n", + "e.g. 32, 64 or 128. We use powers of 2 in practice because many\n", + "vectorized operation implementations work faster when their inputs are\n", + "sized in powers of 2.\n", + "\n", + "In our notes with SGD we mean stochastic gradient descent with mini-batches." + ] + }, + { + "cell_type": "markdown", + "id": "284b8663", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent\n", + "\n", + "Stochastic gradient descent (SGD) and variants thereof address some of\n", + "the shortcomings of the Gradient descent method discussed above.\n", + "\n", + "The underlying idea of SGD comes from the observation that the cost\n", + "function, which we want to minimize, can almost always be written as a\n", + "sum over $n$ data points $\\{\\mathbf{x}_i\\}_{i=1}^n$," + ] + }, + { + "cell_type": "markdown", + "id": "5e8cc1b8", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "510505a0", + "metadata": { + "editable": true + }, + "source": [ + "## Computation of gradients\n", + "\n", + "This in turn means that the gradient can be\n", + "computed as a sum over $i$-gradients" + ] + }, + { + "cell_type": "markdown", + "id": "32fc501f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\beta C(\\mathbf{\\beta}) = \\sum_i^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9dcac7ef", + "metadata": { + "editable": true + }, + "source": [ + "Stochasticity/randomness is introduced by only taking the\n", + "gradient on a subset of the data called minibatches. If there are $n$\n", + "data points and the size of each minibatch is $M$, there will be $n/M$\n", + "minibatches. We denote these minibatches by $B_k$ where\n", + "$k=1,\\cdots,n/M$." + ] + }, + { + "cell_type": "markdown", + "id": "c9464120", + "metadata": { + "editable": true + }, + "source": [ + "## SGD example\n", + "As an example, suppose we have $10$ data points $(\\mathbf{x}_1,\\cdots, \\mathbf{x}_{10})$ \n", + "and we choose to have $M=5$ minibathces,\n", + "then each minibatch contains two data points. In particular we have\n", + "$B_1 = (\\mathbf{x}_1,\\mathbf{x}_2), \\cdots, B_5 =\n", + "(\\mathbf{x}_9,\\mathbf{x}_{10})$. Note that if you choose $M=1$ you\n", + "have only a single batch with all data points and on the other extreme,\n", + "you may choose $M=n$ resulting in a minibatch for each datapoint, i.e\n", + "$B_k = \\mathbf{x}_k$.\n", + "\n", + "The idea is now to approximate the gradient by replacing the sum over\n", + "all data points with a sum over the data points in one the minibatches\n", + "picked at random in each gradient descent step" + ] + }, + { + "cell_type": "markdown", + "id": "59bc847c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\beta}\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}) \\rightarrow \\sum_{i \\in B_k}^n \\nabla_\\beta\n", + "c_i(\\mathbf{x}_i, \\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ba220299", + "metadata": { + "editable": true + }, + "source": [ + "## The gradient step\n", + "\n", + "Thus a gradient descent step now looks like" + ] + }, + { + "cell_type": "markdown", + "id": "8608ba34", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\beta_{j+1} = \\beta_j - \\gamma_j \\sum_{i \\in B_k}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta})\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0faa1782", + "metadata": { + "editable": true + }, + "source": [ + "where $k$ is picked at random with equal\n", + "probability from $[1,n/M]$. An iteration over the number of\n", + "minibathces (n/M) is commonly referred to as an epoch. Thus it is\n", + "typical to choose a number of epochs and for each epoch iterate over\n", + "the number of minibatches, as exemplified in the code below." + ] + }, + { + "cell_type": "markdown", + "id": "97eddb68", + "metadata": { + "editable": true + }, + "source": [ + "## Simple example code" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "309da17e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 10 #number of epochs\n", + "\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for \n", + " j += 1" + ] + }, + { + "cell_type": "markdown", + "id": "7aea4344", + "metadata": { + "editable": true + }, + "source": [ + "Taking the gradient only on a subset of the data has two important\n", + "benefits. First, it introduces randomness which decreases the chance\n", + "that our opmization scheme gets stuck in a local minima. Second, if\n", + "the size of the minibatches are small relative to the number of\n", + "datapoints ($M < n$), the computation of the gradient is much\n", + "cheaper since we sum over the datapoints in the $k-th$ minibatch and not\n", + "all $n$ datapoints." + ] + }, + { + "cell_type": "markdown", + "id": "52f8d549", + "metadata": { + "editable": true + }, + "source": [ + "## When do we stop?\n", + "\n", + "A natural question is when do we stop the search for a new minimum?\n", + "One possibility is to compute the full gradient after a given number\n", + "of epochs and check if the norm of the gradient is smaller than some\n", + "threshold and stop if true. However, the condition that the gradient\n", + "is zero is valid also for local minima, so this would only tell us\n", + "that we are close to a local/global minimum. However, we could also\n", + "evaluate the cost function at this point, store the result and\n", + "continue the search. If the test kicks in at a later stage we can\n", + "compare the values of the cost function and keep the $\\beta$ that\n", + "gave the lowest value." + ] + }, + { + "cell_type": "markdown", + "id": "dcd04de7", + "metadata": { + "editable": true + }, + "source": [ + "## Slightly different approach\n", + "\n", + "Another approach is to let the step length $\\gamma_j$ depend on the\n", + "number of epochs in such a way that it becomes very small after a\n", + "reasonable time such that we do not move at all. Such approaches are\n", + "also called scaling. There are many such ways to [scale the learning\n", + "rate](https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1)\n", + "and [discussions here](https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf). See\n", + "also\n", + "\n", + "for a discussion of different scaling functions for the learning rate." + ] + }, + { + "cell_type": "markdown", + "id": "4ecf0b16", + "metadata": { + "editable": true + }, + "source": [ + "## Time decay rate\n", + "\n", + "As an example, let $e = 0,1,2,3,\\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \\cdot m + i$ where $m$ is the number of minibatches and $i=0,\\cdots,m-1$. Then the function $$\\gamma_j(t; t_0, t_1) = \\frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$.\n", + "\n", + "In this way we can fix the number of epochs, compute $\\beta$ and\n", + "evaluate the cost function at the end. Repeating the computation will\n", + "give a different result since the scheme is random by design. Then we\n", + "pick the final $\\beta$ that gives the lowest value of the cost\n", + "function." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "71eb89db", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "def step_length(t,t0,t1):\n", + " return t0/(t+t1)\n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 500 #number of epochs\n", + "t0 = 1.0\n", + "t1 = 10\n", + "\n", + "gamma_j = t0/t1\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for beta\n", + " t = epoch*m+i\n", + " gamma_j = step_length(t,t0,t1)\n", + " j += 1\n", + "\n", + "print(\"gamma_j after %d epochs: %g\" % (n_epochs,gamma_j))" + ] + }, + { + "cell_type": "markdown", + "id": "10ce3fc7", + "metadata": { + "editable": true + }, + "source": [ + "## Code with a Number of Minibatches which varies\n", + "\n", + "In the code here we vary the number of mini-batches." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f2d402a8", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "from math import exp, sqrt\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ ((X @ theta)-y)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ffd1e92d", + "metadata": { + "editable": true + }, + "source": [ + "## Replace or not\n", + "\n", + "In the above code, we have use replacement in setting up the\n", + "mini-batches. The discussion\n", + "[here](https://sebastianraschka.com/faq/docs/sgd-methods.html) may be\n", + "useful." + ] + }, + { + "cell_type": "markdown", + "id": "4b72d3f9", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum based GD\n", + "\n", + "The stochastic gradient descent (SGD) is almost always used with a\n", + "*momentum* or inertia term that serves as a memory of the direction we\n", + "are moving in parameter space. This is typically implemented as\n", + "follows" + ] + }, + { + "cell_type": "markdown", + "id": "1b60ec02", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e1fa24ba", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t},\n", + "\\label{_auto1} \\tag{1}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "7a1fc9fc", + "metadata": { + "editable": true + }, + "source": [ + "where we have introduced a momentum parameter $\\gamma$, with\n", + "$0\\le\\gamma\\le 1$, and for brevity we dropped the explicit notation to\n", + "indicate the gradient is to be taken over a different mini-batch at\n", + "each step. We call this algorithm gradient descent with momentum\n", + "(GDM). From these equations, it is clear that $\\mathbf{v}_t$ is a\n", + "running average of recently encountered gradients and\n", + "$(1-\\gamma)^{-1}$ sets the characteristic time scale for the memory\n", + "used in the averaging procedure. Consistent with this, when\n", + "$\\gamma=0$, this just reduces down to ordinary SGD as discussed\n", + "earlier. An equivalent way of writing the updates is" + ] + }, + { + "cell_type": "markdown", + "id": "23abda5d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\boldsymbol{\\theta}_{t+1} = \\gamma \\Delta \\boldsymbol{\\theta}_t -\\ \\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0f0067a9", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined $\\Delta \\boldsymbol{\\theta}_{t}= \\boldsymbol{\\theta}_t-\\boldsymbol{\\theta}_{t-1}$." + ] + }, + { + "cell_type": "markdown", + "id": "b86ec959", + "metadata": { + "editable": true + }, + "source": [ + "## More on momentum based approaches\n", + "\n", + "Let us try to get more intuition from these equations. It is helpful\n", + "to consider a simple physical analogy with a particle of mass $m$\n", + "moving in a viscous medium with drag coefficient $\\mu$ and potential\n", + "$E(\\mathbf{w})$. If we denote the particle's position by $\\mathbf{w}$,\n", + "then its motion is described by" + ] + }, + { + "cell_type": "markdown", + "id": "7685ff97", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m {d^2 \\mathbf{w} \\over dt^2} + \\mu {d \\mathbf{w} \\over dt }= -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "28203c1e", + "metadata": { + "editable": true + }, + "source": [ + "We can discretize this equation in the usual way to get" + ] + }, + { + "cell_type": "markdown", + "id": "1495b23f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m { \\mathbf{w}_{t+\\Delta t}-2 \\mathbf{w}_{t} +\\mathbf{w}_{t-\\Delta t} \\over (\\Delta t)^2}+\\mu {\\mathbf{w}_{t+\\Delta t}- \\mathbf{w}_{t} \\over \\Delta t} = -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "15bbc6e6", + "metadata": { + "editable": true + }, + "source": [ + "Rearranging this equation, we can rewrite this as" + ] + }, + { + "cell_type": "markdown", + "id": "ffeffc05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\mathbf{w}_{t +\\Delta t}= - { (\\Delta t)^2 \\over m +\\mu \\Delta t} \\nabla_w E(\\mathbf{w})+ {m \\over m +\\mu \\Delta t} \\Delta \\mathbf{w}_t.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "166cfafc", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum parameter\n", + "\n", + "Notice that this equation is identical to previous one if we identify\n", + "the position of the particle, $\\mathbf{w}$, with the parameters\n", + "$\\boldsymbol{\\theta}$. This allows us to identify the momentum\n", + "parameter and learning rate with the mass of the particle and the\n", + "viscous drag as:" + ] + }, + { + "cell_type": "markdown", + "id": "486fadf0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\gamma= {m \\over m +\\mu \\Delta t }, \\qquad \\eta = {(\\Delta t)^2 \\over m +\\mu \\Delta t}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cdb0769d", + "metadata": { + "editable": true + }, + "source": [ + "Thus, as the name suggests, the momentum parameter is proportional to\n", + "the mass of the particle and effectively provides inertia.\n", + "Furthermore, in the large viscosity/small learning rate limit, our\n", + "memory time scales as $(1-\\gamma)^{-1} \\approx m/(\\mu \\Delta t)$.\n", + "\n", + "Why is momentum useful? SGD momentum helps the gradient descent\n", + "algorithm gain speed in directions with persistent but small gradients\n", + "even in the presence of stochasticity, while suppressing oscillations\n", + "in high-curvature directions. This becomes especially important in\n", + "situations where the landscape is shallow and flat in some directions\n", + "and narrow and steep in others. It has been argued that first-order\n", + "methods (with appropriate initial conditions) can perform comparable\n", + "to more expensive second order methods, especially in the context of\n", + "complex deep learning models.\n", + "\n", + "These beneficial properties of momentum can sometimes become even more\n", + "pronounced by using a slight modification of the classical momentum\n", + "algorithm called Nesterov Accelerated Gradient (NAG).\n", + "\n", + "In the NAG algorithm, rather than calculating the gradient at the\n", + "current parameters, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t)$, one\n", + "calculates the gradient at the expected value of the parameters given\n", + "our current momentum, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma\n", + "\\mathbf{v}_{t-1})$. This yields the NAG update rule" + ] + }, + { + "cell_type": "markdown", + "id": "e65569bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma \\mathbf{v}_{t-1}) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d72d316", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t}.\n", + "\\label{_auto2} \\tag{2}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b7b9811e", + "metadata": { + "editable": true + }, + "source": [ + "One of the major advantages of NAG is that it allows for the use of a\n", + "larger learning rate than GDM for the same choice of $\\gamma$." + ] + }, + { + "cell_type": "markdown", + "id": "d8f3c523", + "metadata": { + "editable": true + }, + "source": [ + "## Second moment of the gradient\n", + "\n", + "In stochastic gradient descent, with and without momentum, we still\n", + "have to specify a schedule for tuning the learning rates $\\eta_t$\n", + "as a function of time. As discussed in the context of Newton's\n", + "method, this presents a number of dilemmas. The learning rate is\n", + "limited by the steepest direction which can change depending on the\n", + "current position in the landscape. To circumvent this problem, ideally\n", + "our algorithm would keep track of curvature and take large steps in\n", + "shallow, flat directions and small steps in steep, narrow directions.\n", + "Second-order methods accomplish this by calculating or approximating\n", + "the Hessian and normalizing the learning rate by the\n", + "curvature. However, this is very computationally expensive for\n", + "extremely large models. Ideally, we would like to be able to\n", + "adaptively change the step size to match the landscape without paying\n", + "the steep computational price of calculating or approximating\n", + "Hessians.\n", + "\n", + "During the last decade a number of methods have been introduced that accomplish\n", + "this by tracking not only the gradient, but also the second moment of\n", + "the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and\n", + "[ADAM](https://arxiv.org/abs/1412.6980)." + ] + }, + { + "cell_type": "markdown", + "id": "22d6be74", + "metadata": { + "editable": true + }, + "source": [ + "## Challenge: Choosing a Fixed Learning Rate\n", + "A fixed $\\eta$ is hard to get right:\n", + "1. If $\\eta$ is too large, the updates can overshoot the minimum, causing oscillations or divergence\n", + "\n", + "2. If $\\eta$ is too small, convergence is very slow (many iterations to make progress)\n", + "\n", + "In practice, one often uses trial-and-error or schedules (decaying $\\eta$ over time) to find a workable balance.\n", + "For a function with steep directions and flat directions, a single global $\\eta$ may be inappropriate:\n", + "1. Steep coordinates require a smaller step size to avoid oscillation.\n", + "\n", + "2. Flat/shallow coordinates could use a larger step to speed up progress.\n", + "\n", + "3. This issue is pronounced in high-dimensional problems with **sparse or varying-scale features** – we need a method to adjust step sizesper feature." + ] + }, + { + "cell_type": "markdown", + "id": "091ace5a", + "metadata": { + "editable": true + }, + "source": [ + "## Motivation for Adaptive Step Sizes\n", + "\n", + "1. Instead of a fixed global $\\eta$, use an \\textbf{adaptive learning rate} for each parameter that depends on the history of gradients.\n", + "\n", + "2. Parameters that have large accumulated gradient magnitude should get smaller steps (they've been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.\n", + "\n", + "3. This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected\n", + "\n", + "4. Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates\n", + "\n", + "5. Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive **AdaGrad**, one of the first adaptive methods." + ] + }, + { + "cell_type": "markdown", + "id": "5c05fb8b", + "metadata": { + "editable": true + }, + "source": [ + "## Derivation of the AdaGrad Algorithm\n", + "\n", + "**Accumulating Gradient History.**\n", + "\n", + "1. AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)\n", + "\n", + "2. Let $g_t = \\nabla \\ell_{i_t}(x_t)$ be the gradient at step $t$ (or a subgradient for nondifferentiable cases).\n", + "\n", + "3. Initialize $h_0 = 0$ (an all-zero vector in $\\mathbb{R}^d$).\n", + "\n", + "4. At each iteration $t$, update the accumulation:" + ] + }, + { + "cell_type": "markdown", + "id": "21fb8bd0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_t \\;=\\; h_{t-1} + g_t \\circ g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f71dfafa", + "metadata": { + "editable": true + }, + "source": [ + "1. Here $g_t \\circ g_t$ denotes element-wise square of the gradient vector. $h_t^{(j)} = h_{t-1}^{(j)} + (g_{t,j})^2$ for each parameter $j$.\n", + "\n", + "2. We can view $H_t = \\mathrm{diag}(h_t)$ as a diagonal matrix of past squared gradients. Initially $H_0 = 0$." + ] + }, + { + "cell_type": "markdown", + "id": "64869e2e", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Update Rule Derivation\n", + "\n", + "We scale the gradient by the inverse square root of the accumulated matrix $H_t$. The AdaGrad update at step $t$ is:" + ] + }, + { + "cell_type": "markdown", + "id": "c2d8d6b1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1} \\;=\\; x_t \\;-\\; \\eta\\, H_t^{-1/2}\\, g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2171198a", + "metadata": { + "editable": true + }, + "source": [ + "where $H_t^{-1/2}$ is the diagonal matrix with entries $(h_{t}^{(1)})^{-1/2}, \\dots, (h_{t}^{(d)})^{-1/2}$\n", + "In coordinates, this means each parameter $j$ has an individual step size:" + ] + }, + { + "cell_type": "markdown", + "id": "353e21d3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j} =x_{t,j} -\\frac{\\eta}{\\sqrt{h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b59e6add", + "metadata": { + "editable": true + }, + "source": [ + "In practice we add a small constant $\\epsilon$ in the denominator for numerical stability to avoid division by zero:" + ] + }, + { + "cell_type": "markdown", + "id": "707caefe", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j}= x_{t,j}-\\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "62dafce4", + "metadata": { + "editable": true + }, + "source": [ + "Equivalently, the effective learning rate for parameter $j$ at time $t$ is $\\displaystyle \\alpha_{t,j} = \\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}$. This decreases over time as $h_{t,j}$ grows." + ] + }, + { + "cell_type": "markdown", + "id": "01a2ba61", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Properties\n", + "1. AdaGrad automatically tunes the step size for each parameter. Parameters with more **volatile or large gradients** get smaller steps, and those with **small or infrequent gradients** get relatively larger steps\n", + "\n", + "2. No manual schedule needed: The accumulation $h_t$ keeps increasing (or stays the same if gradient is zero), so step sizes $\\eta/\\sqrt{h_t}$ are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.\n", + "\n", + "3. Sparse data benefit: For very sparse features, $h_{t,j}$ grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal\n", + "\n", + "4. Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate (e.g. $O(1/\\sqrt{T})$ regret bound) comparable to the best fixed learning rate tuned for the problem\n", + "\n", + "It effectively reduces the need to tune $\\eta$ by hand.\n", + "1. Limitations: Because $h_t$ accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)" + ] + }, + { + "cell_type": "markdown", + "id": "b4bc88e0", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp: Adaptive Learning Rates\n", + "Addresses AdaGrad’s diminishing learning rate issue.\n", + "Uses a decaying average of squared gradients (instead of a cumulative sum):" + ] + }, + { + "cell_type": "markdown", + "id": "4933fddb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "v_t = \\beta_2\\, v_{t-1} + (1-\\beta_2)\\, (\\nabla L(w_t))^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0dd8edf7", + "metadata": { + "editable": true + }, + "source": [ + "with $\\beta_2$ typically $0.9$ (or $0.99$).\n", + "1. Update: $w_{t+1} = w_t - \\frac{\\alpha}{\\sqrt{v_t + \\epsilon}} \\nabla L(w_t)$.\n", + "\n", + "2. Recent gradients have more weight, so $v_t$ adapts to the current landscape.\n", + "\n", + "3. Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.\n", + "\n", + "RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 - unpublished.)" + ] + }, + { + "cell_type": "markdown", + "id": "1ce98f28", + "metadata": { + "editable": true + }, + "source": [ + "## Adam Optimizer\n", + "\n", + "Why Combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.\n", + "\n", + "1. Fast convergence by smoothing gradients (accelerates in long-term gradient direction).\n", + "\n", + "2. Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).\n", + "\n", + "3. Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)\n", + "\n", + "4. Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)\n", + "\n", + "**Result**: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice." + ] + }, + { + "cell_type": "markdown", + "id": "d7d646bc", + "metadata": { + "editable": true + }, + "source": [ + "## RMS prop\n", + "\n", + "In RMS prop, in addition to keeping a running average of the first\n", + "moment of the gradient, we also keep track of the second moment\n", + "denoted by $\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}_t^2]$. The update rule\n", + "for RMS prop is given by" + ] + }, + { + "cell_type": "markdown", + "id": "3a212292", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto3} \\tag{3}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "88c0ec96", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta \\mathbf{s}_{t-1} +(1-\\beta)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9cfb794f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\mathbf{g}_t \\over \\sqrt{\\mathbf{s}_t +\\epsilon}}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "aaa4936c", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta$ controls the averaging time of the second moment and is\n", + "typically taken to be about $\\beta=0.9$, $\\eta_t$ is a learning rate\n", + "typically chosen to be $10^{-3}$, and $\\epsilon\\sim 10^{-8} $ is a\n", + "small regularization constant to prevent divergences. Multiplication\n", + "and division by vectors is understood as an element-wise operation. It\n", + "is clear from this formula that the learning rate is reduced in\n", + "directions where the norm of the gradient is consistently large. This\n", + "greatly speeds up the convergence by allowing us to use a larger\n", + "learning rate for flat directions." + ] + }, + { + "cell_type": "markdown", + "id": "5493a6cd", + "metadata": { + "editable": true + }, + "source": [ + "## [ADAM optimizer](https://arxiv.org/abs/1412.6980)\n", + "\n", + "A related algorithm is the ADAM optimizer. In\n", + "[ADAM](https://arxiv.org/abs/1412.6980), we keep a running average of\n", + "both the first and second moment of the gradient and use this\n", + "information to adaptively change the learning rate for different\n", + "parameters. The method isefficient when working with large\n", + "problems involving lots data and/or parameters. It is a combination of the\n", + "gradient descent with momentum algorithm and the RMSprop algorithm\n", + "discussed above.\n", + "\n", + "In addition to keeping a running average of the first and\n", + "second moments of the gradient\n", + "(i.e. $\\mathbf{m}_t=\\mathbb{E}[\\mathbf{g}_t]$ and\n", + "$\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}^2_t]$, respectively), ADAM\n", + "performs an additional bias correction to account for the fact that we\n", + "are estimating the first two moments of the gradient using a running\n", + "average (denoted by the hats in the update rule below). The update\n", + "rule for ADAM is given by (where multiplication and division are once\n", + "again understood to be element-wise operations below)" + ] + }, + { + "cell_type": "markdown", + "id": "35795070", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto4} \\tag{4}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1e67e6dd", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{m}_t = \\beta_1 \\mathbf{m}_{t-1} + (1-\\beta_1) \\mathbf{g}_t \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39d582bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta_2 \\mathbf{s}_{t-1} +(1-\\beta_2)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a7e2423", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{m}}_t={\\mathbf{m}_t \\over 1-\\beta_1^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "53da77c2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{s}}_t ={\\mathbf{s}_t \\over1-\\beta_2^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a4c7df45", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\boldsymbol{\\mathbf{m}}_t \\over \\sqrt{\\boldsymbol{\\mathbf{s}}_t} +\\epsilon}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f645a0aa", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\label{_auto5} \\tag{5}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "505619bb", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta_1$ and $\\beta_2$ set the memory lifetime of the first and\n", + "second moment and are typically taken to be $0.9$ and $0.99$\n", + "respectively, and $\\eta$ and $\\epsilon$ are identical to RMSprop.\n", + "\n", + "Like in RMSprop, the effective step size of a parameter depends on the\n", + "magnitude of its gradient squared. To understand this better, let us\n", + "rewrite this expression in terms of the variance\n", + "$\\boldsymbol{\\sigma}_t^2 = \\boldsymbol{\\mathbf{s}}_t -\n", + "(\\boldsymbol{\\mathbf{m}}_t)^2$. Consider a single parameter $\\theta_t$. The\n", + "update rule for this parameter is given by" + ] + }, + { + "cell_type": "markdown", + "id": "6cb4c9a9", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\theta_{t+1}= -\\eta_t { \\boldsymbol{m}_t \\over \\sqrt{\\sigma_t^2 + m_t^2 }+\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39627950", + "metadata": { + "editable": true + }, + "source": [ + "## Algorithms and codes for Adagrad, RMSprop and Adam\n", + "\n", + "The algorithms we have implemented are well described in the text by [Goodfellow, Bengio and Courville, chapter 8](https://www.deeplearningbook.org/contents/optimization.html).\n", + "\n", + "The codes which implement these algorithms are discussed after our presentation of automatic differentiation." + ] + }, + { + "cell_type": "markdown", + "id": "f240057d", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

Figure 1:

\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "428ee09d", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

Figure 1:

\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "9ce3cc3e", + "metadata": { + "editable": true + }, + "source": [ + "## ADAM algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

Figure 1:

\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "193b3778", + "metadata": { + "editable": true + }, + "source": [ + "## Practical tips\n", + "\n", + "* **Randomize the data when making mini-batches**. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.\n", + "\n", + "* **Transform your inputs**. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.\n", + "\n", + "* **Monitor the out-of-sample performance.** Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This *early stopping* significantly improves performance in many settings.\n", + "\n", + "* **Adaptive optimization methods don't always have good generalization.** Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications." + ] + }, + { + "cell_type": "markdown", + "id": "8d7187f5", + "metadata": { + "editable": true + }, + "source": [ + "## Sneaking in automatic differentiation using Autograd\n", + "\n", + "In the examples here we take the liberty of sneaking in automatic\n", + "differentiation (without having discussed the mathematics). In\n", + "project 1 you will write the gradients as discussed above, that is\n", + "hard-coding the gradients. By introducing automatic differentiation\n", + "via the library **autograd**, which is now replaced by **JAX**, we have\n", + "more flexibility in setting up alternative cost functions.\n", + "\n", + "The\n", + "first example shows results with ordinary leats squares." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "68d35b76", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f39b471c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "265059ef", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x#+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 30\n", + "\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "# Now improve with momentum gradient descent\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "for iter in range(Niterations):\n", + " # calculate gradient\n", + " gradients = training_gradient(theta)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd wth momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "18183fb0", + "metadata": { + "editable": true + }, + "source": [ + "## Including Stochastic Gradient Descent with Autograd\n", + "\n", + "In this code we include the stochastic gradient descent approach\n", + "discussed above. Note here that we specify which argument we are\n", + "taking the derivative with respect to when using **autograd**." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4d9d93b3", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "1652a471", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6092712c", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "\n", + "for epoch in range(n_epochs):\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + "print(\"theta from own sdg with momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "4d2f230d", + "metadata": { + "editable": true + }, + "source": [ + "## But none of these can compete with Newton's method\n", + "\n", + "Note that we here have introduced automatic differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b46cedcc", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Newton's method\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+5*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(beta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "# Note that here the Hessian does not depend on the parameters beta\n", + "invH = np.linalg.pinv(H)\n", + "beta = np.random.randn(3,1)\n", + "Niterations = 5\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(beta)\n", + " beta -= invH @ gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"beta from own Newton code\")\n", + "print(beta)" + ] + }, + { + "cell_type": "markdown", + "id": "f048d5c7", + "metadata": { + "editable": true + }, + "source": [ + "## Similar (second order function now) problem but now with AdaGrad" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d8bdd180", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " Giter += gradients*gradients\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + " theta -= update\n", + "print(\"theta from own AdaGrad\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "145ce394", + "metadata": { + "editable": true + }, + "source": [ + "Running this code we note an almost perfect agreement with the results from matrix inversion." + ] + }, + { + "cell_type": "markdown", + "id": "ec0ee110", + "metadata": { + "editable": true + }, + "source": [ + "## RMSprop for adaptive learning rate with Stochastic Gradient Descent" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b6e8e3e4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameter rho\n", + "rho = 0.99\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + "\t# Accumulated gradient\n", + "\t# Scaling with rho the new and the previous results\n", + " Giter = (rho*Giter+(1-rho)*gradients*gradients)\n", + "\t# Taking the diagonal only and inverting\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + "\t# Hadamard product\n", + " theta -= update\n", + "print(\"theta from own RMSprop\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "79d8c0ae", + "metadata": { + "editable": true + }, + "source": [ + "## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c16ec5ac", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980\n", + "beta1 = 0.9\n", + "beta2 = 0.999\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-7\n", + "iter = 0\n", + "for epoch in range(n_epochs):\n", + " first_moment = 0.0\n", + " second_moment = 0.0\n", + " iter += 1\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " # Computing moments first\n", + " first_moment = beta1*first_moment + (1-beta1)*gradients\n", + " second_moment = beta2*second_moment+(1-beta2)*gradients*gradients\n", + " first_term = first_moment/(1.0-beta1**iter)\n", + " second_term = second_moment/(1.0-beta2**iter)\n", + "\t# Scaling with rho the new and the previous results\n", + " update = eta*first_term/(np.sqrt(second_term)+delta)\n", + " theta -= update\n", + "print(\"theta from own ADAM\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "7c581f8b", + "metadata": { + "editable": true + }, + "source": [ + "## Material for the lab sessions\n", + "\n", + "1. Exercise set for week 37 and reminder on scaling (from lab sessions of week 35)\n", + "\n", + "2. Work on project 1\n", + "\n", + "\n", + " * For more discussions of Ridge regression and calculation of averages, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended." + ] + }, + { + "cell_type": "markdown", + "id": "d8a82e01", + "metadata": { + "editable": true + }, + "source": [ + "## Reminder on different scaling methods\n", + "\n", + "Before fitting a regression model, it is good practice to normalize or\n", + "standardize the features. This ensures all features are on a\n", + "comparable scale, which is especially important when using\n", + "regularization. In the exercises this week we will perform standardization, scaling each\n", + "feature to have mean 0 and standard deviation 1.\n", + "\n", + "Here we compute the mean and standard deviation of each column (feature) in our design/feature matrix $\\boldsymbol{X}$.\n", + "Then we subtract the mean and divide by the standard deviation for each feature.\n", + "\n", + "In the example here we\n", + "we will also center the target $\\boldsymbol{y}$ to mean $0$. Centering $\\boldsymbol{y}$\n", + "(and each feature) means the model does not require a separate intercept\n", + "term, the data is shifted such that the intercept is effectively 0\n", + ". (In practice, one could include an intercept in the model and not\n", + "penalize it, but here we simplify by centering.)\n", + "Choose $n=100$ data points and set up $\\boldsymbol{x}, $\\boldsymbol{y}$ and the design matrix $\\boldsymbol{X}$." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "3fa174a9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Standardize features (zero mean, unit variance for each feature)\n", + "X_mean = X.mean(axis=0)\n", + "X_std = X.std(axis=0)\n", + "X_std[X_std == 0] = 1 # safeguard to avoid division by zero for constant features\n", + "X_norm = (X - X_mean) / X_std\n", + "\n", + "# Center the target to zero mean (optional, to simplify intercept handling)\n", + "y_mean = ?\n", + "y_centered = ?" + ] + }, + { + "cell_type": "markdown", + "id": "b02de7a6", + "metadata": { + "editable": true + }, + "source": [ + "Do we need to center the values of $y$?\n", + "\n", + "After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n", + "and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This can make the optimization landscape\n", + "nicer and ensures the regularization penalty $\\lambda \\sum_j\n", + "\\theta_j^2$ in Ridge regression treats each coefficient fairly (since features are on the\n", + "same scale)." + ] + }, + { + "cell_type": "markdown", + "id": "cd3f85ef", + "metadata": { + "editable": true + }, + "source": [ + "## Functionality in Scikit-Learn\n", + "\n", + "**Scikit-Learn** has several functions which allow us to rescale the\n", + "data, normally resulting in much better results in terms of various\n", + "accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n", + "ensures that for each feature/predictor we study the mean value is\n", + "zero and the variance is one (every column in the design/feature\n", + "matrix). This scaling has the drawback that it does not ensure that\n", + "we have a particular maximum or minimum in our data set. Another\n", + "function included in **Scikit-Learn** is the **MinMaxScaler** which\n", + "ensures that all features are exactly between $0$ and $1$. The" + ] + }, + { + "cell_type": "markdown", + "id": "1e0198a0", + "metadata": { + "editable": true + }, + "source": [ + "## More preprocessing\n", + "\n", + "The **Normalizer** scales each data\n", + "point such that the feature vector has a euclidean length of one. In other words, it\n", + "projects a data point on the circle (or sphere in the case of higher dimensions) with a\n", + "radius of 1. This means every data point is scaled by a different number (by the\n", + "inverse of it’s length).\n", + "This normalization is often used when only the direction (or angle) of the data matters,\n", + "not the length of the feature vector.\n", + "\n", + "The **RobustScaler** works similarly to the StandardScaler in that it\n", + "ensures statistical properties for each feature that guarantee that\n", + "they are on the same scale. However, the RobustScaler uses the median\n", + "and quartiles, instead of mean and variance. This makes the\n", + "RobustScaler ignore data points that are very different from the rest\n", + "(like measurement errors). These odd data points are also called\n", + "outliers, and might often lead to trouble for other scaling\n", + "techniques." + ] + }, + { + "cell_type": "markdown", + "id": "04da94ca", + "metadata": { + "editable": true + }, + "source": [ + "## Frequently used scaling functions\n", + "\n", + "Many features are often scaled using standardization to improve performance. In **Scikit-Learn** this is given by the **StandardScaler** function as discussed above. It is easy however to write your own. \n", + "Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:" + ] + }, + { + "cell_type": "markdown", + "id": "aa1ad257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6587016d", + "metadata": { + "editable": true + }, + "source": [ + "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard deviation, respectively, of the feature $x_j$.\n", + "This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2e0350b4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "8179b037", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "3ebed257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta_0, \\theta_1, ... , \\theta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij}\\theta_j\\right)^2,.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "51b239a4", + "metadata": { + "editable": true + }, + "source": [ + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", + "\n", + "What we have done is to single out the $\\theta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\theta_0$, we want the derivative to obey" + ] + }, + { + "cell_type": "markdown", + "id": "295790f3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_j} = 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b68338d6", + "metadata": { + "editable": true + }, + "source": [ + "for all $j$. For $\\theta_0$ we have" + ] + }, + { + "cell_type": "markdown", + "id": "c015cb3b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij} \\theta_j\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5e4917fa", + "metadata": { + "editable": true + }, + "source": [ + "Multiplying away the constant $2/n$, we obtain" + ] + }, + { + "cell_type": "markdown", + "id": "150fedf5", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\sum_{i=0}^{n-1} \\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6d782b1", + "metadata": { + "editable": true + }, + "source": [ + "Let us specialize first to the case where we have only two parameters $\\theta_0$ and $\\theta_1$.\n", + "Our result for $\\theta_0$ simplifies then to" + ] + }, + { + "cell_type": "markdown", + "id": "a4065dd3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "n\\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\theta_1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "06324d79", + "metadata": { + "editable": true + }, + "source": [ + "We obtain then" + ] + }, + { + "cell_type": "markdown", + "id": "9e3e6b35", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\theta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cd9c3754", + "metadata": { + "editable": true + }, + "source": [ + "If we define" + ] + }, + { + "cell_type": "markdown", + "id": "8361829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8f5f46fd", + "metadata": { + "editable": true + }, + "source": [ + "and the mean value of the outputs as" + ] + }, + { + "cell_type": "markdown", + "id": "cf210a6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d3af09fc", + "metadata": { + "editable": true + }, + "source": [ + "we have" + ] + }, + { + "cell_type": "markdown", + "id": "ec91f8c1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\mu_y - \\theta_1\\mu_{\\boldsymbol{x}_1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0c9f35e6", + "metadata": { + "editable": true + }, + "source": [ + "In the general case with more parameters than $\\theta_0$ and $\\theta_1$, we have" + ] + }, + { + "cell_type": "markdown", + "id": "de80fa4b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "34c99b80", + "metadata": { + "editable": true + }, + "source": [ + "We can rewrite the latter equation as" + ] + }, + { + "cell_type": "markdown", + "id": "758ade4d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\theta_j,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff0f4725", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined" + ] + }, + { + "cell_type": "markdown", + "id": "997b829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "621e7d85", + "metadata": { + "editable": true + }, + "source": [ + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", + "\n", + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" + ] + }, + { + "cell_type": "markdown", + "id": "5530e459", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\theta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "fd9bace8", + "metadata": { + "editable": true + }, + "source": [ + "If we minimize with respect to $\\boldsymbol{\\theta}$ we have then" + ] + }, + { + "cell_type": "markdown", + "id": "d949be75", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ad52617", + "metadata": { + "editable": true + }, + "source": [ + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\theta}^T\\boldsymbol{\\theta}$ to the cost function and get then" + ] + }, + { + "cell_type": "markdown", + "id": "41cabc72", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1fd416a6", + "metadata": { + "editable": true + }, + "source": [ + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f13cacb4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_theta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_theta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_theta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "theta = fit_theta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True theta: {true_theta}\")\n", + "print(f\"Fitted theta: {theta}\")\n", + "print(f\"Sklearn fitted theta: {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ theta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "theta = fit_theta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ theta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted theta (without intercept): {theta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted theta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ theta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ff4b28ed", + "metadata": { + "editable": true + }, + "source": [ + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", + "\n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\theta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", + "\n", + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" + ] + }, + { + "cell_type": "markdown", + "id": "03b4dc3c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\theta_j^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6cff2148", + "metadata": { + "editable": true + }, + "source": [ + "but when we take out the intercept, this equation becomes" + ] + }, + { + "cell_type": "markdown", + "id": "51d823c4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\theta_j^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5ab270f9", + "metadata": { + "editable": true + }, + "source": [ + "For Lasso regression we have" + ] + }, + { + "cell_type": "markdown", + "id": "2b2e1492", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\theta_j\\vert.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "063be947", + "metadata": { + "editable": true + }, + "source": [ + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", + "\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "5e841a10", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeTheta\n", + " ypredictOwnRidge = X_test @ OwnRidgeTheta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta)\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "40e54083", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "edd7dd42", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", + "\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", + "\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeTheta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeTheta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta) #Intercept is given by mean of target variable\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f9cc046b", + "metadata": { + "editable": true + }, + "source": [ + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\theta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/LectureNotes/_build/html/chapter1.html b/doc/LectureNotes/_build/html/chapter1.html index c323aa723..40f74179e 100644 --- a/doc/LectureNotes/_build/html/chapter1.html +++ b/doc/LectureNotes/_build/html/chapter1.html @@ -230,6 +230,8 @@
  • Week 36: Linear Regression and Gradient descent
  • Exercises week 37
    • +
  • Week 37: Statistical interpretations and Resampling Methods
    • +
  • Exercises week 37

    • Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

      @@ -470,7 +472,7 @@ y_centered = ? -

      Fill in the necessary details.

      +

      Fill in the necessary details. Do we need to center the \(y\)-values?

      After this preprocessing, each column of \(\boldsymbol{X}_{\mathrm{norm}}\) has mean zero and standard deviation \(1\) and \(\boldsymbol{y}_{\mathrm{centered}}\) has mean 0. This makes the optimization landscape nicer and ensures the regularization penalty \(\lambda \sum_j @@ -487,7 +489,9 @@ same scale).

      # Set regularization parameter, either a single value or a vector of values
-lambda = ?
+# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called "anonymous functions" or "lambda functions."
+lam = ?
+
 
 # Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y
 I = np.eye(n_features)
@@ -533,19 +537,8 @@ num_iters = 1000
 # Initialize weights for gradient descent
 theta = np.zeros(n_features)
 
-# Arrays to store history for plotting
-cost_history = np.zeros(num_iters)
-
 # Gradient descent loop
-m = n_samples  # number of data points
 for t in range(num_iters):
-    # Compute prediction error
-    error = X_norm.dot(theta) - y_centered 
-    # Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring
-    cost_OLS = ?
-    cost_Ridge = ?
-    # You could add a history for both methods (optional)
-    cost_history[t] = ?
     # Compute gradients for OSL and Ridge
     grad_OLS = ?
     grad_Ridge = ?
@@ -564,11 +557,13 @@ print("Gradient Descent Ridge coefficients:", theta_gdRidge)

      4a)#

      -

      Discuss the results as function of the learning rate parameters and the number of iterations.

      +

      Write first a gradient descent code for OLS only using the above template. +Discuss the results as function of the learning rate parameters and the number of iterations

      4b)#

      -

      Try to add a stopping parameter as function of the number iterations and the difference between the new and old \(\theta\) values. How would you define a stopping criterion?

      +

      Write then a similar code for Ridge regression using the above template. +Try to add a stopping parameter as function of the number iterations and the difference between the new and old \(\theta\) values. How would you define a stopping criterion?

      @@ -667,11 +662,11 @@ should be in the same ballpark. Which method (OLS or Ridge) gives the best resu

      next

      -

      Project 1 on Machine Learning, deadline October 6 (midnight), 2025

      +

      Week 37: Statistical interpretations and Resampling Methods

       diff --git a/doc/LectureNotes/_build/html/exercisesweek38.html b/doc/LectureNotes/_build/html/exercisesweek38.html new file mode 100644 index 000000000..ce95a5e50 --- /dev/null +++ b/doc/LectureNotes/_build/html/exercisesweek38.html @@ -0,0 +1,690 @@ + + + + + + + + + + + Exercises week 37 — Applied Data Analysis and Machine Learning + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + + + + + + + + + +
      +
      +
      +
      + + + Ctrl+K +
      +
      + +
      + +
      + + +
      +
      + + +
      +
      + + + +
      + + + +
      + + + + +
      + +
      +
      + + + + + +
      +
      + +
      +
      +
      + + +
      +
      + + + + +
      + +
      + + + +
      + +
      +
      + +
      +
      + +
      + +
      + +
      + + +
      + +
      + +
      + + + + + +
      + + +
      + + + + + + + + + + + + + +
      + +
      + +
      +
      + + + +
      +

      Exercises week 37

      + +
      + +
      +
      + + + + +
      + +
      +

      Exercises week 37#

      +
      +

      September 15-19#

      +
      +
      +

      Resampling and the Bias-Variance Trade-off#

      +
      +
      +

      Learning goals#

      +

      After completing these exercises, you will know how to

      +
        +

      • Derive expectation and variances values related to linear regression

        • +

      • Compute expectation and variances values related to linear regression

        • +

      • Compute and evaluate the trade-off between bias and variance of a model

        • +
      +

      This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer). The exercises are also a part of project 1 and can be reused in the theory part of the project.

      +

      For more discussions on Ridge regression and calculation of expectation values, Wessel van Wieringen’s article is highly recommended.

      +

      We assume that there exists a continuous function \(f(\boldsymbol{x})\) and a normal distributed error \(\boldsymbol{\varepsilon}\sim N(0, \sigma^2)\) which describes our data

      +
      +\[ +\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon} +\]
      +

      We further assume that this continous function can be modeled with a linear model \(\mathbf{\tilde{y}}\) of some features \(\mathbf{X}\).

      +
      +\[ +\boldsymbol{y} = \boldsymbol{\tilde{y}} + \boldsymbol{\varepsilon} = \boldsymbol{X}\boldsymbol{\beta} +\boldsymbol{\varepsilon} +\]
      +

      We therefore get that our data \(\boldsymbol{y}\) has an expectation value \(\boldsymbol{X}\boldsymbol{\beta}\) and variance \(\sigma^2\), that is \(\boldsymbol{y}\) follows a normal distribution with mean value \(\boldsymbol{X}\boldsymbol{\beta}\) and variance \(\sigma^2\).

      +
      +
      +

      Exercise 1: Expectation values for ordinary least squares expressions#

      +

      a) With the expressions for the optimal parameters \(\boldsymbol{\hat{\beta}_{OLS}}\) show that

      +
      +\[ +\mathbb{E}(\boldsymbol{\hat{\beta}_{OLS}}) = \boldsymbol{\beta}. +\]
      +

      b) Show that the variance of \(\boldsymbol{\hat{\beta}_{OLS}}\) is

      +
      +\[ +\mathbf{Var}(\boldsymbol{\hat{\beta}_{OLS}}) = \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}. +\]
      +

      We can use the last expression when we define a confidence interval for the parameters \(\boldsymbol{\hat{\beta}_{OLS}}\). +A given parameter \({\boldsymbol{\hat{\beta}_{OLS}}}_j\) is given by the diagonal matrix element of the above matrix.

      +
      +
      +

      Exercise 2: Expectation values for Ridge regression#

      +

      a) With the expressions for the optimal parameters \(\boldsymbol{\hat{\beta}_{Ridge}}\) show that

      +
      +\[ +\mathbb{E} \big[ \hat{\boldsymbol{\beta}}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\boldsymbol{\beta} +\]
      +

      We see that \(\mathbb{E} \big[ \hat{\boldsymbol{\beta}}^{\mathrm{Ridge}} \big] \not= \mathbb{E} \big[\hat{\boldsymbol{\beta}}^{\mathrm{OLS}}\big ]\) for any \(\lambda > 0\).

      +

      b) Why do we say that Ridge regression gives a biased estimate? Is this a problem?

      +

      c) Show that the variance is

      +
      +\[ +\mathbf{Var}[\hat{\boldsymbol{\beta}}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T} +\]
      +

      We see that if the parameter \(\lambda\) goes to infinity then the variance of the Ridge parameters \(\boldsymbol{\beta}\) goes to zero.

      +
      +
      +

      Exercise 3: Deriving the expression for the Bias-Variance Trade-off#

      +

      The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.

      +

      The parameters \(\boldsymbol{\hat{\beta}_{OLS}}\) are found by optimizing the mean squared error via the so-called cost function

      +
      +\[ +C(\boldsymbol{X},\boldsymbol{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right] +\]
      +

      a) Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a +term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function \(f\) with the model \({\bf y}\). +show that

      +
      +\[ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, +\]
      +

      with

      +
      +\[ +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], +\]
      +

      and

      +
      +\[ +\mathrm{var}[\tilde{y}]=\mathbb{E}\left[\left(\tilde{\boldsymbol{y}}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2. +\]
      +

      b) Explain what the terms mean and discuss their interpretations.

      +
      +
      +

      Exercise 4: Computing the Bias and Variance#

      +

      Before you compute the bias and variance of a real model for different complexities, let’s for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.

      +

      a) Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error.

      +
      +
      +
      import numpy as np
+
+n = 100
+bootstraps = 1000
+
+predictions = np.random.rand(bootstraps, n) * 10 + 10
+targets = np.random.rand(bootstraps, n)
+
+mse = ...
+bias = ...
+variance = ...
+
      +
      +
      +
      +

      b) Change the prediction values in some way to increase the bias while decreasing the variance.

      +

      c) Change the prediction values in some way to increase the variance while decreasing the bias.

      +

      d) Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model.

      +
      +
      +
      import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import PolynomialFeatures # use the fit_transform method of the created object!
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import train_test_split
+from sklearn.utils import resample
+
      +
      +
      +
      +
      +
      +
      n = 100
+bootstraps = 1000
+
+x = np.linspace(-3, 3, n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1)
+
+biases = []
+variances = []
+mses = []
+
+#for p in range(1, 5):
+#    predictions = ...
+#    targets = ...
+#    for b in range(bootstraps):
+#        x_sample, y_sample = ...
+#        X = ...
+#        X_train, X_test, y_train, y_test = ...
+#
+#        predictions[b, :] = 
+#        targets[b, :] = 
+#        
+#    biases.append(...)
+#    variances.append(...)
+#    mses.append(...)
+
      +
      +
      +
      +

      e) Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).

      +

      f) Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting).

      +
      +
      + + + + +
      + + + + + + + + +
      + + + +
      + + +
      +
      + Contents +
      +
      + +
      + + +
      +
      + +
      + +
      + +

      +By Morten Hjorth-Jensen +

      + +
      + +
      + + +

      + + © Copyright 2023. +
      + +

      + +
      + +
      + +
      + +
      + +
      + +
      +
      + + +
      +
      +
      + + + + + +
      +
      + + \ No newline at end of file diff --git a/doc/LectureNotes/_build/html/genindex.html b/doc/LectureNotes/_build/html/genindex.html index 5583cc411..f18de25c1 100644 --- a/doc/LectureNotes/_build/html/genindex.html +++ b/doc/LectureNotes/_build/html/genindex.html @@ -227,6 +227,8 @@
    • Exercises week 36
    • Week 36: Linear Regression and Gradient descent
    • Exercises week 37
      • +
    • Week 37: Statistical interpretations and Resampling Methods
      • +
    • Exercises week 37

    Projects

    Projects

    Projects

      diff --git a/doc/LectureNotes/_build/html/objects.inv b/doc/LectureNotes/_build/html/objects.inv index 26f7645df8b70df815a8c193558a8dc008fc31b8..ba834f2e093015eabde694e18afe8c1d72d8a0d9 100644 GIT binary patch delta 912 zcmV;B18@B02Zjicgnvs@+b|Hv@BS2fEHi0`mlMb>Ezrk6lL6YE6??H&Y{`}6g!I#Q zWy!{N6nX-$y}w=U0W&?cRyGJ*Feka*jN+>`~}a)C_2J!T5E=2ArM`{R() zvGe*^^CIOU0|qyUDcnf4Wh#a5SSwc0AG4L}xkClx`ve|jv463oEB^F^3WebWxc*Z! zL&<1>EjKw_Ay#lpi3<9(!=8q)rw;o`ij6F43uzMS=cPNsS%`4v2=|!O$|eNDc?<77 zhvfzpQ%P>~-f-Nuyih_upL?EW2u!4K%e0n(f`@YHQqioe3LEYYO)k^!1m^*#bKhB+ zacy`K2o(psVt?O`x~ily@Dmdw72GjLsOY%j0|5?GdmdqgK;?l~GdEn$XtT z57=4#QoVjwof!+(V7h%h@$n1uLn>&BFO6%#)>v5Q*6O^z+J7TX-Io0bjlq4f zs7N`eL08PM*Acb-8xKT83PUr|=QzvYs#p{mt`(cW7h*J;MDw*!itQY=l#144N4q8H za$u$=@rg>A298qg-nTDYh&!3qw4Clxm6E?ng?|Q@M{9GRuiO)y0L2=gW&*UYHbI0X zY1tYTOyOP0Q^B*`98KW#_%x1sDf}llf22W*I?1tl?4{V++Iy&0n%<+aIuvQN0yvE1 zY+iod`xVPD(uNk=D!3ltJF6d?f`m?(MwqA`YqsVEHzbBRO0{b`^3b}FAPlA;GDt;3 zHGc@u+8q9|{lbDiD`szfa(gjrDW6Av-q5Ysv z)thv8nM-4(Skjz^8;+hI@A2b4KlT^LQqchz)n)H@j}7_8CA}{sef@Ajrt<){tDC(c mQq)$NqkU27Yl)-H=7Yx3Mx&|}8vnz{atQ;!3;qG=xrH|z;l$Md delta 878 zcmV-!1CjiO2;~Qmgnvy_+b|G@@BS5ggqhF*f)mIs5a`E1lL6YE6??H&Y{`}6g!I>U zWy!`4f}X%D@3X7@c%39?l|gj^+N85oCUA5DiMM^qOS%vP%B4i${=6L^rt#($C){OJi53d0F-{ikMz zlFnj@yy!`-3DW!l}~EZ}tJJ1aA; z4Nn50;(%A|+ka74l~e|PVq&C%J7#Z*dIYj}XU5AegGqojajXR<+=d0C@FT=DfS5YO zwa{3u3vBN_gBvAF8oLLAkol3wU?k-!cFlusYJB}dAxnfyDX9b^8<9U6Fge1vx^a|G z`=vY&WO44tooT8gPmyWsXfZ3rMZzm)C#|fcuyZn}QGeTMuHj&HH9u^*7Xj{t<9;6w zR_h@v!%P0R*?@p^|JZSsJ$W)AtF}Fku>HNaaZJrEx9T8Vl>(TAkNd`)|alSF#_WF}OG8741EhcfkyM z9WYz{{dPvA57f~<$5{qKKQA&|D>i~J#ApAyodwu1e;C}=t*8XiqK-*>$L|Bp%tx>@Q4oaR1 zp5^9n1gFQRanwuUKe_oU4N}xej?H5)#n#q-MQWw#9F0|xNTZdnVI*hs;_1#;EW=0} zT1TtDdVue&eryVoDPbB}p?a*@nit%Vl;tSZuIb1_>q6^1n1aY46%ExOKx=dO$My{i z`e&?|{brNfi&;zgyyxeQt)qX8-fnS!d4IW;ol!+;WYaonG>Zhe5uv7fqwY)O(pV{$ zG`Hb~qvywO_;H^fd)Z$qx&WiPy!q8)Lw@jyeJca~?RP;YvjDcMo7MLewbj+=9}YuK EK>&HNZvX%Q diff --git a/doc/LectureNotes/_build/html/project1.html b/doc/LectureNotes/_build/html/project1.html index 44d08f6bb..62f675674 100644 --- a/doc/LectureNotes/_build/html/project1.html +++ b/doc/LectureNotes/_build/html/project1.html @@ -62,7 +62,7 @@ - + @@ -229,6 +229,8 @@
    • Exercises week 36
    • Week 36: Linear Regression and Gradient descent
    • Exercises week 37
      • +
    • Week 37: Statistical interpretations and Resampling Methods
      • +
    • Exercises week 37

    Projects

    Projects

    Projects

      diff --git a/doc/LectureNotes/_build/html/searchindex.js b/doc/LectureNotes/_build/html/searchindex.js index 4c7aa149b..58a74b5a1 100644 --- a/doc/LectureNotes/_build/html/searchindex.js +++ b/doc/LectureNotes/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"1a)": [[18, "a"]], "3a)": [[18, "id1"]], "3b)": [[18, "b"]], "4a)": [[18, "id2"]], "4b)": [[18, "id3"]], "A Classification Tree": [[9, "a-classification-tree"]], "A Frequentist approach to data analysis": [[0, "a-frequentist-approach-to-data-analysis"], [26, "a-frequentist-approach-to-data-analysis"]], "A better approach": [[8, "a-better-approach"]], "A first summary": [[26, "a-first-summary"]], "A quick Reminder on Lagrangian Multipliers": [[8, "a-quick-reminder-on-lagrangian-multipliers"]], "A simple example": [[4, "a-simple-example"]], "A soft classifier": [[8, "a-soft-classifier"]], "A top-down perspective on Neural networks": [[1, "a-top-down-perspective-on-neural-networks"]], "ADAM optimizer": [[13, "adam-optimizer"]], "Activation functions": [[12, "activation-functions"]], "Adaptive boosting: AdaBoost, Basic Algorithm": [[10, "adaptive-boosting-adaboost-basic-algorithm"]], "Adding error analysis and training set up": [[26, "adding-error-analysis-and-training-set-up"], [27, "adding-error-analysis-and-training-set-up"]], "Adjust hyperparameters": [[1, "adjust-hyperparameters"]], "Algorithms for Setting up Decision Trees": [[9, "algorithms-for-setting-up-decision-trees"]], "An Overview of Ensemble Methods": [[10, "an-overview-of-ensemble-methods"]], "An extrapolation example": [[4, "an-extrapolation-example"]], "An optimization/minimization problem": [[26, "an-optimization-minimization-problem"]], "And finally \\boldsymbol{X}\\boldsymbol{X}^T": [[27, "and-finally-boldsymbol-x-boldsymbol-x-t"]], "And what about using neural networks?": [[26, "and-what-about-using-neural-networks"]], "Another Example, now with a polynomial fit": [[28, "another-example-now-with-a-polynomial-fit"]], "Another example, the moons again": [[9, "another-example-the-moons-again"]], "Applied Data Analysis and Machine Learning": [[19, null]], "Autocorrelation function": [[23, "autocorrelation-function"]], "Automatic differentiation": [[13, "automatic-differentiation"]], "Back to Ridge and LASSO Regression": [[27, "back-to-ridge-and-lasso-regression"], [28, "back-to-ridge-and-lasso-regression"]], "Back to the Cancer Data": [[11, "back-to-the-cancer-data"]], "Background literature": [[21, "background-literature"]], "Bagging": [[10, "bagging"]], "Bagging Examples": [[10, "bagging-examples"]], "Basic Matrix Features": [[20, "basic-matrix-features"]], "Basic ideas of the Principal Component Analysis (PCA)": [[11, null]], "Basic math of the SVD": [[5, "basic-math-of-the-svd"], [27, "basic-math-of-the-svd"], [28, "basic-math-of-the-svd"]], "Basics": [[7, "basics"]], "Basics of a tree": [[9, "basics-of-a-tree"]], "Batch Normalization": [[1, "batch-normalization"]], "Bayes\u2019 Theorem and Ridge and Lasso Regression": [[5, "bayes-theorem-and-ridge-and-lasso-regression"]], "Boosting, a Bird\u2019s Eye View": [[10, "boosting-a-bird-s-eye-view"]], "Bootstrap": [[6, "bootstrap"]], "Bringing it together, first back propagation equation": [[12, "bringing-it-together-first-back-propagation-equation"]], "Building a Feed Forward Neural Network": [[1, null]], "Building a tree, regression": [[9, "building-a-tree-regression"]], "Building neural networks in Tensorflow and Keras": [[1, "building-neural-networks-in-tensorflow-and-keras"]], "CNNs in more detail, building convolutional neural networks in Tensorflow and Keras": [[3, "cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras"]], "Cancer Data again now with Decision Trees and other Methods": [[9, "cancer-data-again-now-with-decision-trees-and-other-methods"]], "Choose cost function and optimizer": [[1, "choose-cost-function-and-optimizer"]], "Classical PCA Theorem": [[11, "classical-pca-theorem"]], "Clustering and Unsupervised Learning": [[14, null]], "Code for SVD and Inversion of Matrices": [[5, "code-for-svd-and-inversion-of-matrices"]], "Codes and Approaches": [[14, "codes-and-approaches"]], "Codes for the SVD": [[5, "codes-for-the-svd"], [27, "codes-for-the-svd"], [28, "codes-for-the-svd"]], "Coding Setup and Linear Regression": [[15, "coding-setup-and-linear-regression"]], "Collect and pre-process data": [[1, "collect-and-pre-process-data"]], "Communication channels": [[26, "communication-channels"]], "Compare Bagging on Trees with Random Forests": [[10, "compare-bagging-on-trees-with-random-forests"]], "Comparing with a numerical scheme": [[2, "comparing-with-a-numerical-scheme"]], "Comparison with OLS": [[28, "comparison-with-ols"]], "Computing the Gini index": [[9, "computing-the-gini-index"]], "Conditions on convex functions": [[28, "conditions-on-convex-functions"]], "Conjugate gradient method": [[13, "conjugate-gradient-method"]], "Convex function": [[28, "convex-function"]], "Convex functions": [[13, "convex-functions"], [28, "convex-functions"]], "Convolution Examples: Polynomial multiplication": [[3, "convolution-examples-polynomial-multiplication"]], "Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)": [[3, "convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms"]], "Convolutional Neural Network": [[12, "convolutional-neural-network"]], "Convolutional Neural Networks": [[3, null]], "Correlation Function and Design/Feature Matrix": [[27, "correlation-function-and-design-feature-matrix"]], "Correlation Matrix": [[11, "correlation-matrix"], [27, "correlation-matrix"]], "Correlation Matrix with Pandas": [[27, "correlation-matrix-with-pandas"]], "Course Format": [[26, "course-format"]], "Course setting": [[22, null]], "Covariance Matrix Examples": [[27, "covariance-matrix-examples"]], "Covariance and Correlation Matrix": [[27, "covariance-and-correlation-matrix"]], "Cross-validation": [[6, "cross-validation"]], "Deadlines for projects (tentative)": [[26, "deadlines-for-projects-tentative"]], "Decision trees, overarching aims": [[9, null]], "Deep learning methods": [[26, "deep-learning-methods"]], "Define model and architecture": [[1, "define-model-and-architecture"]], "Defining the cost function": [[1, "defining-the-cost-function"]], "Deliverables": [[15, "deliverables"], [16, "deliverables"]], "Derivatives and the chain rule": [[12, "derivatives-and-the-chain-rule"]], "Derivatives, example 1": [[27, "derivatives-example-1"]], "Deriving OLS from a probability distribution": [[5, "deriving-ols-from-a-probability-distribution"]], "Deriving and Implementing Ordinary Least Squares": [[16, "deriving-and-implementing-ordinary-least-squares"]], "Deriving and Implementing Ridge Regression": [[17, "deriving-and-implementing-ridge-regression"]], "Deriving the Lasso Regression Equations": [[27, "deriving-the-lasso-regression-equations"], [28, "deriving-the-lasso-regression-equations"], [28, "id6"]], "Deriving the Ridge Regression Equations": [[27, "deriving-the-ridge-regression-equations"], [28, "deriving-the-ridge-regression-equations"], [28, "id3"]], "Deriving the back propagation code for a multilayer perceptron model": [[12, "deriving-the-back-propagation-code-for-a-multilayer-perceptron-model"]], "Developing a code for doing neural networks with back propagation": [[1, "developing-a-code-for-doing-neural-networks-with-back-propagation"]], "Diagonalize the sample covariance matrix to obtain the principal components": [[11, "diagonalize-the-sample-covariance-matrix-to-obtain-the-principal-components"]], "Different kernels and Mercer\u2019s theorem": [[8, "different-kernels-and-mercer-s-theorem"]], "Disadvantages": [[9, "disadvantages"]], "Discriminative Modeling": [[26, "discriminative-modeling"]], "Domains and probabilities": [[23, "domains-and-probabilities"]], "Dropout": [[1, "dropout"]], "Economy-size SVD": [[27, "economy-size-svd"], [28, "economy-size-svd"]], "Elements of Probability Theory and Statistical Data Analysis": [[23, null]], "Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods": [[10, null]], "Entropy and the ID3 algorithm": [[9, "entropy-and-the-id3-algorithm"]], "Essential elements of ML": [[26, "essential-elements-of-ml"]], "Evaluate model performance on test data": [[1, "evaluate-model-performance-on-test-data"]], "Example 2": [[27, "example-2"]], "Example 3": [[27, "example-3"]], "Example 4": [[27, "example-4"]], "Example Matrix": [[27, "example-matrix"], [28, "example-matrix"]], "Example of discriminative modeling, taken from Generative Deep Learning by David Foster": [[26, "example-of-discriminative-modeling-taken-from-generative-deep-learning-by-david-foster"]], "Example of generative modeling, taken from Generative Deep Learning by David Foster": [[26, "example-of-generative-modeling-taken-from-generative-deep-learning-by-david-foster"]], "Example of own Standard scaling": [[27, "example-of-own-standard-scaling"]], "Example relevant for the exercises": [[27, "example-relevant-for-the-exercises"]], "Example: Exponential decay": [[2, "example-exponential-decay"]], "Example: Population growth": [[2, "example-population-growth"]], "Example: The diffusion equation": [[2, "example-the-diffusion-equation"]], "Example: binary classification problem": [[1, "example-binary-classification-problem"]], "Examples": [[26, "examples"]], "Examples of likelihood functions used in logistic regression and neural networks": [[7, "examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks"]], "Exercise 1 - Choice of model and degrees of freedom": [[17, "exercise-1-choice-of-model-and-degrees-of-freedom"]], "Exercise 1 - Finding the derivative of Matrix-Vector expressions": [[16, "exercise-1-finding-the-derivative-of-matrix-vector-expressions"]], "Exercise 1 - Github Setup": [[15, "exercise-1-github-setup"]], "Exercise 1, scale your data": [[18, "exercise-1-scale-your-data"]], "Exercise 1: Setting up various Python environments": [[0, "exercise-1-setting-up-various-python-environments"]], "Exercise 2 - Deriving the expression for OLS": [[16, "exercise-2-deriving-the-expression-for-ols"]], "Exercise 2 - Deriving the expression for Ridge Regression": [[17, "exercise-2-deriving-the-expression-for-ridge-regression"]], "Exercise 2 - Setting up a Github repository": [[15, "exercise-2-setting-up-a-github-repository"]], "Exercise 2, calculate the gradients": [[18, "exercise-2-calculate-the-gradients"]], "Exercise 2: making your own data and exploring scikit-learn": [[0, "exercise-2-making-your-own-data-and-exploring-scikit-learn"]], "Exercise 3 - Creating feature matrix and implementing OLS using the analytical expression": [[16, "exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression"]], "Exercise 3 - Fitting an OLS model to data": [[15, "exercise-3-fitting-an-ols-model-to-data"]], "Exercise 3 - Scaling data": [[17, "exercise-3-scaling-data"]], "Exercise 3 - Setting up a Python virtual environment": [[15, "exercise-3-setting-up-a-python-virtual-environment"]], "Exercise 3, using the analytical formulae for OLS and Ridge regression to find the optimal paramters \\boldsymbol{\\theta}": [[18, "exercise-3-using-the-analytical-formulae-for-ols-and-ridge-regression-to-find-the-optimal-paramters-boldsymbol-theta"]], "Exercise 3: Normalizing our data": [[0, "exercise-3-normalizing-our-data"]], "Exercise 4 - Fitting a polynomial": [[16, "exercise-4-fitting-a-polynomial"]], "Exercise 4 - Implementing Ridge Regression": [[17, "exercise-4-implementing-ridge-regression"]], "Exercise 4 - Testing multiple hyperparameters": [[17, "exercise-4-testing-multiple-hyperparameters"]], "Exercise 4 - The train-test split": [[15, "exercise-4-the-train-test-split"]], "Exercise 4, Implementing the simplest form for gradient descent": [[18, "exercise-4-implementing-the-simplest-form-for-gradient-descent"]], "Exercise 4: Adding Ridge Regression": [[0, "exercise-4-adding-ridge-regression"]], "Exercise 5 - Comparing your code with sklearn": [[16, "exercise-5-comparing-your-code-with-sklearn"]], "Exercise 5, Ridge regression and a new Synthetic Dataset": [[18, "exercise-5-ridge-regression-and-a-new-synthetic-dataset"]], "Exercise 5: Analytical exercises": [[0, "exercise-5-analytical-exercises"]], "Exercise: Cross-validation as resampling techniques, adding more complexity": [[6, "exercise-cross-validation-as-resampling-techniques-adding-more-complexity"]], "Exercise: Analysis of real data": [[6, "exercise-analysis-of-real-data"]], "Exercise: Bias-variance trade-off and resampling techniques": [[6, "exercise-bias-variance-trade-off-and-resampling-techniques"]], "Exercise: Lasso Regression on the Franke function with resampling": [[6, "exercise-lasso-regression-on-the-franke-function-with-resampling"]], "Exercise: Ordinary Least Square (OLS) on the Franke function": [[6, "exercise-ordinary-least-square-ols-on-the-franke-function"]], "Exercise: Ridge Regression on the Franke function with resampling": [[6, "exercise-ridge-regression-on-the-franke-function-with-resampling"]], "Exercises": [[0, "exercises"]], "Exercises and Projects": [[6, "exercises-and-projects"]], "Exercises week 34": [[15, null]], "Exercises week 35": [[16, null]], "Exercises week 36": [[17, null]], "Exercises week 37": [[18, null]], "Expectation values": [[23, "expectation-values"]], "Extending to more than one variable": [[28, "extending-to-more-than-one-variable"]], "Extremely useful tools, strongly recommended": [[26, "extremely-useful-tools-strongly-recommended"]], "Feed-forward neural networks": [[12, "feed-forward-neural-networks"]], "Feed-forward pass": [[1, "feed-forward-pass"]], "Final back propagating equation": [[12, "final-back-propagating-equation"]], "Fine-tuning neural network hyperparameters": [[1, "fine-tuning-neural-network-hyperparameters"]], "Fitting an Equation of State for Dense Nuclear Matter": [[0, "fitting-an-equation-of-state-for-dense-nuclear-matter"]], "Fixing the singularity": [[27, "fixing-the-singularity"], [28, "fixing-the-singularity"]], "Format for electronic delivery of report and programs": [[21, "format-for-electronic-delivery-of-report-and-programs"]], "Frequently used scaling functions": [[27, "frequently-used-scaling-functions"]], "From OLS to Ridge and Lasso": [[28, "from-ols-to-ridge-and-lasso"]], "From one to many layers, the universal approximation theorem": [[12, "from-one-to-many-layers-the-universal-approximation-theorem"]], "Functionality in Scikit-Learn": [[27, "functionality-in-scikit-learn"]], "Further Dimensionality Remarks": [[3, "further-dimensionality-remarks"]], "Further properties (important for our analyses later)": [[5, "further-properties-important-for-our-analyses-later"], [27, "further-properties-important-for-our-analyses-later"], [28, "further-properties-important-for-our-analyses-later"]], "Gaussian Elimination": [[20, "gaussian-elimination"]], "General Features": [[9, "general-features"]], "General linear models and linear algebra": [[26, "general-linear-models-and-linear-algebra"]], "Generalizing the fitting procedure as a linear algebra problem": [[26, "generalizing-the-fitting-procedure-as-a-linear-algebra-problem"], [26, "id1"]], "Generative Adversarial Networks": [[4, "generative-adversarial-networks"]], "Generative Models": [[4, "generative-models"]], "Generative Versus Discriminative Modeling": [[26, "generative-versus-discriminative-modeling"]], "Geometric Interpretation and link with Singular Value Decomposition": [[11, "geometric-interpretation-and-link-with-singular-value-decomposition"]], "Gradient Boosting, Classification Example": [[10, "gradient-boosting-classification-example"]], "Gradient Boosting, Examples of Regression": [[10, "gradient-boosting-examples-of-regression"]], "Gradient Clipping": [[1, "gradient-clipping"]], "Gradient Descent Example": [[28, "id1"]], "Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent": [[10, "gradient-boosting-basics-with-steepest-descent-functional-gradient-descent"]], "Gradient descent": [[2, "gradient-descent"]], "Gradient descent and Ridge": [[28, "gradient-descent-and-ridge"]], "Gradient descent example": [[28, "gradient-descent-example"]], "Grading": [[24, "grading"], [24, "id2"], [26, "grading"]], "How to take derivatives of Matrix-Vector expressions": [[16, "how-to-take-derivatives-of-matrix-vector-expressions"]], "Hyperplanes and all that": [[8, "hyperplanes-and-all-that"]], "Important Matrix and vector handling packages": [[20, "important-matrix-and-vector-handling-packages"]], "Important technicalities: More on Rescaling data": [[27, "important-technicalities-more-on-rescaling-data"]], "Improving performance": [[1, "improving-performance"]], "In summary": [[24, "in-summary"]], "Including Stochastic Gradient Descent with Autograd": [[13, "including-stochastic-gradient-descent-with-autograd"]], "Incremental PCA": [[11, "incremental-pca"]], "Installing R, C++, cython or Julia": [[26, "installing-r-c-cython-or-julia"]], "Installing R, C++, cython, Numba etc": [[26, "installing-r-c-cython-numba-etc"]], "Instructor information": [[24, "instructor-information"]], "Interpretations and optimizing our parameters": [[26, "interpretations-and-optimizing-our-parameters"], [26, "id2"], [26, "id3"], [27, "interpretations-and-optimizing-our-parameters"], [27, "id1"], [27, "id2"]], "Interpreting the Ridge results": [[27, "interpreting-the-ridge-results"], [28, "interpreting-the-ridge-results"], [28, "id4"]], "Introducing JAX": [[13, "introducing-jax"]], "Introducing the Covariance and Correlation functions": [[11, "introducing-the-covariance-and-correlation-functions"], [27, "introducing-the-covariance-and-correlation-functions"]], "Introduction": [[0, "introduction"], [6, "introduction"], [19, "introduction"], [20, "introduction"]], "Introduction to numerical projects": [[21, "introduction-to-numerical-projects"]], "Iterative Fitting, Classification and AdaBoost": [[10, "iterative-fitting-classification-and-adaboost"]], "Iterative Fitting, Regression and Squared-error Cost Function": [[10, "iterative-fitting-regression-and-squared-error-cost-function"]], "Kernel PCA": [[11, "kernel-pca"]], "Kernels and non-linearity": [[8, "kernels-and-non-linearity"]], "LU Decomposition, the inverse of a matrix": [[20, "lu-decomposition-the-inverse-of-a-matrix"]], "Lasso Regression": [[28, "lasso-regression"]], "Lasso case": [[28, "lasso-case"]], "Layers": [[1, "layers"]], "Layers used to build CNNs": [[3, "layers-used-to-build-cnns"]], "Learning goals": [[15, "learning-goals"], [16, "learning-goals"], [17, "learning-goals"], [18, "learning-goals"]], "Learning outcomes": [[19, "learning-outcomes"], [26, "learning-outcomes"]], "Lectures and ComputerLab": [[26, "lectures-and-computerlab"]], "Limitations of supervised learning with deep networks": [[1, "limitations-of-supervised-learning-with-deep-networks"]], "Linear Algebra, Handling of Arrays and more Python Features": [[20, null]], "Linear Regression": [[0, null]], "Linear Regression Problems": [[27, "linear-regression-problems"], [28, "linear-regression-problems"]], "Linear Regression and the SVD": [[28, "linear-regression-and-the-svd"]], "Linear Regression, basic elements": [[0, "linear-regression-basic-elements"]], "Linking Bayes\u2019 Theorem with Ridge and Lasso Regression": [[5, "linking-bayes-theorem-with-ridge-and-lasso-regression"]], "Linking the regression analysis with a statistical interpretation": [[5, "linking-the-regression-analysis-with-a-statistical-interpretation"]], "Linking with the SVD": [[5, "linking-with-the-svd"], [27, "linking-with-the-svd"]], "Links to relevant courses at the University of Oslo": [[25, "links-to-relevant-courses-at-the-university-of-oslo"]], "Logistic Regression": [[7, null], [7, "id1"]], "MNIST and GANs": [[4, "mnist-and-gans"]], "Machine Learning": [[26, "machine-learning"]], "Machine learning": [[19, "machine-learning"]], "Main textbooks": [[26, "main-textbooks"]], "Making a tree": [[9, "making-a-tree"]], "Making your own Bootstrap: Changing the Level of the Decision Tree": [[10, "making-your-own-bootstrap-changing-the-level-of-the-decision-tree"]], "Making your own test-train splitting": [[27, "making-your-own-test-train-splitting"]], "Material for exercises week 35": [[27, "material-for-exercises-week-35"]], "Material for lab sessions sessions Tuesday and Wednesday": [[28, "material-for-lab-sessions-sessions-tuesday-and-wednesday"]], "Material for lecture Monday September 2": [[28, "material-for-lecture-monday-september-2"]], "Mathematical Interpretation of Ordinary Least Squares": [[5, "mathematical-interpretation-of-ordinary-least-squares"], [27, "mathematical-interpretation-of-ordinary-least-squares"], [28, "mathematical-interpretation-of-ordinary-least-squares"]], "Mathematical optimization of convex functions": [[8, "mathematical-optimization-of-convex-functions"]], "Mathematics of CNNs": [[3, "mathematics-of-cnns"]], "Mathematics of the SVD and implications": [[5, "mathematics-of-the-svd-and-implications"], [27, "mathematics-of-the-svd-and-implications"], [28, "mathematics-of-the-svd-and-implications"]], "Matrices in Python": [[26, "matrices-in-python"]], "Matrix multiplication": [[1, "matrix-multiplication"]], "Matrix-vector notation and activation": [[12, "matrix-vector-notation-and-activation"]], "Meet the covariance!": [[23, "meet-the-covariance"]], "Meet the Covariance Matrix": [[5, "meet-the-covariance-matrix"], [27, "meet-the-covariance-matrix"]], "Meet the Hessian Matrix": [[27, "meet-the-hessian-matrix"]], "Meet the Pandas": [[26, "meet-the-pandas"]], "Min-Max Scaling": [[27, "min-max-scaling"]], "Momentum based GD": [[13, "momentum-based-gd"]], "More complicated Example: The Ising model": [[6, "more-complicated-example-the-ising-model"]], "More interpretations": [[27, "more-interpretations"], [28, "more-interpretations"], [28, "id5"]], "More on Dimensionalities": [[3, "more-on-dimensionalities"]], "More on Rescaling data": [[6, "more-on-rescaling-data"]], "More on Steepest descent": [[28, "more-on-steepest-descent"]], "More on convex functions": [[28, "more-on-convex-functions"]], "More preprocessing": [[27, "more-preprocessing"]], "Multilayer perceptrons": [[12, "multilayer-perceptrons"]], "Network requirements": [[2, "network-requirements"]], "Neural Networks vs CNNs": [[3, "neural-networks-vs-cnns"]], "Neural networks": [[12, null]], "Note about SVD Calculations": [[27, "note-about-svd-calculations"], [28, "note-about-svd-calculations"]], "Note on Scikit-Learn": [[28, "note-on-scikit-learn"]], "Numerical experiments and the covariance, central limit theorem": [[23, "numerical-experiments-and-the-covariance-central-limit-theorem"]], "Numpy and arrays": [[20, "numpy-and-arrays"], [26, "numpy-and-arrays"]], "Numpy examples and Important Matrix and vector handling packages": [[26, "numpy-examples-and-important-matrix-and-vector-handling-packages"]], "Optimization and gradient descent, the central part of any Machine Learning algortithm": [[28, "optimization-and-gradient-descent-the-central-part-of-any-machine-learning-algortithm"]], "Optimization, the central part of any Machine Learning algortithm": [[13, null]], "Optimizing our parameters": [[26, "optimizing-our-parameters"]], "Optimizing our parameters, more details": [[26, "optimizing-our-parameters-more-details"]], "Optimizing the cost function": [[1, "optimizing-the-cost-function"]], "Organizing our data": [[0, "organizing-our-data"], [26, "organizing-our-data"]], "Other Matrix and Vector Operations": [[20, "other-matrix-and-vector-operations"]], "Other Types of Recurrent Neural Networks": [[4, "other-types-of-recurrent-neural-networks"]], "Other courses on Data science and Machine Learning at UiO": [[26, "other-courses-on-data-science-and-machine-learning-at-uio"]], "Other courses on Data science and Machine Learning at UiO, contn": [[26, "other-courses-on-data-science-and-machine-learning-at-uio-contn"]], "Other popular texts": [[26, "other-popular-texts"]], "Other techniques": [[11, "other-techniques"]], "Other types of networks": [[12, "other-types-of-networks"]], "Other ways of visualizing the trees": [[9, "other-ways-of-visualizing-the-trees"]], "Our model for the nuclear binding energies": [[26, "our-model-for-the-nuclear-binding-energies"]], "Overview of first week": [[26, "overview-of-first-week"]], "Own code for Ordinary Least Squares": [[26, "own-code-for-ordinary-least-squares"], [27, "own-code-for-ordinary-least-squares"]], "PCA and scikit-learn": [[11, "pca-and-scikit-learn"]], "Pandas AI": [[26, "pandas-ai"]], "Part a : Ordinary Least Square (OLS) for the Runge function": [[21, "part-a-ordinary-least-square-ols-for-the-runge-function"]], "Part b: Adding Ridge regression for the Runge function": [[21, "part-b-adding-ridge-regression-for-the-runge-function"]], "Part c: Writing your own gradient descent code": [[21, "part-c-writing-your-own-gradient-descent-code"]], "Part d: Including momentum and more advanced ways to update the learning the rate": [[21, "part-d-including-momentum-and-more-advanced-ways-to-update-the-learning-the-rate"]], "Part e: Writing our own code for Lasso regression": [[21, "part-e-writing-our-own-code-for-lasso-regression"]], "Part f: Stochastic gradient descent": [[21, "part-f-stochastic-gradient-descent"]], "Part g: Bias-variance trade-off and resampling techniques": [[21, "part-g-bias-variance-trade-off-and-resampling-techniques"]], "Part h): Cross-validation as resampling techniques, adding more complexity": [[21, "part-h-cross-validation-as-resampling-techniques-adding-more-complexity"]], "Partial Differential Equations": [[2, "partial-differential-equations"]], "Plans for week 35": [[27, "plans-for-week-35"]], "Plans for week 36": [[28, "plans-for-week-36"]], "Practical tips": [[13, "practical-tips"]], "Practicalities": [[24, "practicalities"], [24, "id1"]], "Preamble: Note on writing reports, using reference material, AI and other tools": [[21, "preamble-note-on-writing-reports-using-reference-material-ai-and-other-tools"]], "Predicting New Points With A Trained Recurrent Neural Network": [[4, "predicting-new-points-with-a-trained-recurrent-neural-network"]], "Preprocessing our data": [[27, "preprocessing-our-data"]], "Prerequisites": [[26, "prerequisites"]], "Prerequisites and background": [[19, "prerequisites-and-background"]], "Prerequisites: Collect and pre-process data": [[3, "prerequisites-collect-and-pre-process-data"]], "Probability Distribution Functions": [[23, "probability-distribution-functions"]], "Program example for gradient descent with Ridge Regression": [[28, "program-example-for-gradient-descent-with-ridge-regression"]], "Program for stochastic gradient": [[13, "program-for-stochastic-gradient"]], "Project 1 on Machine Learning, deadline October 6 (midnight), 2025": [[21, null]], "Properties of PDFs": [[23, "properties-of-pdfs"]], "Pros and cons of trees, pros": [[9, "pros-and-cons-of-trees-pros"]], "Python installers": [[19, "python-installers"], [26, "python-installers"]], "RMS prop": [[13, "rms-prop"]], "Random Numbers": [[23, "random-numbers"]], "Random forests": [[10, "random-forests"]], "Randomized PCA": [[11, "randomized-pca"]], "Reading material": [[26, "reading-material"]], "Reading recommendations:": [[27, "reading-recommendations"]], "Reading suggestions week 34": [[26, "reading-suggestions-week-34"]], "Recurrent neural networks": [[12, "recurrent-neural-networks"]], "Recurrent neural networks: Overarching view": [[4, null]], "Reducing the number of degrees of freedom, overarching view": [[0, "reducing-the-number-of-degrees-of-freedom-overarching-view"], [27, "reducing-the-number-of-degrees-of-freedom-overarching-view"]], "Reformulating the problem": [[2, "reformulating-the-problem"]], "Regression Case": [[10, "regression-case"]], "Regression analysis and resampling methods": [[21, "regression-analysis-and-resampling-methods"]], "Regression analysis, overarching aims": [[26, "regression-analysis-overarching-aims"]], "Regression analysis, overarching aims II": [[26, "regression-analysis-overarching-aims-ii"]], "Regularization": [[1, "regularization"]], "Reminder from last week": [[27, "reminder-from-last-week"]], "Reminder on Newton-Raphson\u2019s method": [[28, "reminder-on-newton-raphson-s-method"]], "Reminder on Statistics": [[6, "reminder-on-statistics"]], "Replace or not": [[13, "replace-or-not"]], "Required Technologies": [[19, "required-technologies"]], "Resampling Methods": [[6, null]], "Resampling methods": [[6, "id1"]], "Residual Error": [[27, "residual-error"], [28, "residual-error"]], "Resources on differential equations and deep learning": [[2, "resources-on-differential-equations-and-deep-learning"]], "Revisiting Ordinary Least Squares": [[28, "revisiting-ordinary-least-squares"]], "Revisiting our Linear Regression Solvers": [[13, "revisiting-our-linear-regression-solvers"]], "Rewriting the Covariance and/or Correlation Matrix": [[27, "rewriting-the-covariance-and-or-correlation-matrix"]], "Rewriting the fitting procedure as a linear algebra problem": [[26, "rewriting-the-fitting-procedure-as-a-linear-algebra-problem"]], "Rewriting the fitting procedure as a linear algebra problem, more details": [[26, "rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details"]], "Ridge Regression": [[28, "ridge-regression"]], "Ridge and LASSO Regression": [[27, "ridge-and-lasso-regression"], [28, "ridge-and-lasso-regression"], [28, "id2"]], "Ridge and Lasso Regression": [[5, null], [5, "id1"]], "SVD analysis": [[28, "svd-analysis"]], "Same code but now with momentum gradient descent": [[13, "same-code-but-now-with-momentum-gradient-descent"]], "Schedule first week": [[26, "schedule-first-week"]], "Schematic Regression Procedure": [[9, "schematic-regression-procedure"]], "Setting up the Back propagation algorithm": [[12, "setting-up-the-back-propagation-algorithm"]], "Setting up the Matrix to be inverted": [[27, "setting-up-the-matrix-to-be-inverted"], [28, "setting-up-the-matrix-to-be-inverted"]], "Setting up the network using Autograd; The full program": [[2, "setting-up-the-network-using-autograd-the-full-program"]], "Similar (second order function now) problem but now with AdaGrad": [[13, "similar-second-order-function-now-problem-but-now-with-adagrad"]], "Simple Python Code to read in Data and perform Classification": [[9, "simple-python-code-to-read-in-data-and-perform-classification"]], "Simple case": [[27, "simple-case"], [28, "simple-case"]], "Simple code for solving the above problem": [[28, "simple-code-for-solving-the-above-problem"]], "Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression": [[28, "simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression"]], "Simple geometric interpretation": [[28, "simple-geometric-interpretation"]], "Simple linear regression model using scikit-learn": [[0, "simple-linear-regression-model-using-scikit-learn"], [26, "simple-linear-regression-model-using-scikit-learn"]], "Simple one-dimensional second-order polynomial": [[18, "simple-one-dimensional-second-order-polynomial"]], "Simple program": [[28, "simple-program"]], "Software and needed installations": [[21, "software-and-needed-installations"], [26, "software-and-needed-installations"]], "Solving Differential Equations with Deep Learning": [[2, null]], "Solving the one dimensional Poisson equation": [[2, "solving-the-one-dimensional-poisson-equation"]], "Solving the wave equation with Neural Networks": [[2, "solving-the-wave-equation-with-neural-networks"]], "Some famous Matrices": [[20, "some-famous-matrices"]], "Some simple problems": [[13, "some-simple-problems"], [28, "some-simple-problems"]], "Some useful matrix and vector expressions": [[27, "some-useful-matrix-and-vector-expressions"]], "Splitting our Data in Training and Test data": [[0, "splitting-our-data-in-training-and-test-data"], [27, "splitting-our-data-in-training-and-test-data"]], "Standard steepest descent": [[13, "standard-steepest-descent"]], "Statistical analysis and optimization of data": [[19, "statistical-analysis-and-optimization-of-data"], [26, "statistical-analysis-and-optimization-of-data"]], "Steepest descent": [[13, "steepest-descent"], [28, "steepest-descent"]], "Stochastic Gradient Descent (SGD)": [[13, "stochastic-gradient-descent-sgd"]], "Stochastic variables and the main concepts, the discrete case": [[23, "stochastic-variables-and-the-main-concepts-the-discrete-case"]], "Support Vector Machines, overarching aims": [[8, null]], "Systematic reduction": [[3, "systematic-reduction"]], "Teachers": [[26, "teachers"]], "Teachers and Grading": [[24, null]], "Teaching Assistants Fall semester 2023": [[24, "teaching-assistants-fall-semester-2023"]], "Tentative deadllines for projects": [[24, "tentative-deadllines-for-projects"]], "Testing the Means Squared Error as function of Complexity": [[0, "testing-the-means-squared-error-as-function-of-complexity"], [27, "testing-the-means-squared-error-as-function-of-complexity"]], "Textbooks": [[25, null]], "The Algorithm before theorem": [[11, "the-algorithm-before-theorem"]], "The Breast Cancer Data, now with Keras": [[1, "the-breast-cancer-data-now-with-keras"]], "The CART algorithm for Classification": [[9, "the-cart-algorithm-for-classification"]], "The CART algorithm for Regression": [[9, "the-cart-algorithm-for-regression"]], "The CIFAR01 data set": [[3, "the-cifar01-data-set"]], "The Hessian matrix": [[28, "the-hessian-matrix"]], "The Hessian matrix for Ridge Regression": [[28, "the-hessian-matrix-for-ridge-regression"]], "The Jacobian": [[27, "the-jacobian"]], "The MNIST dataset again": [[3, "the-mnist-dataset-again"]], "The OLS case": [[28, "the-ols-case"]], "The RELU function family": [[1, "the-relu-function-family"]], "The Ridge case": [[28, "the-ridge-case"]], "The SVD, a Fantastic Algorithm": [[27, "the-svd-a-fantastic-algorithm"], [28, "the-svd-a-fantastic-algorithm"]], "The Softmax function": [[1, "the-softmax-function"]], "The \\chi^2 function": [[0, "the-chi-2-function"], [26, "the-chi-2-function"], [26, "id4"], [26, "id5"], [26, "id6"], [26, "id7"], [26, "id8"]], "The bias-variance tradeoff": [[6, "the-bias-variance-tradeoff"]], "The code for solving the ODE": [[2, "the-code-for-solving-the-ode"]], "The complete code with a simple data set": [[27, "the-complete-code-with-a-simple-data-set"]], "The cost/loss function": [[27, "the-cost-loss-function"]], "The course has two central parts": [[19, "the-course-has-two-central-parts"]], "The derivative of the cost/loss function": [[28, "the-derivative-of-the-cost-loss-function"]], "The equations": [[28, "the-equations"]], "The equations for ordinary least squares": [[27, "the-equations-for-ordinary-least-squares"]], "The first Case": [[28, "the-first-case"]], "The ideal": [[28, "the-ideal"]], "The logistic function": [[7, "the-logistic-function"]], "The mean squared error and its derivative": [[27, "the-mean-squared-error-and-its-derivative"]], "The moons example": [[8, "the-moons-example"]], "The multilayer perceptron (MLP)": [[12, "the-multilayer-perceptron-mlp"]], "The network with one input layer, specified number of hidden layers, and one output layer": [[2, "the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer"]], "The plethora of machine learning algorithms/methods": [[26, "the-plethora-of-machine-learning-algorithms-methods"]], "The sensitiveness of the gradient descent": [[28, "the-sensitiveness-of-the-gradient-descent"]], "The singular value decomposition": [[5, "the-singular-value-decomposition"], [27, "the-singular-value-decomposition"], [28, "the-singular-value-decomposition"]], "The two-dimensional case": [[8, "the-two-dimensional-case"]], "To our real data: nuclear binding energies. Brief reminder on masses and binding energies": [[26, "to-our-real-data-nuclear-binding-energies-brief-reminder-on-masses-and-binding-energies"]], "Topics covered in this course: Statistical analysis and optimization of data": [[26, "topics-covered-in-this-course-statistical-analysis-and-optimization-of-data"]], "Towards the PCA theorem": [[11, "towards-the-pca-theorem"]], "Train and test datasets": [[1, "train-and-test-datasets"]], "Two-dimensional Objects": [[3, "two-dimensional-objects"]], "Type of problem": [[2, "type-of-problem"]], "Types of Machine Learning": [[26, "types-of-machine-learning"]], "Useful Python libraries": [[19, "useful-python-libraries"], [26, "useful-python-libraries"]], "Using Autograd": [[13, "using-autograd"]], "Using forward Euler to solve the ODE": [[2, "using-forward-euler-to-solve-the-ode"]], "Using gradient descent methods, limitations": [[13, "using-gradient-descent-methods-limitations"], [28, "using-gradient-descent-methods-limitations"]], "Visualization": [[1, "visualization"], [1, "id1"]], "Visualizing the Tree, Classification": [[9, "visualizing-the-tree-classification"]], "Week 34: Introduction to the course, Logistics and Practicalities": [[26, null]], "Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression": [[27, null]], "Week 36: Linear Regression and Gradient descent": [[28, null]], "What Is Generative Modeling?": [[26, "what-is-generative-modeling"]], "What does it mean?": [[27, "what-does-it-mean"], [28, "what-does-it-mean"]], "What is Machine Learning?": [[0, "what-is-machine-learning"]], "What is a good model?": [[0, "what-is-a-good-model"], [26, "what-is-a-good-model"]], "What is a good model? Can we define it?": [[26, "what-is-a-good-model-can-we-define-it"]], "Which activation function should I use?": [[1, "which-activation-function-should-i-use"]], "Why Linear Regression (aka Ordinary Least Squares and family)": [[26, "why-linear-regression-aka-ordinary-least-squares-and-family"]], "Wisconsin Cancer Data": [[7, "wisconsin-cancer-data"]], "With Lasso Regression": [[28, "with-lasso-regression"]], "Writing Our First Generative Adversarial Network": [[4, "writing-our-first-generative-adversarial-network"]], "Writing our own PCA code": [[11, "writing-our-own-pca-code"]], "Writing the Cost Function": [[28, "writing-the-cost-function"]], "XGBoost: Extreme Gradient Boosting": [[10, "xgboost-extreme-gradient-boosting"]], "Yet another Example": [[28, "yet-another-example"]], "a) Expression for Ridge regression": [[17, "a-expression-for-ridge-regression"]], "scikit-learn implementation": [[1, "scikit-learn-implementation"]]}, "docnames": ["chapter1", "chapter10", "chapter11", "chapter12", "chapter13", "chapter2", "chapter3", "chapter4", "chapter5", "chapter6", "chapter7", "chapter8", "chapter9", "chapteroptimization", "clustering", "exercisesweek34", "exercisesweek35", "exercisesweek36", "exercisesweek37", "intro", "linalg", "project1", "schedule", "statistics", "teachers", "textbooks", "week34", "week35", "week36"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1}, "filenames": ["chapter1.ipynb", "chapter10.ipynb", "chapter11.ipynb", "chapter12.ipynb", "chapter13.ipynb", "chapter2.ipynb", "chapter3.ipynb", "chapter4.ipynb", "chapter5.ipynb", "chapter6.ipynb", "chapter7.ipynb", "chapter8.ipynb", "chapter9.ipynb", "chapteroptimization.ipynb", "clustering.ipynb", "exercisesweek34.ipynb", "exercisesweek35.ipynb", "exercisesweek36.ipynb", "exercisesweek37.ipynb", "intro.md", "linalg.ipynb", "project1.ipynb", "schedule.md", "statistics.ipynb", "teachers.md", "textbooks.md", "week34.ipynb", "week35.ipynb", "week36.ipynb"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 26, 27], "0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 23, 24, 26, 27, 28], "00": [0, 1, 5, 11, 26, 27], "000": [1, 3], "00000000e": [], "001": [2, 8, 13, 28], "004": 5, "004113634617443131": 27, "004113634617443139": 27, "00411363461744314": 27, "004113634617443147": 27, "00727646693": [0, 26], "0086649156": [0, 26], "01": [0, 1, 2, 5, 9, 11, 13, 17, 25, 26, 27], "0110": 23, "01719003e": [], "02": [0, 4, 7, 12, 26], "02334824": [], "02857": 4, "02f": 6, "03077640549": 4, "03097597e": [], "031": 5, "04": 11, "0458": 9, "05": [4, 6], "062292565": 4, "062435": [], "06730814": [], "07": [], "0713": [0, 26], "07285": 3, "08": 23, "08078025e": [], "08336233266": 4, "08376632": 27, "083766322923899": 27, "0837663229239043": 27, "0917": 9, "0n": [0, 26], "0x113e21950": 17, "1": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 22, 23, 24, 25, 26, 28], "10": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 20, 22, 23, 24, 26, 27, 28], "100": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 20, 23, 24, 26, 27, 28], "1000": [0, 1, 2, 4, 5, 8, 11, 13, 14, 18, 19, 23, 26, 28], "10000": [2, 5, 6, 10, 11, 13, 23], "100000": 8, "10001": 10, "1001": 23, "1002": 23, "1003": 23, "1005": 23, "1009": 23, "101": 16, "1011": 23, "1013": 23, "1013904243": 23, "1015": 23, "102": 16, "1023": 23, "1024": 3, "1026": 23, "1027": 23, "103": 1, "1030": 23, "1037": 23, "1038": 23, "1040": 23, "1047": 23, "107": 16, "108": [], "10th": 9, "10x": [0, 26], "11": [0, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 20, 21, 23, 25, 26, 27, 28], "110": [], "1100": 23, "1101": 23, "111": [1, 7, 12], "112": 16, "11340253": [], "11590451": [], "116": 16, "117": 16, "118": 16, "12": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 18, 20, 23, 25, 26, 27, 28], "120": 3, "121": [8, 9, 10, 16], "1215pm": [24, 26], "122": [8, 9, 10], "124": [0, 26], "125": 16, "127": [4, 16], "128": [3, 4, 13], "129": 16, "1298": 9, "12pm": [24, 26], "13": [0, 2, 9, 12, 20, 23, 26], "131": 16, "133": 7, "135": 16, "136": 16, "14": [0, 2, 4, 6, 8, 9, 10, 12, 20, 23, 25, 27], "141": 16, "143": 16, "1446729567": 4, "149": 16, "14g": 6, "15": [0, 2, 4, 6, 7, 8, 9, 12, 13, 21, 23, 26, 28], "150": [4, 8], "152": 16, "153760": [], "156": 16, "157": [], "158": [], "159": 16, "15g": 6, "15pm": 26, "16": [1, 2, 3, 4, 5, 8, 9, 10, 23, 26, 28], "160": 16, "1603": 3, "161": 16, "162": 16, "16231451": 4, "163": 16, "16384": 3, "164": 16, "167": 16, "17": [1, 2, 8, 23], "172": 16, "173": 16, "176": 16, "178": 16, "179": 16, "1797": 1, "18": [2, 6, 7, 8, 9, 10, 23, 26], "1807": 4, "18392847": [], "19": [2, 23, 26], "1940": [], "1943": 12, "19569961": 27, "1970": [20, 26], "1973": 9, "1979": 6, "1_1": 12, "1_2": 12, "1_3": 12, "1cm": [0, 8, 10, 23, 26], "1d": [1, 2, 3], "1e": [2, 4, 13, 14], "1e10": 14, "1e4": 6, "1f": 1, "1k": 20, "1n": [0, 26], "1x": [0, 26], "2": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19, 20, 21, 23, 25], "20": [0, 1, 2, 6, 7, 8, 16, 17, 23, 24, 26, 27, 28], "200": [0, 2, 3, 4, 8, 9, 10], "2000": [0, 27], "2004": [13, 28], "2006": 25, "20072279": [], "2008": 26, "2010": 1, "2011": 1, "2014": 4, "2015": 1, "2016": [0, 26], "2018": [0, 6, 27], "2021": [6, 14, 27], "2022": 26, "2025": [18, 26, 27, 28], "21": [0, 1, 5, 7, 9, 12, 20, 26, 27, 28], "2116753732": 4, "215pm": [24, 26], "2167072": [], "22": [0, 1, 5, 12, 13, 20, 26, 27, 28], "221": 8, "225": 4, "22948497": [], "23": [1, 12, 20], "24": [0, 1, 20, 26], "25": [2, 3, 4, 5, 6, 8, 9, 11, 27], "250": [2, 4, 7, 9], "25000": [], "250154": [], "253775": [], "255": 3, "256": 4, "25x": 21, "26": [], "26303845": [], "264": [], "265": [], "265109911": 4, "266": [], "269": [], "27": 1, "270": [], "278": 28, "27n_": 23, "28": [1, 3, 4], "283": 28, "2830637392": 4, "2861": 23, "2873": 9, "2882": 23, "2886": 23, "2890": [0, 26], "2892": 23, "29": 27, "2915": 23, "2931": 26, "29364655": [], "294399745619595": [], "296247": [], "2968": 26, "2980": 26, "298273": [], "298375": [], "2990": 26, "2_": 12, "2_1": 12, "2_2": 12, "2_3": 12, "2_i": 12, "2_m": [6, 23], "2_t": 13, "2_x": 23, "2a": 17, "2b": 23, "2cm": 8, "2d": [1, 3, 11, 12, 19, 26], "2e": 6, "2f": [0, 7, 9, 10, 11, 12, 26], "2g": 2, "2g_i": 2, "2k": 3, "2m": 6, "2mvizaqfst8": 27, "2n": [0, 2, 3, 26, 27], "2nd": 9, "2p": 23, "2pt": 4, "2x": [0, 3, 8, 13, 26], "2x_ix_jy_iy_j": 8, "2x_j": 8, "2y_i": 10, "2y_j": 8, "3": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19, 20, 21, 22, 23, 24, 26, 28], "30": [0, 1, 4, 6, 7, 10, 13, 24], "30000": [0, 26], "3072": 3, "31": [12, 20, 23], "315": [6, 27], "3155": [0, 5, 6, 27, 28], "32": [3, 4, 6, 12, 13, 20, 23], "3200": 1, "3250": 1, "3297": [], "33": [12, 20, 24], "3303": [], "3310": [], "332331": [], "333": 7, "3331": [], "3337": [], "34": 20, "3436": [0, 26], "3437": [0, 26], "35": [0, 6, 21, 26, 28], "3581341341": 4, "359": [5, 28], "36": [0, 5, 6, 18, 21, 23], "37": [21, 28], "370782966": 4, "38": [21, 23], "39": [0, 24, 26], "3d": [2, 3, 4, 6, 13, 16], "3f": [1, 3, 9], "3n": 20, "3x": [2, 8], "3x_i": 2, "3y": 8, "4": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 23, 26, 28], "40": [1, 6, 24, 26], "400": 4, "4000": 26, "4050": [25, 26], "41": 20, "4155": [2, 15], "41589548": [], "42": [1, 4, 8, 9, 10, 20], "43": [0, 7, 20], "4310": 26, "436462435": 4, "44": [0, 20, 28], "45": [24, 26], "46": [24, 26], "462": 7, "47": [24, 26], "479465113": 4, "47958494": [], "48": [], "48257387": [24, 26], "49": [5, 6, 11], "49152": 3, "4940954": [0, 26], "4990": 23, "4992": 23, "4997": 23, "4c4c7f": [9, 10], "4d": 3, "4f": 6, "4pm": [24, 26], "4y": 8, "4y_i": 10, "5": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 20, 21, 23, 26, 27, 28], "50": [1, 2, 3, 4, 6, 7, 8, 10, 13, 26, 27], "500": [1, 3, 4, 6, 9, 10, 13], "5018": 23, "506": [], "507d50": [9, 10], "50j": 13, "50x10": 1, "51": 10, "510": 1, "512132": [], "5177783846": 4, "53": 9, "54": [6, 23], "5411205": [], "54894451": [], "55": 1, "56": 1, "56536": [0, 26], "569": 1, "57": [0, 8, 24, 26], "571": [5, 28], "58": [10, 24, 26], "591317992": 4, "5cm": 23, "5f": 8, "5x": [8, 18], "5y": 8, "6": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 18, 20, 23, 24, 26, 27, 28], "60": [1, 3], "60000": 4, "6019067271": 4, "606439": [], "625": 7, "63": 1, "64": [1, 3, 4, 13, 20, 26], "64x50": 1, "65": [1, 8, 9], "6887363571": 4, "69": [16, 23], "69069n_": 23, "691": [], "6n_": 23, "7": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 20, 21, 23, 25, 26, 27], "70": [1, 7], "70653767": 4, "71": 1, "724": 3, "73": [], "7304881": [], "75": [5, 6, 8, 11], "76": [24, 26], "765": 7, "77": [24, 26], "7718": 9, "7782028952": 4, "77893972": [], "78": [], "7d7d58": [9, 10], "8": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 18, 20, 23, 24, 26, 28], "80": [0, 1, 5, 8, 17, 27], "800": [4, 7], "81": 1, "815am": [24, 26], "85": 1, "8702784034": 4, "88": 26, "8f": 6, "8g": 6, "8n": 20, "8x8": 1, "9": [0, 1, 2, 4, 5, 6, 7, 8, 9, 11, 12, 13, 20, 23, 26], "90": 1, "9040": 9, "91": [24, 26], "92": [24, 26], "93": 16, "931": [0, 26], "933": [5, 28], "937": 23, "938": 23, "939": [0, 23, 26], "94": 23, "95": [1, 11], "954": 23, "955820c21e8b": 4, "96": 6, "960": 23, "961": 23, "962": 23, "9649652536": 4, "96611194e": [], "9780387310732": 25, "9780387848570": 25, "9781098134174": 26, "9781492032632": 25, "9781801819312": 26, "97898392": 27, "98": [0, 1, 16], "985": 23, "986": 23, "989": 23, "9898ff": [9, 10], "99": [13, 16], "991": 23, "992": 23, "993": 23, "996": 5, "999": [9, 23], "9x": 6, "9y": 6, "A": [2, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 19, 20, 22, 23, 24, 25, 27, 28], "AND": 2, "And": [0, 3, 4, 5, 6, 9, 13, 19, 21, 23, 28], "As": [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 20, 21, 23, 26, 27, 28], "At": [0, 4, 6, 13, 26], "BE": [0, 26], "Be": [2, 18, 19, 26], "Being": 13, "But": [0, 1, 2, 3, 5, 6, 9, 10, 16, 23, 27], "By": [0, 3, 5, 6, 12, 13, 17, 20, 26, 27, 28], "For": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 19, 20, 21, 23, 25, 26, 27, 28], "IF": [6, 27], "IN": 25, "If": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 23, 26, 27, 28], "In": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 25, 26, 27, 28], "Ising": [5, 12, 27, 28], "It": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 19, 20, 21, 23, 26, 27, 28], "Its": [1, 2, 4, 11], "No": [6, 9, 26, 27], "Not": [0, 1, 5, 6, 27, 28], "OR": 23, "Of": 23, "On": [0, 3, 21, 23, 24, 25, 26], "One": [0, 1, 3, 4, 5, 6, 7, 8, 11, 12, 13, 17, 23, 27, 28], "Or": [0, 1, 6, 26], "Such": [0, 6, 12, 16, 23], "That": [0, 5, 7, 10, 11, 12, 14, 21, 23, 26], "The": [4, 10, 13, 14, 16, 17, 18, 20, 21, 22, 23, 24, 25], "Then": [0, 1, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 26, 28], "There": [0, 3, 4, 5, 6, 8, 9, 11, 12, 14, 15, 20, 21, 23, 24, 26, 27, 28], "These": [0, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 17, 20, 21, 23, 24, 26, 27, 28], "To": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 20, 23, 27, 28], "With": [0, 5, 6, 8, 9, 10, 11, 12, 14, 16, 20, 21, 23, 26, 27], "_": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 20, 21, 26, 27, 28], "_0": [5, 8, 10, 11, 13, 27, 28], "_1": [2, 5, 6, 8, 10, 11, 12, 13, 14, 20, 27, 28], "_2": [2, 5, 8, 11, 12, 13, 20, 27], "_3": 20, "_4": 20, "_9": 13, "__class__": 10, "__doc__": 6, "__future__": [8, 9], "__init__": 1, "__main__": 2, "__name__": [2, 10], "_auto1": [2, 3, 4, 5, 6, 7, 12, 13, 20, 23, 27, 28], "_auto10": [6, 12], "_auto11": 6, "_auto12": 6, "_auto2": [2, 3, 4, 5, 6, 12, 13, 20, 23], "_auto3": [3, 4, 5, 6, 12, 13, 20], "_auto4": [4, 6, 12, 13, 20], "_auto5": [4, 6, 12, 13, 20], "_auto6": [4, 6, 12, 20], "_auto7": [4, 6, 12, 20], "_auto8": [6, 12], "_auto9": [6, 12], "_build": [0, 19, 21, 25, 26], "_c": 1, "_center": [], "_compon": 11, "_depth": 9, "_export": [15, 16], "_fraction": 9, "_i": [0, 1, 2, 5, 6, 7, 8, 11, 12, 13, 21, 26, 27, 28], "_j": [0, 1, 2, 3, 5, 6, 8, 13, 21, 27, 28], "_k": [13, 28], "_l": 12, "_lambda": 6, "_leaf": 9, "_m": 10, "_multilayer_perceptron": [], "_n": [2, 5, 8, 11, 13, 27, 28], "_node": 9, "_norm": [], "_p": [5, 8, 27, 28], "_ratio": 11, "_sampl": 9, "_split": [6, 9, 21], "_t": 13, "_test": [6, 21], "_varianc": 11, "_weight": 9, "a0": 3, "a0faa0": [9, 10], "a1": [0, 26], "a2": [0, 26], "a3": [0, 26], "a4": [0, 26], "a_": [0, 1, 16, 20, 26, 27], "a_0": [0, 26], "a_1a": [0, 26], "a_2a": [0, 26], "a_3": [0, 26], "a_3a": [0, 26], "a_4": [0, 26], "a_4a": [0, 26], "a_h": 1, "a_i": [0, 1, 2, 12, 26], "a_j": [1, 12], "a_k": [0, 1, 12], "aaron": 25, "ab": [0, 2, 5, 13, 14, 26, 27], "ab_channel": 19, "abandon": 1, "abid": 23, "abil": [0, 10], "abl": [0, 1, 4, 5, 6, 7, 10, 12, 13, 16, 18, 21, 27, 28], "about": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 19, 20, 21, 24], "abov": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 23, 25, 26, 27], "abovement": [6, 21, 26], "abscissa": [13, 28], "absolut": [0, 2, 5, 6, 13, 26, 27, 28], "absorb": [27, 28], "abstract": 1, "acceler": 13, "accept": [0, 3, 6, 9, 21, 27], "access": [3, 11, 23, 26], "accid": [4, 6], "accompani": [0, 26, 27], "accomplish": [8, 9, 13], "accord": [0, 1, 2, 5, 6, 9, 12, 13, 14, 23, 26, 28], "accordingli": 11, "account": [0, 3, 5, 13, 15, 16, 23, 26], "accumul": [12, 13, 23], "accur": [0, 3, 4, 6, 10, 13], "accuraci": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 12, 26, 27, 28], "accuracy_scor": [0, 1, 10, 26], "accuracy_score_numpi": 1, "achiev": [0, 1, 5, 6, 8, 12, 20, 26], "aco": 23, "acquaint": 19, "acquir": [1, 19, 26], "acr": [], "across": [1, 3, 6, 9, 17, 19, 26], "act": [1, 3, 20], "action": 23, "activ": [0, 2, 3, 4, 9, 15, 22, 24, 26], "actual": [0, 1, 4, 5, 6, 8, 11, 15, 16, 18, 20, 23, 26, 27, 28], "ad": [1, 3, 4, 5, 8, 13, 15, 16, 20, 28], "ada_clf": 10, "adaboostclassifi": 10, "adadelta": 13, "adagrad": 21, "adam": [1, 3, 4, 21, 26], "adapt": [4, 6, 13, 17, 25, 28], "add": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 15, 16, 17, 18, 23, 24, 26, 27, 28], "add_subplot": [1, 7, 12, 14], "addendum": 5, "addit": [0, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 19, 20, 21, 23, 24, 25, 26, 27], "addition": [12, 13, 28], "address": [1, 9, 11, 13, 26], "adjac": [3, 12], "adjoint": [5, 27], "adjust": [0, 5, 12, 13, 28], "admir": [0, 26], "advanc": [4, 6, 12, 25, 26], "advantag": [1, 3, 5, 6, 10, 13, 20, 28], "adversari": 26, "afecionado": 26, "affect": [3, 15], "affin": [0, 3, 8, 11, 27], "afford": 3, "aficionado": 26, "aforement": 14, "african": [], "after": [0, 1, 2, 4, 5, 6, 9, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 23, 26, 27, 28], "afterward": [0, 26], "ag": [0, 7, 26, 27], "ag_0": 2, "again": [0, 1, 4, 5, 6, 7, 8, 10, 11, 12, 13, 21, 23, 26, 27, 28], "against": [1, 4, 7, 10], "agegroup": 7, "agegroupmean": 7, "aggreg": [9, 10], "agorithm": 10, "agre": [5, 6, 23, 27, 28], "agreement": 13, "ahead": 9, "ai": [0, 25], "aid": 11, "aim": [0, 1, 4, 6, 7, 11, 14, 16, 17, 19, 20, 21, 27], "ainv": 5, "airplan": 3, "aka": 5, "al": [0, 2, 4, 16, 17, 25, 26, 27, 28], "alarm": [5, 7], "aldo": 27, "algebra": [0, 3, 5, 13, 19, 27, 28], "algorithm": [0, 1, 2, 4, 5, 6, 7, 8, 13, 14, 16, 19, 20, 21, 23, 25], "align": [0, 2, 5, 6, 7, 8, 13, 23, 26, 27, 28], "all": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "allevi": [1, 13, 28], "alloc": [3, 20], "allow": [0, 1, 2, 3, 5, 6, 8, 10, 13, 15, 19, 20, 21, 26, 27, 28], "almost": [0, 1, 6, 8, 11, 13, 23, 28], "alon": [2, 9], "along": [2, 3, 4, 5, 6, 9, 10, 11, 15, 19, 20, 26, 27, 28], "alpha": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 13, 14, 23, 26, 27, 28], "alpha_": 10, "alpha_0": 3, "alpha_1": 3, "alpha_2": 3, "alpha_i": [3, 13], "alpha_k": 13, "alpha_m": 10, "alpha_n": 3, "alpha_opt": 13, "alreadi": [2, 3, 4, 5, 6, 10, 12, 15, 19, 20, 23, 26, 27, 28], "also": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 26, 27, 28], "alter": 1, "altern": [0, 1, 4, 5, 6, 8, 9, 11, 13, 15, 18, 20, 21, 26, 27], "although": [0, 1, 5, 6, 8, 10, 13, 16, 26], "alwai": [0, 3, 5, 6, 12, 13, 16, 21, 23, 26, 27, 28], "am": 4, "ame2016": [0, 26], "american": [], "among": [0, 3, 5, 9, 10, 12, 20, 26, 27], "amongst": 5, "amount": [0, 1, 3, 4, 6, 8, 10, 14, 19], "an": [1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 27, 28], "an_": 23, "anaconda": [0, 1, 19, 21, 26], "analogi": 13, "analys": 6, "analysi": [1, 3, 4, 7, 14, 20, 25], "analyt": [2, 3, 5, 6, 7, 12, 13, 17, 19, 21, 26, 27, 28], "analyz": [0, 1, 3, 4, 5, 6, 16, 21, 23, 27, 28], "andrew": 1, "angl": [0, 3, 9, 27], "anharmon": 3, "ani": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 23, 26, 27], "anim": [4, 12], "ann": 12, "annot": [0, 1, 3, 7, 8, 26], "announc": 26, "anoth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 20, 21, 23, 26, 27], "ansatz": [0, 18, 26], "answer": [0, 1, 3, 5, 6, 20, 21, 24, 26], "antialias": [2, 6], "anticip": 4, "anymor": [1, 8], "anyon": [4, 8, 15], "anyth": [1, 15, 16, 23], "anytim": [24, 26], "apach": 1, "apart": [11, 13, 28], "api": [1, 19, 26], "appar": 2, "appear": [0, 1, 3, 13, 20, 23], "append": [1, 3, 4, 8, 9, 13, 26], "appendix": 21, "appli": [0, 1, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 18, 21, 23, 25, 26, 27], "applic": [0, 1, 3, 4, 5, 6, 7, 9, 12, 13, 16, 20, 23, 25, 26, 27, 28], "apply_gradi": 4, "approach": [1, 2, 4, 5, 6, 9, 10, 11, 12, 13, 15, 16, 18, 19, 21, 23, 25, 27, 28], "approch": 21, "appropri": [2, 6, 9, 12, 13, 17, 19, 23], "approv": 26, "approx": [0, 2, 3, 6, 10, 11, 13, 18, 21, 23, 26, 28], "approxim": [0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 13, 21, 23, 26, 27, 28], "apt": [0, 19, 21, 26], "aq": 23, "ar": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], "aragorn": 26, "arang": [1, 3, 4, 6, 7, 9, 10, 12, 13, 26], "arbitrari": [1, 4, 6, 8, 12, 13, 23, 28], "arbitrarili": [0, 1, 11, 26], "arc": 6, "architectur": [3, 4, 12], "archiv": 21, "area": [0, 3, 6, 25, 26], "argmax": [1, 11], "argmin": [4, 10, 14], "argsort": 11, "argu": [1, 13], "argument": [0, 2, 3, 5, 11, 12, 13, 17, 26, 27], "aris": [0, 6, 12, 13, 23, 26, 28], "arithmet": [0, 13, 20, 26], "arm": [6, 27], "armadillo": 20, "around": [0, 1, 4, 5, 6, 11, 18, 21, 23, 26], "arrai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 16, 18, 19, 21, 23, 27, 28], "arrang": [3, 26], "arraybox": 13, "arriv": [0, 6, 9, 11, 20, 23, 26], "arrow": 12, "arrowprop": 8, "art": [0, 1, 19], "articl": [0, 3, 4, 6, 10, 26, 27, 28], "artifici": [0, 2, 7, 12, 25, 26], "artificialneuron": 12, "arug": 13, "arxiv": [3, 4], "asarrai": [0, 6, 9, 27], "asid": 27, "ask": [5, 6, 11, 12, 15, 21], "aspect": [0, 6, 19, 26, 27], "assembl": 3, "assembli": [0, 26], "assert": 4, "assess": [0, 6, 21, 26, 27], "assici": 4, "assign": [0, 7, 8, 9, 12, 13, 14, 15, 22, 24, 25, 26], "associ": [0, 6, 9, 12, 14, 23, 26], "assum": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 20, 21, 23, 26, 27, 28], "assumpt": [0, 3, 5, 6, 9, 11, 23, 26, 27], "ast": [0, 5, 6, 26], "astyp": [4, 9, 10], "asymmetri": [0, 26], "asymptot": [4, 6], "atom": [0, 26], "attempt": [0, 4, 6, 7, 8, 10, 26, 27], "attend": 26, "attent": [0, 20, 26], "attract": [0, 10, 26], "attribut": [0, 9, 26], "audi": [0, 26], "audio": [3, 4], "august": [26, 27], "aurelien": [0, 25, 26], "austfjel": 6, "auth": 15, "authent": 15, "author": [0, 1, 10, 23], "authour": 26, "auto": [9, 10, 23], "auto_exampl": [21, 27], "autocor": 23, "autocorrelation_tim": 23, "autocorrelform": 23, "autocovari": 23, "autoencod": [4, 19, 26], "autoencond": 19, "autograd": [19, 26], "autom": [0, 19, 25, 26], "automac": 20, "automag": 26, "automat": [0, 1, 2, 3, 4, 11, 16, 19, 20, 26], "automobil": 3, "autonom": 4, "avail": [0, 1, 4, 6, 10, 11, 19, 20, 21, 22, 24, 25, 26], "averag": [0, 1, 3, 6, 9, 10, 13, 14, 23, 24, 26, 27], "avoid": [0, 4, 5, 6, 9, 11, 13, 18, 20, 27], "awai": [2, 3, 6, 27], "awar": [2, 10], "award": [24, 26], "ax": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 26], "axes3d": [2, 6, 13, 28], "axes_grid1": 6, "axhlin": 8, "axi": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 20, 23, 26, 27, 28], "axiom": 5, "axvlin": [4, 8], "axvspan": 4, "b": [0, 1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 23, 24, 26, 27, 28], "b1": 8, "b2": 8, "b3": 8, "b_": [0, 1, 20], "b_0": 0, "b_1": [0, 2, 12, 13], "b_2": [0, 13], "b_5": 13, "b_group": 9, "b_i": [0, 1, 2, 12, 26], "b_ia_": [0, 26], "b_ia_i": 0, "b_index": 9, "b_j": [1, 12], "b_k": [0, 1, 12, 13], "b_m": 12, "b_score": 9, "b_valu": 9, "babcock": 26, "bachelor": [22, 24], "back": [0, 3, 4, 5, 6, 8, 9, 10, 15, 16, 20, 23, 26], "backbon": 20, "backend": [1, 4], "background": [25, 26], "backpropag": 1, "backtrack": 9, "backup": 20, "backward": [1, 2, 4, 12, 20], "bad": [6, 17, 27], "badli": 23, "bag": [9, 19, 26], "bag_clf": 10, "baggin": 26, "baggingboot": 10, "baggingclassifi": 10, "baggingtre": 10, "balanc": 6, "ballpark": 18, "band": 20, "bandwidth": 20, "bar": [0, 6, 11, 21, 26], "barber": 25, "bare": [4, 10], "base": [0, 1, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17, 19, 23, 24, 25, 26, 27, 28], "basi": [5, 7, 8, 10, 11, 12, 13, 20, 27, 28], "basic": [6, 8, 12, 13, 14, 15, 19, 21, 23, 26], "batch": [3, 4, 11, 12, 13, 28], "batch_shap": 4, "batch_siz": [1, 3, 4], "batchnorm": 4, "bay": 7, "bayesian": [5, 19, 25, 26], "becaus": [0, 1, 2, 3, 4, 5, 6, 8, 9, 12, 13, 14, 26, 27, 28], "becom": [0, 1, 2, 5, 6, 7, 9, 12, 13, 23, 26, 27, 28], "been": [0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 19, 20, 21, 26, 27], "befor": [0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 13, 14, 16, 17, 18, 20, 21, 23, 26, 27], "beforehand": [0, 23, 26], "begin": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 20, 23, 24, 26, 27, 28], "behav": [1, 6, 13, 28], "behavior": [0, 1, 13, 26, 28], "behaviour": 12, "behind": [0, 1, 6, 8, 13, 26, 28], "being": [0, 1, 2, 3, 4, 5, 7, 8, 10, 11, 12, 13, 17, 23, 26, 27, 28], "believ": [9, 20], "belong": [7, 8, 9, 13, 14, 28], "below": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 18, 20, 21, 23, 26, 27, 28], "benchmark": 10, "benefici": [1, 13], "benefit": [0, 1, 4, 11, 13, 19, 26, 28], "bengio": [1, 25, 26, 27], "benign": [1, 7], "besid": [4, 5, 28], "bessel": [5, 27], "best": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 18, 24, 26, 27, 28], "beta": [1, 3, 10, 11, 13, 16, 17, 26, 27, 28], "beta_": [3, 13, 17, 27], "beta_0": [1, 3, 13, 27], "beta_1": [1, 3, 10, 13, 27], "beta_1x_i": 13, "beta_2": [3, 13], "beta_3": 3, "beta_i": 3, "beta_j": [13, 27], "beta_k": 13, "beta_linreg": 13, "beta_m": 10, "beta_mg_m": 10, "beta_n": 3, "better": [0, 1, 2, 3, 4, 6, 9, 10, 11, 12, 13, 26, 27], "between": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 21, 23, 26, 27, 28], "beyond": [0, 1, 5, 6, 8, 13, 26, 27, 28], "bf": [13, 14, 20, 23, 28], "bg": 26, "bgd": 13, "bia": [0, 1, 2, 3, 5, 8, 9, 10, 12, 13, 26, 27, 28], "bias": [1, 2, 3, 5, 6, 9, 12], "bibliographi": 21, "big": [0, 1, 2, 5, 6, 14], "bigger": [1, 6, 27], "bigr": 12, "bike": 9, "bilbo": 26, "billion": [3, 12, 19], "bin": [7, 23], "binari": [0, 3, 5, 7, 9, 10, 12, 26], "binarycrossentropi": 4, "bind": 0, "binomi": [19, 23, 26], "binsboot": 6, "bioinformat": 0, "biolog": [1, 12], "bios1100": [19, 26], "bird": [0, 3], "birth": 26, "bishop": [25, 26], "bit": [1, 4, 20, 23, 26], "bitwis": 23, "bivari": 2, "bk": 13, "bla": [20, 26], "black": [8, 9, 14], "block": [6, 10, 19, 20, 23, 26], "blog": 26, "blogpost": 4, "blue": [0, 3], "bm": [], "bmatrix": [0, 1, 3, 5, 7, 8, 11, 13, 20, 26, 27, 28], "bmi": 1, "bodi": [0, 1, 4, 12], "bold": 1, "boldfac": [0, 5, 16, 27, 28], "boldsymbol": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 13, 14, 16, 17, 21, 26, 28], "boltzmann": [12, 19, 26], "book": [17, 21, 25, 26, 27], "book1": 25, "boolean": [4, 17], "boost": [1, 9, 19, 26], "boostrap": 10, "bootstrap": [1, 13, 19, 21, 26], "borrow": 26, "boston_dataset": [], "bot": 8, "both": [0, 1, 4, 5, 6, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 26, 27, 28], "bottl": 7, "bound": [8, 12], "boundari": [2, 4, 8, 11, 12], "box": [4, 9], "boyd": [8, 13, 28], "bracket": [4, 23], "brain": [1, 7, 12], "branch": [9, 26], "break": [0, 4, 6, 11, 14, 26], "breast": [5, 7, 11], "breviti": 13, "brew": [0, 19, 21, 26], "brg": 8, "brief": [21, 27], "briefli": [0, 16, 26], "bring": [0, 5, 6, 10, 27], "britt": [24, 26], "broad": 0, "broadli": 26, "brought": [13, 19, 26], "brownle": 4, "browser": [15, 26], "brute": [3, 5, 11, 27], "buffer_s": 4, "bui": 4, "build": [0, 4, 5, 6, 10, 16, 20, 23, 26], "built": [1, 3, 4, 6], "bunch": 11, "busi": [], "byte": [20, 26], "c": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 22, 23, 24, 25, 27, 28], "c1": [8, 11], "c2": [8, 11], "c_": [8, 9, 10, 13, 23, 28], "c_0": 23, "c_1": 12, "c_2": 12, "c_3": 12, "c_4": 12, "c_i": [12, 13], "c_k": 23, "ca": [1, 26], "cach": 10, "cal": [0, 8, 10, 12, 13, 28], "calcul": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 16, 20, 23, 26], "california": 21, "call": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 24, 26, 27, 28], "calor": [0, 27], "cambridg": [13, 25, 28], "can": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 27, 28], "cancel": [0, 13, 26, 27], "cancer": [5, 10], "cancerpd": 7, "candid": [8, 9, 10], "cannot": [0, 1, 4, 5, 6, 7, 8, 9, 21, 23, 27, 28], "canopi": [0, 19, 21, 26], "canva": [15, 16, 21, 26], "cap": 5, "capabl": [0, 1, 8, 13, 19, 26], "capac": [2, 24], "capita": [], "caption": 21, "captur": [4, 11, 12, 26], "car": [3, 4], "card": [0, 7, 26], "cardin": 1, "care": [11, 15], "carefulli": 13, "carlo": [0, 6, 19, 23, 25, 26], "carri": [2, 6, 7, 21], "cart": 10, "case": [0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 19, 20, 21, 26], "casella": 25, "cast": 1, "cat": [3, 4], "catch": 0, "categor": [0, 1, 3, 9, 11, 26], "categori": [0, 1, 3, 7, 10, 12, 14, 26], "categorical_crossentropi": [1, 3], "caus": [0, 5, 6, 23, 26, 27, 28], "causal": 0, "causat": [0, 26], "cax": 1, "cb": [6, 26], "cbar": 1, "cc": [0, 1, 5, 13, 26, 27, 28], "ccc": [5, 12, 28], "cdf": 23, "cdot": [0, 2, 6, 12, 13, 14, 20, 23, 26, 28], "celebr": [13, 28], "cell": 4, "center": [0, 1, 6, 7, 8, 9, 11, 14, 18, 21, 23, 26, 27], "central": [0, 3, 5, 6, 8, 16, 20, 26, 27], "centroid": [14, 23], "centroid_differ": 14, "centuri": 3, "certain": [0, 3, 6, 7, 9, 23, 26, 27], "cg": 13, "cha": [], "chain": [0, 1, 13, 19, 23, 26], "challeng": 15, "chanc": [1, 5, 13, 23], "chang": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 15, 16, 20, 21, 23, 26, 27, 28], "channel": 3, "chapter": [0, 6, 10, 11, 16, 17, 20, 21, 25, 26, 27, 28], "chapter3": [0, 21], "charact": [0, 3, 5, 26, 27, 28], "character": [8, 9, 10, 12, 23], "characterist": [0, 1, 3, 10, 13, 26], "charg": [0, 26], "charl": [], "chase": 4, "chatgpt": [15, 21], "chd": 7, "chddata": 7, "cheap": [5, 27, 28], "cheaper": [1, 13], "check": [1, 3, 4, 5, 11, 13, 15, 16, 20, 26], "checkmark": 3, "checkpoint": 4, "checkpoint_dir": 4, "checkpoint_prefix": 4, "chen": 10, "cheng": 27, "chiaramont": 2, "childcar": 16, "children": 16, "choic": [0, 1, 2, 3, 4, 6, 9, 12, 13, 14, 20, 26, 27, 28], "choleski": [5, 20, 27, 28], "choos": [2, 3, 6, 9, 10, 11, 13, 14, 15, 18, 21, 28], "chosen": [0, 1, 2, 6, 8, 9, 10, 13, 16, 23, 26, 28], "chosen_datapoint": 1, "christian": 25, "christoph": [25, 26], "cifar": 3, "cifar10": 3, "circ": [1, 12], "circl": [0, 8, 12, 27], "circuit": 3, "circumfer": 9, "circumv": [1, 5, 13, 27, 28], "cite": 21, "ckpt": 4, "clariti": 23, "class": [0, 1, 3, 4, 6, 7, 8, 9, 11, 12, 13, 23, 26], "class_nam": [3, 9], "class_val": 9, "class_valu": 9, "classic": [7, 9, 13], "classif": [0, 3, 5, 6, 7, 8, 11, 12, 19, 21, 25, 26, 27], "classifi": [0, 1, 4, 7, 9, 10, 11, 26], "classificaton": 1, "classifii": 10, "clean": 1, "clear": [1, 5, 10, 12, 13], "clearli": [0, 3, 5, 6, 7, 8, 23, 27, 28], "clever": [1, 10], "clf": [0, 6, 8, 9, 10, 26, 27], "clf3": 0, "clf_lasso": 6, "clf_ridg": 6, "cli": 15, "clip": [3, 23], "clone": [15, 24], "close": [0, 1, 2, 4, 6, 8, 9, 11, 12, 13, 14, 18, 23, 25, 26, 28], "closer": [3, 5, 13, 27, 28], "closest": [8, 11, 13, 14], "closur": [19, 26], "cloud": [19, 26], "cluster": [0, 1, 4, 6, 11, 19, 26], "cluster_label": 14, "cm": [1, 2, 3, 6, 8, 13, 28], "cmap": [0, 1, 2, 3, 4, 6, 8, 9, 10, 26], "cmap_arg": 6, "cmd": [9, 15], "cn_": 23, "cnn": 12, "cnn_kera": 3, "cntk": [19, 26], "co": [0, 2, 3, 6, 9, 13, 26], "code": [0, 3, 4, 6, 7, 8, 18, 19, 20, 23, 25], "coef": [0, 26], "coef0": 8, "coef_": [0, 5, 6, 8, 9, 13, 16, 26, 27, 28], "coeff": 5, "coeffici": [0, 3, 5, 6, 7, 8, 9, 13, 18, 20, 26, 27], "coerc": [0, 6, 26], "coin": [10, 23], "coin_toss": 10, "col": [0, 11, 26, 27], "colab": [19, 26], "cold": 9, "colinear": [], "collabor": 21, "collaps": 8, "collect": [2, 6, 10, 11, 17, 19, 23, 25, 26], "collinear": [5, 27, 28], "color": [0, 3, 4, 6, 8, 9, 10, 23], "color_channel": 3, "color_cod": 6, "colorbar": [1, 6], "colsample_bytre": 10, "colsaobject": 10, "column": [0, 1, 2, 5, 6, 7, 8, 9, 11, 12, 16, 17, 18, 20, 26, 27, 28], "columntransform": 9, "com": [4, 6, 15, 16, 19, 21, 25, 26, 28], "combin": [1, 2, 5, 6, 7, 10, 15, 18, 23], "come": [0, 1, 3, 4, 5, 12, 13, 14, 15, 26, 27, 28], "command": [0, 1, 15], "comment": [0, 4, 5, 6, 21], "commerci": [0, 19, 21, 26], "commit": 15, "commod": [0, 26], "common": [0, 1, 3, 5, 6, 7, 9, 11, 13, 14, 16, 21, 23, 26, 27, 28], "commonli": [0, 1, 4, 6, 7, 9, 13, 14, 27], "commun": [0, 12, 15, 21], "commut": 3, "commutatitav": 3, "compact": [0, 1, 3, 5, 6, 7, 9, 11, 12, 13, 14, 26, 27], "compair": 0, "compar": [0, 3, 4, 5, 6, 11, 13, 18, 20, 21, 26, 27, 28], "comparison": [2, 4, 13], "compat": 7, "compet": 0, "competit": 10, "compil": [0, 1, 3, 4, 13, 19, 20, 26], "complet": [0, 2, 3, 4, 9, 12, 15, 16, 17, 18, 26], "completenn": 12, "complex": [1, 5, 8, 9, 11, 12, 13, 16, 26, 28], "complic": [0, 1, 9, 13, 21, 26, 28], "compoment": 27, "compon": [0, 1, 3, 4, 5, 6, 7, 9, 14, 16, 19, 26, 27, 28], "components_": 11, "compos": [9, 12, 13, 14, 19, 26], "compphys": [0, 6, 16, 19, 21, 22, 24, 25, 26, 27, 28], "compress": [0, 26, 27], "compris": 6, "compromis": [5, 27, 28], "compulsori": [19, 26], "comput": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28], "computation": [0, 3, 6, 9, 13, 23, 26, 28], "computationalscienceuio": 26, "computerlab": 21, "concaten": [2, 4, 6, 14], "concav": [1, 13, 27, 28], "concentr": 10, "concept": [0, 2, 19, 26, 27], "conceptu": [12, 13, 28], "concern": [0, 1, 4, 7, 26, 28], "concic": 26, "conclud": [0, 5, 13], "conclus": 1, "cond": 2, "conda": [0, 1, 19, 21, 26], "condis": 27, "condit": [0, 2, 4, 5, 6, 8, 9, 11, 13, 23, 26, 27], "conduct": 19, "condwav": 2, "confid": [0, 5, 6, 7, 8, 26, 27], "configur": 3, "confirm": [5, 12], "confus": [5, 6, 7, 10, 20, 27], "confusion_matrix": 9, "congruenti": 23, "conjug": [4, 8], "conjugaci": 13, "conjunct": 3, "connect": [0, 1, 3, 4, 9, 11, 12, 13, 20, 26, 27, 28], "consequ": [5, 6, 8, 10, 12, 13, 27, 28], "conserv": [5, 14, 27, 28], "consid": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 16, 20, 21, 23, 26, 27, 28], "consider": [0, 1, 5, 13, 26, 27, 28], "consist": [1, 2, 3, 4, 6, 12, 13, 21, 23, 27, 28], "constant": [0, 2, 4, 5, 6, 8, 12, 13, 16, 18, 23, 26, 27, 28], "constitu": [0, 26], "constitut": [2, 6], "constrain": [1, 3, 5, 7, 11, 28], "constraint": [5, 6, 8, 13, 27, 28], "construct": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 20, 23, 26, 27], "contact": [0, 26], "contain": [0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 18, 20, 21, 23, 25, 26, 27, 28], "contemporari": 26, "content": [1, 15, 19, 20, 26, 28], "context": [6, 10, 13, 21, 28], "contigu": 20, "continu": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 20, 21, 23, 26, 27, 28], "contour": [9, 10, 13], "contourf": [8, 9, 10], "contrast": [1, 4, 9, 10, 12, 26], "contribut": [0, 3, 5, 13, 18, 23, 26, 27, 28], "contributor": [0, 21], "control": [0, 1, 3, 9, 13, 15, 19, 26], "conv": [3, 4], "conv2d": [3, 4], "conv2dtranspos": 4, "convei": 26, "conveni": [5, 6, 12, 13, 20, 21, 26, 28], "convent": [12, 27], "converg": [1, 2, 4, 5, 8, 13, 14, 18, 27, 28], "convergencewarn": [], "convert": [0, 1, 4, 5, 9, 11, 13, 20, 26, 27, 28], "converttomatrix": 4, "convex": [4, 5, 7, 27], "convinc": [13, 28], "convolut": [1, 4, 19, 26], "cool": [4, 9], "coolwarm": 6, "coordin": [5, 12, 14, 27, 28], "coorel": [], "copi": [0, 1, 14, 15, 27], "core": 10, "corel": 26, "coronari": 7, "corr": [5, 7, 11, 27], "correalt": [11, 19], "correct": [0, 1, 2, 3, 4, 5, 7, 13, 15, 20, 23, 26, 27, 28], "correctli": [1, 2, 6, 7, 10, 18, 21], "correl": [0, 1, 3, 5, 6, 7, 10, 12, 13, 19, 23, 26, 28], "correlation_matrix": [5, 7, 11, 27], "correspond": [0, 3, 5, 6, 8, 9, 11, 12, 19, 20, 21, 23, 26, 27, 28], "cortex": 12, "cosin": [3, 6], "cost": [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 16, 17, 18, 21, 26], "cost_deep_grad": 2, "cost_funct": 2, "cost_function_deep": 2, "cost_function_deep_grad": 2, "cost_function_grad": 2, "cost_grad": 2, "cost_histori": 18, "cost_ol": 18, "cost_ridg": 18, "cost_sum": 2, "costol": 13, "could": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23, 26, 27, 28], "coulomb": [0, 26], "count": [0, 9, 15, 22, 23, 24, 26], "counterpart": 26, "countor": 13, "coupl": [4, 5, 6], "cours": [0, 1, 3, 5, 11, 15, 16, 17, 21, 24, 27], "coursework": 15, "courvil": [25, 26, 27], "cov": [5, 6, 11, 20, 23, 26, 27], "cov_xi": [5, 11, 27], "cov_xx": [5, 11, 27], "cov_yi": [5, 11, 27], "covari": [0, 7, 19, 20, 26, 28], "covariance_matrix": [5, 11, 14], "cover": [0, 5, 19, 24, 25, 27, 28], "covert": [0, 26], "covxi": 23, "covxx": 23, "covxz": 23, "covyi": 23, "covyz": 23, "covzz": 23, "cpu": 1, "craft": 3, "creat": [1, 3, 4, 5, 9, 10, 11, 12, 15, 18, 19, 26], "create_biases_and_weight": 1, "create_convolutional_neural_network_kera": 3, "create_neural_network_kera": 1, "create_x": [5, 11], "credit": [0, 7, 24, 26], "crim": [], "crime": [], "criteria": [0, 4, 9, 10, 14, 23, 26], "criterion": [9, 10, 13, 18, 28], "critic": [6, 21, 27], "critiqu": 21, "cross": [0, 1, 3, 7, 9, 10, 13, 15, 19, 23, 26, 27, 28], "cross_entropi": 4, "cross_val_scor": 6, "cross_valid": [7, 10], "crossvalid": 6, "crucial": [1, 23], "cs231": 3, "csr_matrix": [20, 26], "csv": [0, 4, 6, 7, 9], "ctnk": 1, "cubic": 0, "cumbersom": 5, "cumsum": [10, 11, 26], "cumul": [7, 10, 23], "cumulative_heads_ratio": 10, "cup": 5, "current": [1, 2, 3, 4, 13, 14, 15, 16, 25, 28], "curs": [0, 27], "curv": [6, 7, 10, 12, 21], "curvatur": [13, 28], "custom": [6, 14], "custom_cmap": [9, 10], "custom_cmap2": [9, 10], "cutpoint": 9, "cv": [6, 7, 10], "cvxbook": [13, 28], "cvxopt": [5, 8, 27], "cycl": [1, 12], "d": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 20, 23, 24, 26, 27, 28], "d2_g_t": 2, "d_f": [13, 28], "d_g_t": 2, "d_net_out": 2, "da": 3, "dagger": [5, 20, 27, 28], "dai": [1, 9, 19], "damp": 3, "darget": 9, "darkr": 23, "dat": [0, 26], "dat_id": [0, 6, 7, 9, 26], "data": [2, 4, 5, 8, 10, 12, 13, 14, 16, 20, 21, 25, 28], "data1": 14, "data2": 14, "data3": 14, "data4": 14, "data_id": [0, 6, 7, 9, 26], "data_indic": 1, "data_panda": 26, "data_path": [0, 6, 7, 9, 26], "databas": 1, "datafil": [0, 6, 7, 9, 26], "datafram": [0, 4, 5, 7, 9, 11, 26, 27], "datapoint": [1, 5, 6, 7, 11, 13, 16, 28], "datasci": [15, 16], "dataset": [0, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 21, 26, 28], "datatyp": 4, "date": [15, 18, 21, 26, 27, 28], "daughter": 10, "david": 25, "dbh": 1, "dbo": 1, "dcomposit": 20, "ddot": 2, "dead": 1, "deadlin": 15, "deal": [0, 1, 3, 5, 6, 8, 11, 13, 14, 20, 23, 26, 27, 28], "dealt": 0, "debt": 7, "debug": [0, 5, 6, 27, 28], "decad": [0, 3], "decai": [0, 13, 23, 26], "decemb": [24, 26], "decent": 10, "decid": [0, 2, 3, 5, 6, 9, 18, 27, 28], "decim": [0, 26], "decis": [0, 1, 8, 11, 19, 25, 26], "decision_funct": 8, "decision_tre": 9, "decisiontreeclassifi": [9, 10], "decisiontreeregressor": [0, 9, 10], "declar": [0, 4, 20, 26], "decompos": [5, 6, 20, 27, 28], "decomposit": [0, 6, 12, 26], "decompost": [5, 27, 28], "deconvolut": 3, "decorrel": [10, 13], "decreas": [1, 2, 4, 5, 6, 10, 11, 13, 28], "deduc": [0, 26], "deep": [3, 7, 12, 13, 19, 25, 27, 28], "deep_neural_network": 2, "deep_param": 2, "deep_tree_clf": [9, 10], "deep_tree_clf1": 9, "deep_tree_clf2": 9, "deepen": [5, 19, 26], "deeper": [0, 3, 4, 26], "deeplearningbook": [25, 26, 28], "deer": 3, "def": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 23, 26, 27, 28], "def_covari": 23, "default": [0, 1, 2, 4, 6, 7, 20, 26, 27], "default_tim": 4, "defect": [5, 27, 28], "defici": [5, 27, 28], "defin": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 23, 27, 28], "definit": [1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 20, 23, 27, 28], "defint": 23, "degre": [3, 5, 6, 8, 9, 10, 11, 15, 16, 21, 23, 26, 28], "deisenroth": 27, "del": 1, "delet": [6, 15], "delimit": 4, "deliv": [15, 22, 26], "delta": [0, 2, 3, 6, 8, 12, 13, 14, 26], "delta_": [1, 20], "delta_0": 3, "delta_1": 3, "delta_2": 3, "delta_3": 3, "delta_4": 3, "delta_5": 3, "delta_h": [0, 1, 26], "delta_j": [3, 12], "delta_k": 12, "delta_l": [1, 3], "delta_momentum": 13, "delta_n": [0, 3, 26], "delug": 19, "delv": 0, "demand": [13, 28], "demonstr": [0, 3, 5, 6, 7, 11, 12, 19, 26, 27, 28], "den": 4, "denomin": [1, 5], "denot": [1, 2, 6, 7, 13, 23, 28], "dens": [1, 3, 4], "densiti": [0, 2, 6, 23], "depart": [24, 26, 27, 28], "depend": [0, 1, 2, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 19, 20, 21, 23, 26, 27, 28], "depict": 23, "deploy": [0, 19, 21, 26], "depth": [0, 3, 9, 10, 20], "deriv": [0, 1, 2, 6, 7, 8, 10, 11, 13, 18, 19, 21, 26], "derivati": 13, "derivative_fn": 13, "descend": [5, 9, 11, 27, 28], "descent": [0, 1, 3, 7, 8, 12, 26, 27], "describ": [0, 2, 4, 5, 6, 8, 10, 11, 12, 13, 20, 21, 26], "descript": [0, 8, 9, 21, 26], "design": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 17, 18, 21, 26, 28], "designmatrix": [0, 26], "desir": [0, 2, 4, 5, 13, 14, 26, 27, 28], "desktop": 15, "despit": [1, 12], "destroi": 20, "det": [5, 20, 27, 28], "detail": [0, 6, 11, 13, 14, 18, 20, 21, 27, 28], "detect": [3, 8, 12], "determin": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 18, 20, 23, 26, 27, 28], "determinist": [7, 13, 23, 28], "dev": [1, 21], "develop": [0, 3, 5, 8, 10, 11, 12, 19, 20, 21, 26, 27], "deviat": [0, 1, 2, 4, 5, 6, 17, 18, 21, 23, 26, 27], "devis": 12, "df": [4, 8, 11, 13, 26], "df1": 26, "di": [], "diag": [5, 8, 27, 28], "diagnost": [1, 10], "diagon": [0, 5, 7, 13, 18, 20, 23, 26, 27, 28], "diagonaliz": [5, 27, 28], "diagram": 10, "diagsvd": 6, "dice": [6, 23], "dict": [6, 8], "dictionari": [], "did": [0, 1, 5, 6, 7, 10, 11, 14, 16, 21, 26], "die": 1, "diff": 2, "diff1": 2, "diff2": 2, "diff_ag": 2, "diffeent": 8, "differ": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28], "differenti": [0, 3, 16, 19, 20, 26, 27, 28], "difficult": [0, 1, 6, 10, 13, 23, 26], "difficulti": [0, 1, 13, 26, 28], "diffonedim": 2, "digit": [0, 1, 3, 4, 6, 24, 26], "dilemma": 13, "dilut": 1, "dim": [4, 11, 14, 20], "dimens": [0, 1, 2, 3, 4, 5, 8, 11, 14, 16, 20, 26, 27, 28], "dimension": [0, 4, 5, 6, 9, 11, 13, 14, 19, 20, 21, 26, 27, 28], "dimensionless": [0, 3, 26], "diment": 20, "dimnsion": 4, "diod": 3, "direct": [0, 1, 2, 4, 11, 12, 13, 14, 26, 27, 28], "directli": [1, 4, 5, 6, 18, 23, 27, 28], "disadvantag": [0, 26], "disappear": [3, 6], "disc_loss": 4, "disc_tap": 4, "discard": [6, 11], "disciplin": [0, 3, 12], "disclaim": 23, "discord": 26, "discourag": [13, 15, 28], "discov": [0, 26], "discover": 5, "discret": [1, 3, 5, 7, 13], "discrimin": [4, 7, 10, 11], "discriminator_loss": 4, "discriminator_loss_list": 4, "discriminator_model": 4, "discriminator_optim": 4, "discuss": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 23, 25, 26, 27, 28], "diseas": 7, "disguis": [6, 27], "disord": [1, 7], "displai": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 21, 23, 26, 27], "displaystyl": [0, 5, 17, 26, 27, 28], "disregard": [0, 26], "dissimilar": [11, 14], "dist": 14, "distanc": [8, 9, 11, 14, 23], "distance_list": 9, "distinct": [3, 7, 8, 9, 10, 14], "distinctli": 8, "distinguish": [0, 4, 7, 8, 23, 26], "distplot": [], "distribut": [0, 1, 4, 6, 7, 10, 11, 13, 14, 18, 19, 20, 21, 26, 27, 28], "distrubut": [0, 19, 21, 26], "dive": [0, 8, 20, 26], "diverg": [1, 13, 28], "divid": [0, 1, 3, 5, 6, 7, 8, 9, 11, 12, 18, 23, 26, 27], "divis": [6, 8, 9, 13, 18, 20, 23], "dna": 7, "dnn": [0, 1, 2, 4, 12, 26], "dnn1": 4, "dnn2_gru2": 4, "dnn_kera": 1, "dnn_model": 1, "dnn_numpi": 1, "dnn_scikit": [0, 1, 26], "do": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 21, 26, 27, 28], "doc": [0, 15, 16, 19, 21, 22, 24, 25, 26], "document": [4, 13, 15], "doe": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23, 26], "doesn": [3, 9, 12, 26], "dog": [1, 3, 4], "domain": [5, 8, 13, 21, 28], "domin": [0, 26], "don": [0, 1, 3, 5, 6, 8, 11, 13, 15, 16, 19, 21, 26, 27], "done": [0, 2, 3, 4, 5, 6, 9, 10, 11, 13, 16, 20, 21, 26, 27, 28], "dot": [0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 18, 20, 21, 23, 26, 27, 28], "doubl": [3, 4, 16, 20, 26], "doubli": 1, "doubt": 21, "down": [0, 3, 6, 9, 11, 12, 13, 28], "download": [0, 1, 3, 5, 6, 15, 20, 25, 26], "downsampl": 3, "dozen": 1, "dq": 6, "drag": 13, "dramat": 11, "drastic": 4, "draw": [4, 6, 10, 13, 28], "drawback": [0, 1, 3, 13, 27, 28], "drawn": [1, 4, 6, 7, 11, 23, 26], "drive": [3, 4], "driven": 3, "drop": [0, 1, 5, 6, 11, 13, 23, 26, 27, 28], "dropna": [0, 6, 26], "dropout": 4, "dt": [2, 3, 13, 23], "dtype": [0, 1, 3, 4, 14, 20, 26], "dub": [0, 26], "due": [1, 2, 5, 6, 8, 10, 12, 13, 18, 24, 26, 27, 28], "dummi": [], "dure": [0, 1, 3, 4, 8, 9, 11, 19, 21, 26], "dwell": [], "dwh": 1, "dwo": 1, "dx": [2, 3, 8, 23], "dx_1": 23, "dx_1p": 6, "dx_2p": 6, "dx_mp": 6, "dx_n": 23, "dxp": 6, "dy": [1, 8, 23], "dynam": 4, "dz": 8, "e": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 23, 24, 26, 27, 28], "e_": [0, 2, 26], "each": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 22, 23, 24, 26, 27, 28], "eapprox": [0, 26], "earli": [1, 13], "earlier": [0, 5, 7, 8, 9, 11, 12, 13, 26, 27], "earthexplor": 6, "eas": [6, 9, 14], "easi": [0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 19, 20, 26, 27, 28], "easier": [5, 6, 8, 9, 13, 15, 21, 23, 26, 27, 28], "easiest": [13, 18], "easili": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 26, 27, 28], "eastern": [24, 26], "ebind": [0, 26], "eblock": 9, "econometr": 26, "economi": 5, "ecosystem": [19, 26], "ect": 22, "edg": 3, "edgecolor": 6, "editor": 15, "edu": [13, 21, 28], "educ": [0, 21, 26], "eff": 23, "effect": [1, 4, 10, 13, 16, 17, 18, 23], "effic": 1, "effici": [0, 3, 10, 13, 19, 20, 23, 26], "efron": 6, "egrad": 13, "eig": [5, 11, 13, 20, 23, 26, 27, 28], "eigen": 23, "eigenpair": [5, 11, 27, 28], "eigenvalu": [0, 5, 8, 11, 13, 20, 26, 27, 28], "eigenvector": [5, 11, 13, 27, 28], "eight": [20, 26], "eigval": [20, 23, 26], "eigvalu": [11, 13, 28], "eigvec": [20, 23, 26], "eigvector": [11, 13, 28], "eir": [24, 26], "eispack": [20, 26], "either": [1, 5, 6, 7, 8, 9, 10, 11, 13, 18, 21, 23, 26, 27, 28], "eivind": 24, "eivinsto": 24, "ekstr\u00f8m": 4, "elabor": 23, "elarn": 3, "electr": [0, 3, 12, 26], "electron": 26, "eleg": 11, "element": [1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 19, 20, 21, 25, 27], "elementari": [10, 13, 20], "elementwis": [3, 13], "elementwise_grad": [2, 13], "elessar": 26, "elif": 14, "elim": 20, "elimin": [3, 8], "elin": [24, 26], "ellipsi": 16, "els": [1, 3, 4, 7, 9, 12, 13, 16, 20], "elu": 1, "elus": [0, 26], "email": [22, 24, 26], "embed": [0, 11, 27], "embodi": [6, 21], "emit": 23, "emner": 25, "emphas": [0, 10, 19, 26], "emphasi": [0, 19, 25, 26], "empir": [1, 11, 23], "emploi": [0, 1, 5, 6, 11, 13, 23, 26, 27, 28], "employ": 0, "empti": [6, 10, 15], "emul": 12, "en": [19, 21, 25], "enabl": 11, "enbodi": 6, "encod": [0, 3, 5, 9, 11, 14, 26, 27, 28], "encompass": [0, 21, 23], "encount": [0, 1, 5, 7, 13, 15, 21, 23, 26, 27, 28], "encourag": [15, 21], "end": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 23, 24, 26, 27, 28], "endpoint": [3, 6], "energi": [0, 4, 6], "enforc": 12, "eng": 25, "engin": [0, 1, 3, 4, 19, 26], "english": 21, "enocurag": 21, "enorm": 3, "enough": [0, 6, 13, 26, 28], "ensembl": [1, 9, 26], "ensur": [0, 1, 2, 3, 5, 6, 11, 13, 18, 23, 27, 28], "entail": 26, "enter": [5, 6, 27, 28], "enthought": [0, 19, 21, 26], "entir": [1, 3, 7, 9, 19, 23, 26], "entiti": [9, 12, 20, 26], "entri": [0, 5, 8, 11, 12, 20, 26, 27], "entropi": [1, 3, 7, 10, 13, 26, 28], "enumer": [0, 1, 2, 3, 4, 6, 8, 26, 27], "env": 23, "environ": [2, 19, 21, 26], "environemnt": 15, "eo": [0, 6], "eol": 0, "eosfit": 0, "epoch": [0, 1, 3, 4, 12, 13, 26], "epsilon": [0, 5, 6, 7, 13, 21, 26, 27, 28], "epsilon_": [0, 26], "epsilon_0": [0, 26], "epsilon_1": [0, 26], "epsilon_2": [0, 26], "epsilon_i": [0, 26, 27], "eq": [3, 13, 14, 20, 23, 28], "eqnarrai": [3, 5, 6], "equal": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 16, 18, 20, 21, 23, 26, 27, 28], "equat": [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 17, 20, 23, 26], "equilibrium": [2, 12], "equiv": [3, 13, 20, 23, 28], "equival": [0, 1, 5, 7, 8, 11, 13, 19, 20, 26, 27, 28], "erf": 23, "eriador": 26, "err": [0, 10], "err_": 6, "err_sqr": 2, "errat": [13, 28], "erron": 2, "error": [1, 2, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 23], "error_estimate_corr_tim": 23, "error_hidden": 1, "error_output": 1, "escap": [13, 28], "especi": [1, 3, 9, 12, 13, 15, 18, 21], "essenti": [0, 5, 6, 9, 10, 12, 14, 15, 21, 23, 27, 28], "establish": [0, 6, 10, 11, 16, 21], "estim": [0, 1, 5, 6, 7, 10, 11, 13, 19, 23, 26, 27, 28], "estimated_mse_fold": 6, "estimated_mse_kfold": 6, "estimated_mse_sklearn": 6, "et": [0, 2, 4, 16, 17, 25, 26, 27, 28], "eta": [0, 1, 3, 8, 12, 13, 18, 26, 28], "eta0": [8, 13], "eta_": 13, "eta_t": 13, "eta_v": [0, 1, 3, 26], "etc": [0, 1, 3, 5, 7, 8, 9, 11, 12, 13, 14, 19, 20, 21, 23, 27, 28], "ethic": 19, "euclidean": [0, 14, 27], "evalu": [0, 2, 3, 4, 5, 6, 9, 13, 15, 16, 17, 21, 23, 26, 27, 28], "evalut": [13, 21], "even": [0, 1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 19, 20, 23, 26, 27, 28], "evenli": 4, "event": [5, 7, 10, 23], "eventu": [0, 5, 6, 11, 12, 13, 21, 24, 27, 28], "everi": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15, 19, 23, 24, 26, 27, 28], "everyth": [4, 12, 16, 18], "everywher": [4, 13, 28], "evolv": 0, "exact": [0, 5, 11, 12, 13, 20, 23, 26, 27], "exactli": [0, 3, 4, 6, 12, 18, 19, 27], "exam": 26, "examin": 6, "exampl": [0, 5, 11, 12, 13, 15, 16, 18, 19, 20, 21, 23, 25], "exce": [1, 12, 13], "excel": [0, 1, 4, 5, 10, 21, 26, 27], "except": [3, 4, 6, 8, 9, 20], "excess": [0, 26], "excit": 0, "exclud": [1, 6, 12, 27], "exclus": [0, 1, 3, 6, 23, 26], "execut": [2, 5, 13, 15, 27, 28], "exemplifi": 13, "exercic": [24, 26], "exercis": [5, 19, 21, 22, 24, 26, 28], "exhaust": 6, "exhibit": [0, 5, 6, 8, 26, 27], "exist": [0, 1, 2, 3, 5, 6, 7, 8, 9, 13, 20, 21, 26, 28], "exit": [5, 20, 27, 28], "exp": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 16, 17, 23, 27, 28], "exp_term": 1, "expand": [5, 7, 11, 13, 28], "expans": [0, 3, 5, 8, 10, 12, 13, 26, 27, 28], "expect": [0, 1, 5, 6, 7, 11, 12, 13, 15, 18, 19, 21, 26, 27], "expectation_value_of_h_wrt_p": 23, "expens": [6, 10, 13, 16, 28], "experi": [0, 1, 6, 8, 13, 15, 19, 21, 26, 27, 28], "experiment": [0, 4, 6, 9, 23, 26], "expert": [1, 9], "explain": [0, 6, 9, 10, 11, 13, 16, 21, 26, 28], "explained_variance_ratio_": 11, "explanatori": [0, 26], "explicit": [0, 3, 6, 13, 20, 21, 26, 27, 28], "explicitli": [0, 4], "explod": 1, "exploit": [0, 3, 12, 13, 26], "explor": [1, 4, 6, 8, 13, 18, 19, 21, 26, 28], "expon": 1, "exponenti": [0, 1, 5, 6, 10, 13, 23, 26, 28], "export": [9, 15, 16], "export_graphviz": 9, "export_text": 9, "exporttext": 9, "expos": 19, "express": [0, 2, 3, 5, 6, 7, 10, 12, 13, 18, 20, 21, 23, 26, 28], "exptmean": 23, "exptvari": 23, "extend": [0, 2, 7, 11, 13, 19, 26], "extens": [0, 12, 15, 19, 26], "extent": [0, 1, 6, 25], "extern": [3, 6, 9], "extra": [1, 3, 5, 15, 24, 26, 27, 28], "extract": [0, 3, 5, 6, 7, 8, 11, 13, 16, 17, 20, 26, 27], "extrapol": [0, 26], "extrem": [0, 1, 4, 5, 6, 7, 8, 9, 13, 15, 16, 20, 27, 28], "extremum": [13, 28], "extrins": 11, "ey": [0, 5, 6, 13, 14, 18, 20, 26, 27, 28], "f": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 23, 24, 26, 27, 28], "f1": 13, "f11": [0, 26], "f12": [0, 26], "f13": [0, 26], "f1_grad": 13, "f1d": 13, "f2": 13, "f2_grad_x1": 13, "f2_grad_x1_analyt": 13, "f2_grad_x2": 13, "f2_grad_x2_analyt": 13, "f3": 13, "f3_grad": 13, "f3_grad_analyt": 13, "f4": 13, "f4_grad": 13, "f4_grad_analyt": 13, "f5": 13, "f5_grad": 13, "f6": 13, "f6_for": 13, "f6_for_grad": 13, "f6_grad_analyt": 13, "f6_while": 13, "f6_while_grad": 13, "f7": 13, "f7_grad": 13, "f7_grad_analyt": 13, "f8": 13, "f8_grad": 13, "f9": [0, 13, 26], "f9_altern": 13, "f9_alternative_grad": 13, "f9_grad": 13, "f_": 10, "f_0": [3, 10], "f_1": [10, 13, 28], "f_2": [12, 13, 28], "f_3": 12, "f_d": 23, "f_grad": 13, "f_grad_analyt": 13, "f_i": [0, 6, 12, 16], "f_m": [3, 10], "f_n": 3, "f_vec": 2, "face": [13, 26, 28], "facecolor": [6, 8, 23], "facil": [0, 19], "facilit": 12, "fact": [0, 1, 3, 5, 9, 11, 12, 13, 26, 27, 28], "factor": [0, 1, 3, 5, 6, 9, 10, 11, 13, 20, 23, 26, 27, 28], "factori": 13, "fade": 6, "fafab0": [9, 10], "fail": [0, 6, 13, 24, 26, 28], "failur": 7, "fairli": [1, 2, 18, 23], "faisal": [16, 27], "fake": 4, "fake_loss": 4, "fake_output": 4, "fall": [8, 9, 22], "fals": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 16, 17, 20, 26, 27, 28], "famili": [0, 7, 8, 23, 27], "familiar": [0, 3, 5, 6, 8, 15, 19, 20, 21, 23, 26], "famou": [6, 12], "far": [0, 3, 4, 5, 6, 8, 11, 12, 13, 14, 16, 26, 27, 28], "fashion": [0, 9, 10, 26], "fast": [1, 3, 6, 10, 12, 13, 19, 23, 26, 28], "faster": [1, 11, 13], "fastest": [13, 20, 28], "favor": 7, "favorit": 23, "fc": 3, "featur": [0, 1, 3, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18, 19, 23, 26, 28], "feature_nam": [1, 7, 9], "feautur": 9, "fed": 1, "feed": [0, 2, 3, 11, 19, 26], "feed_forward": 1, "feed_forward_out": 1, "feed_forward_train": 1, "feedback": [4, 26], "feeddorward": 4, "feedforward": [1, 4, 12], "feel": [0, 5, 6, 11, 13, 15, 16, 18, 19, 21, 24, 26], "feet": [], "felt": 21, "fetch": [6, 15], "few": [1, 3, 4, 5, 9, 17, 18, 23, 26], "fewer": [0, 9, 11, 26], "ffnn": [1, 12], "field": [0, 3, 6, 12, 19], "fifth": [0, 6, 26], "fig": [0, 1, 2, 3, 4, 6, 7, 12, 13, 14, 21, 26], "fig_id": [0, 6, 7, 9, 26], "figaxi": 23, "figsiz": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 26], "figur": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 19, 21, 26, 27, 28], "figure_id": [0, 6, 7, 9, 26], "figurefil": [0, 6, 7, 9, 26], "file": [0, 4, 5, 6, 7, 9, 15, 21, 26], "file_prefix": 4, "filenam": 26, "fill": [5, 9, 18, 27, 28], "filter": [3, 4], "final": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 21, 22, 23, 24, 26, 28], "financ": 0, "find": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 21, 23, 26, 27, 28], "fine": [0, 14], "finish": 2, "finit": [3, 5, 6, 12, 13, 17, 23, 27, 28], "finnicki": 15, "first": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 20, 21, 23, 24, 25, 27], "firsteigvector": 11, "fit": [1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 17, 18, 21, 23, 27], "fit_beta": 27, "fit_intercept": [0, 5, 6, 16, 27, 28], "fit_mod": 9, "fit_theta": 6, "fit_transform": [0, 6, 8, 9, 11, 15], "fiti": [0, 26], "five": [0, 9, 26, 27], "fix": [0, 3, 4, 6, 10, 11, 12, 13, 21, 26], "flag": 4, "flat": [12, 13, 28], "flatten": [1, 3, 4, 5, 20], "flexibl": [1, 6, 8, 10, 12, 26], "flip": [24, 26], "float": [0, 3, 4, 5, 9, 11, 13, 14, 20, 26, 27, 28], "float32": [4, 9], "float64": [4, 20, 26], "flop": [5, 20, 27, 28], "flow": [1, 4, 12], "fluctuat": 5, "fly": 11, "fm": 0, "fmax": 3, "fmesh": 13, "fn": 7, "focu": [0, 3, 4, 5, 6, 15, 19, 21, 25, 26, 27, 28], "focus": [1, 6, 7, 20, 27], "fold": [6, 9, 21], "folder": [0, 4, 6, 15, 21, 26], "follow": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 23, 24, 25, 26, 27, 28], "font": [7, 23, 26], "fontdict": 23, "fontsiz": [1, 6, 8, 9, 10, 23], "fontweight": 1, "footprint": 3, "foral": [8, 27], "forc": [0, 5, 6, 10, 11, 27, 28], "forcast": 4, "forecast": [4, 12], "forest": [0, 1, 9, 19, 26], "forget": 11, "form": [0, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 19, 20, 21, 23, 26, 27, 28], "formal": [3, 4, 14, 23], "format": [0, 1, 3, 4, 6, 7, 8, 9, 10, 11, 19, 23, 25], "format_data": 4, "formatstrformatt": [6, 13, 28], "formul": [4, 6, 11, 14], "formula": [3, 13, 23, 28], "forth": [4, 12], "fortran": [0, 19, 20, 26], "fortran2003": [19, 26], "fortran2008": 21, "fortran90": 23, "fortun": [0, 11, 27], "forward": [0, 3, 6, 19, 20, 26], "found": [1, 2, 4, 5, 6, 12, 13, 21, 26, 27], "foundat": [19, 26], "four": [4, 5, 6, 8, 12, 20, 22, 24, 26, 28], "fourier": [0, 26], "fourierdef1": 3, "fourierdef2": 3, "fourierseriessign": 3, "fourth": [12, 26, 27], "fp": 7, "frac": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 20, 21, 23, 26, 27, 28], "fraction": 9, "frame": 7, "framework": [1, 8, 10, 23], "frank": [5, 11], "frankefunct": [5, 6, 11], "fredli": [24, 26], "free": [0, 6, 11, 13, 15, 16, 18, 19, 20, 21, 23, 24, 25, 26], "freecodecamp": 19, "freedom": [5, 28], "freeli": [0, 21], "freez": 15, "frequenc": [3, 6, 7, 23], "frequent": [0, 8, 9, 13, 28], "frequentist": 19, "fresh": 10, "fridai": [15, 24, 26], "friedman": [6, 21, 25, 26], "friendli": 4, "fro": 21, "frodo": 26, "frog": 3, "from": [0, 1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25], "from_cod": 9, "from_logit": [3, 4], "from_tensor_slic": 4, "front": [0, 4, 5, 26, 27, 28], "frustrat": 15, "fulfil": [2, 5, 12, 27, 28], "full": [1, 3, 5, 7, 9, 10, 13, 23, 26, 27, 28], "full_matric": [5, 27, 28], "fulli": [3, 6, 12, 23], "fun": [19, 26], "func": 2, "function": [2, 3, 4, 5, 9, 14, 15, 16, 17, 18, 19, 20], "functionali": 11, "fundament": [0, 6, 19, 26], "funtion": 2, "further": [2, 7, 9, 26], "furthermor": [0, 3, 5, 6, 7, 11, 12, 13, 19, 21, 26, 27, 28], "futur": [0, 4, 8, 9, 26], "fy": [15, 21, 22, 24, 25, 26], "fys4155": 21, "fys5419": [25, 26], "fys5429": [25, 26], "f\u00f8470": [24, 26], "g": [0, 1, 2, 3, 4, 6, 8, 9, 10, 11, 13, 15, 18, 23, 26, 27, 28], "g0": 2, "g_": [2, 9, 10], "g_0": 2, "g_1": [2, 10], "g_2": [2, 10], "g_analyt": 2, "g_dnn_ag": 2, "g_euler": 2, "g_i": 2, "g_m": [3, 10], "g_n": 3, "g_re": 2, "g_t": 2, "g_t_d2t": 2, "g_t_d2x": 2, "g_t_dt": 2, "g_t_hessian": 2, "g_t_hessian_func": 2, "g_t_jacobian": 2, "g_t_jacobian_func": 2, "g_trial": 2, "g_trial_deep": 2, "g_vec": 2, "gain": [1, 5, 7, 9, 10, 13, 27, 28], "galleri": [0, 26], "game": 4, "gamge": 26, "gamma": [0, 2, 8, 9, 10, 11, 13, 26, 28], "gamma1": 8, "gamma2": 8, "gamma_": [0, 26], "gamma_0": 10, "gamma_1": 10, "gamma_1x": 10, "gamma_i": [0, 8, 23, 26], "gamma_j": 13, "gamma_k": [13, 28], "gamma_m": 10, "gamma_x": [0, 26], "gap": 8, "gate": [4, 12], "gather": [0, 1, 12, 27], "gaug": 12, "gaussbacksub": 20, "gaussian": [4, 5, 6, 8, 14, 18, 23, 26], "gaussian_point": 14, "gaussian_rbf": 8, "gave": 13, "gavra": 26, "gbc": 26, "gca": [2, 6, 8, 13], "gd": [1, 28], "gd_clf": 10, "gdclassiffiercgain": 10, "gdclassiffierconfus": 10, "gdclassiffierroc": 10, "gdm": 13, "gdregress": 10, "ge": [1, 5, 7, 23, 27, 28], "gen_loss": 4, "gen_tap": 4, "gender": [0, 26], "genener": 4, "gener": [0, 1, 2, 3, 5, 6, 8, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 23, 25, 27, 28], "generaliz": 16, "generallay": 12, "generate_and_save_imag": 4, "generate_imag": 4, "generate_latent_point": 4, "generate_simple_clustering_dataset": 14, "generated_imag": 4, "generator_loss": 4, "generator_loss_list": 4, "generator_model": 4, "generator_optim": 4, "genom": 19, "geodes": 11, "geometr": [0, 13, 26], "geometri": 5, "georg": 25, "geotif": 6, "geq": [2, 5, 8, 9, 13, 27, 28], "geron": [0, 25, 26], "get": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 15, 19, 20, 21, 23, 24, 26, 27, 28], "get_dummi": 9, "get_paramet": 2, "get_split": 9, "get_yaxi": 8, "get_yticklabel": 6, "gh": 15, "gibb": [19, 26], "gif": 4, "gini": 10, "gini_index": 9, "ginvers": 13, "git": [0, 15, 19, 26], "giter": 13, "github": [0, 19, 21, 22, 24, 25, 26, 27], "gitignor": 15, "gitlab": [0, 15, 19, 21, 26], "give": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 18, 19, 21, 23, 26, 27, 28], "given": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 20, 23, 26, 27, 28], "global": [6, 7, 13, 28], "glorot": 1, "gnew": 13, "go": [0, 1, 3, 5, 6, 8, 9, 11, 12, 13, 15, 16, 18, 26, 27, 28], "goal": [0, 7, 9, 26], "goe": [0, 1, 2, 5, 6, 13, 14, 15, 20, 26, 27, 28], "golden": 13, "gone": [5, 27, 28], "gong": 1, "good": [1, 3, 4, 5, 6, 9, 10, 11, 13, 15, 18, 19, 23, 25, 27, 28], "goodfellow": [4, 25, 26, 27, 28], "googl": [1, 4, 19, 26], "got": [1, 6, 21], "gotten": 26, "gov": 6, "govern": 26, "gp": 25, "gpu": [1, 13, 19, 26], "grad": [2, 13], "grad_analyt": 13, "grad_ol": 18, "grad_ridg": 18, "grade": [21, 22], "gradient": [0, 3, 4, 7, 8, 9, 12, 19, 26, 27], "gradientboostingclassifi": 10, "gradientboostingregressor": 10, "gradients_of_discrimin": 4, "gradients_of_gener": 4, "gradienttap": 4, "gradual": [1, 14], "grai": [4, 6], "graph": [1, 9, 11, 12, 13, 16, 28], "graph_from_dot_data": 9, "graphic": [0, 1, 9, 15, 26], "grasp": 0, "gray_r": [1, 3], "grayscal": 3, "great": [5, 13, 15, 28], "greater": [1, 7, 23, 27], "greatli": 13, "greedi": 9, "green": [0, 3, 9, 23], "grei": 4, "grid": [1, 3, 6, 7, 8, 12, 23, 27], "grossli": [13, 28], "ground": [0, 26], "group": [0, 6, 7, 9, 14, 15, 19, 21, 22, 24, 26], "groupbi": [0, 26], "grow": [1, 3, 9, 10], "growth": [0, 26], "gru": 4, "guarante": [0, 4, 13, 23, 26, 27, 28], "guess": [1, 4, 10, 13, 14, 28], "guestrin": 10, "gui": 15, "guid": 1, "h": [0, 1, 5, 6, 8, 13, 15, 23, 24, 25, 26, 27, 28], "h1": 2, "h_": [0, 13, 26, 28], "h_1": [2, 13, 28], "h_2": [2, 13, 28], "h_m": 10, "ha": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 20, 21, 23, 26, 27, 28], "haanen": [24, 26], "habit": [0, 27], "had": [0, 1, 6, 7, 13, 26, 28], "hadamard": [1, 12, 13], "half": [1, 8, 9], "halv": 10, "hand": [0, 1, 2, 3, 5, 11, 12, 13, 19, 20, 21, 23, 24, 25, 26, 27, 28], "handi": [3, 21], "handl": [0, 1, 2, 5, 9, 11, 15, 18, 19, 27, 28], "handle_unknown": 9, "handsid": 12, "handwrit": 12, "handwritten": [1, 5], "happen": [1, 2, 3, 4, 5, 6, 10, 13, 23, 27, 28], "hard": [1, 7, 8, 10, 13, 28], "hardcopi": [19, 26], "harder": [0, 1, 27], "harmon": 3, "hasn": [], "hassl": [0, 19, 26], "hast": [19, 26], "hasti": [0, 6, 16, 17, 21, 25, 26, 27], "hat": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13, 16, 17, 18, 20, 27, 28], "have": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 26, 27, 28], "haven": 1, "he": 7, "head": [4, 10, 23], "header": [0, 26], "heads_proba": 10, "health": [0, 27], "hear": [0, 13, 26], "heart": [0, 7, 26], "heatmap": [0, 1, 3, 7, 17, 26], "heavili": 0, "heavisid": 1, "height": [1, 3, 6, 27], "held": 13, "help": [0, 1, 4, 12, 13, 15, 16, 21, 26], "helper": [4, 14], "henc": [0, 5, 6, 8, 9, 10, 12, 13, 26, 27, 28], "henrik": [24, 26], "her": 7, "here": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 23, 26, 27, 28], "hereaft": [0, 8, 12, 26], "hermitian": 20, "hessenberg": 20, "hessian": [0, 2, 5, 13], "heterogen": [9, 10], "hi": 7, "hidden": [1, 3, 4, 12], "hidden_bia": 1, "hidden_bias_gradi": 1, "hidden_layer_s": [0, 1, 26], "hidden_neuron": 4, "hidden_weight": 1, "hidden_weights_gradi": 1, "hierarch": [5, 27, 28], "high": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 13, 14, 19, 20, 21, 26, 27, 28], "higher": [0, 1, 3, 5, 6, 8, 13, 18, 21, 26, 27, 28], "highest": [1, 2], "highli": [0, 3, 4, 10, 19, 20, 25, 26, 27, 28], "highwai": [], "hing": 8, "hint": [13, 15, 16, 27, 28], "hip": 19, "hire": 0, "hist": [4, 6, 7, 23], "histogram": [6, 7, 23], "histor": [7, 11], "histori": [3, 4, 12, 15, 18], "hitherto": 5, "hjorth": [24, 26, 27, 28], "hobbi": 23, "hoc": [5, 27, 28], "hoff": 25, "hold": [1, 3, 6, 13, 14, 28], "holder": [0, 26], "home": [], "homepag": [21, 26], "homework": [6, 13, 28], "homogen": [1, 3, 9, 10, 13], "honchar": 2, "hopefulli": [0, 11, 15, 23, 26], "horizont": 11, "horlyk": [24, 26], "hors": [3, 7, 26], "hot": [1, 9], "hour": [1, 19, 22, 23, 24, 26], "how": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 23, 26, 27, 28], "howev": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19, 20, 21, 23, 26, 27, 28], "hspace": [0, 4, 8, 10, 23, 26], "hstack": 1, "htf": 26, "html": [0, 16, 19, 21, 22, 24, 25, 26, 27, 28], "http": [0, 3, 4, 6, 13, 15, 16, 19, 20, 21, 22, 24, 25, 26, 27, 28], "huang": [0, 26], "huber": [0, 26], "huge": [1, 3, 4, 19], "human": [0, 1, 3, 6, 9, 12, 27], "humid": 9, "hundr": 1, "hungri": 1, "hybrid": 22, "hydrogen": [0, 26], "hyperbol": [1, 4, 12], "hyperparam": 8, "hyperparamet": [3, 4, 5, 6, 9, 13, 18, 21, 27, 28], "hyperplan": 11, "h\u00f8rlyk": [24, 26], "i": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28], "i0": [0, 26], "i1": [0, 6, 8, 12, 26, 27], "i2": [0, 8, 12, 26], "i3": [0, 12, 26], "i5": [0, 26], "i_": [13, 28], "i_1": [5, 6], "i_2": [5, 6], "ian": 25, "ic": [1, 21], "id": [7, 13, 28], "ida": [24, 26], "idea": [0, 1, 2, 3, 4, 6, 9, 10, 12, 13, 20, 21, 27, 28], "ideal": [0, 2, 6, 8, 13, 23, 26], "idem": 6, "ident": [5, 6, 12, 13, 17, 18, 20, 27, 28], "identifi": [0, 1, 7, 9, 11, 12, 13, 14, 26, 27], "ieor": 23, "ifi": 25, "ifs": [19, 26], "ignor": [0, 1, 3, 9, 15, 27], "ii": [20, 23], "iii": [20, 26], "ij": [0, 1, 3, 6, 8, 12, 14, 16, 20, 23, 26, 27], "ik": [0, 20, 26, 27], "illustr": [5, 7, 10, 12, 13, 14, 19, 26], "im": 6, "imag": [1, 3, 4, 6, 9, 11, 12, 14, 25, 26], "image_at_epoch_": 4, "image_batch": 4, "image_height": 3, "image_path": [0, 6, 7, 9, 26], "image_width": 3, "imageio": 6, "images_from_seed_imag": 4, "imagin": 1, "immedi": [0, 3, 4, 6, 19, 26], "implement": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 21, 23, 26, 27, 28], "impli": [3, 5, 6, 7, 13, 20, 27, 28], "implicit": 3, "implicitli": [11, 23], "import": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 23], "importantli": 3, "impos": [0, 6, 11, 12, 26], "imposs": [0, 5, 26, 27, 28], "impress": [0, 12, 26], "improv": [0, 4, 5, 9, 10, 11, 13, 15, 21, 27, 28], "impur": 9, "imread": 6, "imshow": [1, 3, 4, 6], "in3050": [25, 26], "in3310": 26, "in4080": [25, 26], "in4300": [25, 26], "in4310": 25, "in5400": 3, "in5550": 25, "in_out_neuron": 4, "inaccur": [13, 28], "inact": 12, "inadequ": [0, 26], "inch": [6, 27], "includ": [0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 15, 16, 17, 18, 19, 23, 24, 25, 26, 27, 28], "include_bia": [6, 9], "incom": [12, 16], "incorrect": 1, "incoveni": 8, "increas": [0, 1, 3, 4, 5, 6, 9, 12, 13, 21, 23, 26, 27], "increasingli": 23, "ind": 6, "inde": [0, 2, 4, 5, 6, 13, 26, 27, 28], "indefinit": 4, "independ": [0, 5, 6, 7, 8, 12, 13, 23, 26, 27, 28], "index": [0, 1, 3, 4, 10, 14, 19, 20, 21, 23, 25, 26], "index_col": [0, 26], "indic": [0, 1, 3, 4, 5, 6, 9, 10, 11, 13, 16, 21, 26, 27], "indispens": 6, "individu": [1, 6, 7, 10, 12, 23, 26, 27], "indu": [], "indx": 20, "indx1": 2, "indx2": 2, "indx3": 2, "ineffici": [3, 13], "inequ": [8, 13], "inequaltii": 28, "inertia": 13, "inf1000": [19, 26], "inf1100": [19, 26], "inf1100l": [19, 26], "inf1110": [19, 26], "inf3000": 26, "infeas": 9, "infer": [0, 1, 4, 6, 25, 26], "inferenc": 1, "infil": [0, 6, 7, 9, 26], "infin": [5, 6, 7, 11, 27, 28], "infinit": 3, "infinitesim": 23, "influenc": [6, 10, 18], "influenti": 1, "info": 26, "inform": [0, 1, 3, 4, 6, 9, 11, 12, 13, 14, 20, 21, 25, 26, 28], "inforom": 15, "infti": [3, 6, 13, 23, 28], "ingeni": [13, 28], "ingredi": [0, 9, 26], "inher": 6, "inherit": [20, 26], "initi": [0, 1, 2, 6, 10, 13, 14, 18, 20, 23, 26, 28], "inject": 14, "inlin": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 23, 26, 27, 28], "inner": [0, 13, 27], "inp": 4, "inplac": 13, "input": [0, 1, 3, 4, 5, 6, 7, 8, 12, 13, 14, 16, 21, 23, 26, 27, 28], "input_dim": 1, "input_shap": [3, 4], "inputs": 1, "inputs_shuffl": [0, 1, 27], "insert": [3, 5, 6, 8, 10, 23, 27, 28], "insid": [4, 7], "insight": [0, 1, 5, 19, 26, 27, 28], "insist": [6, 13, 27], "inspir": [0, 1, 12, 21, 26], "instabl": 2, "instal": [0, 1, 5, 6, 9, 15], "instanc": [0, 1, 2, 4, 6, 9, 11, 13, 16, 26, 27, 28], "instanti": 10, "instead": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 17, 20, 23, 26, 27], "institut": 1, "instruct": [0, 1, 15], "int": [0, 1, 2, 3, 4, 5, 6, 11, 13, 14, 20, 23, 27], "int32": 10, "int_": [3, 6, 23], "int_0": 23, "int_a": 23, "intak": [0, 27], "integ": [1, 2, 13, 14, 20, 23, 26], "integer_vector": 1, "integr": [3, 6, 23, 26], "intellig": [0, 14, 25, 26], "intend": 10, "intens": [1, 18], "intention": 14, "interact": [0, 6, 9, 12, 19, 21, 26], "intercept": [0, 6, 8, 11, 13, 16, 17, 18, 26, 27, 28], "intercept_": [0, 6, 8, 9, 13, 26, 27], "interchang": [5, 12, 20], "interconnect": 1, "interest": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 19, 21, 23, 26, 27, 28], "interfac": [0, 1, 15, 20, 27], "interior": [0, 9, 26], "intermedi": [20, 27], "intern": [1, 10, 12], "interpol": [1, 3, 4, 6, 12], "interpr": [5, 27, 28], "interpret": [0, 1, 6, 9, 10, 12, 13, 15, 16, 20, 21, 23], "interv": [0, 3, 5, 6, 7, 13, 23, 26, 27, 28], "intial": [13, 28], "intract": [0, 4, 27], "intrins": [3, 11, 20, 23, 26], "intro": [19, 25, 26], "introduc": [0, 1, 5, 6, 8, 10, 12, 20, 21, 23, 26, 28], "introduct": [1, 2, 4, 13, 25, 27, 28], "introductori": [0, 4, 20, 25, 26, 27], "intuit": [0, 5, 6, 8, 12, 13, 21, 26], "inv": [0, 5, 13, 17, 26, 27, 28], "invalu": [0, 13, 19, 26, 28], "invari": 1, "invd": 5, "inver": 8, "invers": [0, 3, 6, 13, 26, 27, 28], "inverse_transform": 8, "invert": [0, 5, 7, 10, 13, 16, 18, 26], "invh": 13, "invok": 8, "involv": [0, 2, 6, 7, 11, 12, 26, 27], "io": [0, 19, 21, 22, 24, 25, 26, 27], "ip": [0, 8, 23, 26], "ipca": 11, "ipynb": [19, 26], "ipython": [0, 5, 7, 9, 11, 14, 19, 21, 26, 27], "iq": 6, "iri": [8, 9], "irreduc": 6, "irrelev": [5, 27, 28], "irrespect": [0, 26], "irvin": 21, "isn": 5, "isnul": [], "isomap": 11, "issu": [1, 9, 15, 20], "it_arrai": 13, "item": [0, 13, 26], "items": [20, 26], "iter": [1, 2, 4, 6, 8, 13, 14, 18, 21, 23, 28], "its": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 23, 26, 28], "itself": [5, 6, 12, 21, 23, 26, 27], "j": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 15, 16, 20, 21, 23, 25, 26, 27, 28], "j1": 20, "j_": 6, "j_lasso_sk": 6, "j_ridge_sk": 6, "j_sk": 6, "jackknif": [6, 19, 26], "jacobian": [2, 13, 28], "jason": 4, "jax": [19, 26], "jensen": [24, 26, 27, 28], "jerom": [21, 25], "ji": [12, 20], "jit": 13, "jj": [0, 5, 6, 26], "jk": [0, 1, 6, 12, 20, 26], "jl": [0, 26], "jm": 20, "jnp": 13, "job": [2, 8, 10, 15], "join": [0, 4, 6, 7, 9, 26], "joint": [4, 5], "judg": [13, 28], "judgement": 6, "julia": [19, 20, 21], "jump": 23, "junk": 4, "jupit": 26, "jupyt": [0, 15, 16, 19, 21, 25, 26], "just": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 23, 26, 27, 28], "justif": 0, "justifi": [3, 10], "k": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19, 20, 21, 23, 24, 26, 27, 28], "k0": 7, "k1": 7, "kaggl": [6, 21], "kappa_d": 23, "karl": [24, 26], "karush": 8, "katrin": [24, 26], "keep": [0, 1, 4, 5, 6, 11, 13, 14, 15, 18, 20, 21, 26, 27, 28], "keepdim": [1, 6, 10, 20], "kei": [1, 3, 6, 12], "kept": [4, 6, 14], "kera": [0, 4, 19, 21, 26], "kernel": [0, 1, 3, 19, 26, 27], "kernel_regular": [1, 3], "kernel_s": 4, "kernelpca": 11, "kev": [0, 26], "kevin": [25, 26], "keyword": [20, 26], "kfold": 6, "kg": 1, "ki": 20, "kick": [1, 13], "kiener": 2, "kilomet": [6, 27], "kind": [0, 2, 3, 4, 8, 12, 13, 14, 26, 27], "kj": [6, 12, 20, 27], "kjm": [19, 26], "kkt": 8, "kl": 23, "km": [12, 26], "kmean": 14, "kmeanspoint": 14, "kn_k": 14, "know": [0, 1, 2, 5, 6, 8, 13, 15, 16, 17, 19, 26, 27, 28], "knowledg": [0, 19, 26], "known": [1, 3, 4, 5, 6, 7, 8, 9, 12, 18, 20, 21, 23, 25, 27], "kondev": [0, 26], "kp": 23, "kpca": 11, "kroneck": 14, "kuhn": 8, "kvalsund": [24, 26], "kwown": [0, 26], "l": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 20, 21, 23, 26, 28], "l0": 7, "l1": [0, 1, 3, 7, 26], "l1_l2": [1, 3], "l1regl": 5, "l2": [1, 3], "l_": 20, "l_1": 7, "l_2": [7, 13, 28], "l_j": 12, "la": 13, "la_i": 12, "la_k": 12, "lab": [19, 21, 26], "label": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 19, 20, 21, 23, 26, 27, 28], "labelencod": [7, 10], "labels": [6, 8, 9], "labels_shuffl": [0, 1, 27], "laboratori": 22, "lack": [0, 26], "lagari": 2, "lagrang": [8, 11], "lam": [], "lambda": [0, 1, 2, 3, 5, 6, 7, 8, 10, 12, 13, 17, 18, 21, 23, 26, 27, 28], "lambda_": 11, "lambda_0": 11, "lambda_1": [5, 8, 11, 27, 28], "lambda_2": [8, 11], "lambda_i": [8, 11], "lambda_iy_i": 8, "lambda_jy_iy_j": 8, "lambda_k": 8, "lambda_n": [5, 8, 27, 28], "lamda": 1, "land": 8, "landmark": 8, "landscap": [13, 18, 28], "langl": [0, 6, 11, 23, 26, 27], "languag": [0, 1, 4, 8, 19, 20, 21, 25, 26], "lapack": [20, 26], "laplac": 5, "laptop": [15, 19], "larg": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 13, 18, 19, 20, 21, 23, 25, 26, 27, 28], "larger": [0, 3, 5, 6, 8, 10, 11, 13, 17, 23, 26, 27, 28], "largest": [4, 8, 11], "lasso": [0, 7, 19, 26], "lasso_sk": 6, "last": [0, 1, 3, 4, 5, 6, 7, 8, 12, 16, 17, 20, 21, 23, 24, 26, 28], "latent": 4, "latent_dim": 4, "latent_point": 4, "latent_space_value_rang": 4, "later": [0, 1, 4, 7, 8, 12, 13, 14, 15, 19, 21, 26], "latest": [4, 15, 19], "latest_checkpoint": 4, "latex": 26, "latter": [0, 3, 6, 7, 8, 11, 13, 20, 23, 26, 27, 28], "lattic": 12, "law": 0, "layer": [0, 4, 13, 26], "lbfg": [7, 9, 10], "lcc": [5, 6], "lda": 11, "ldot": [0, 6, 11, 21, 26], "le": [5, 7, 10, 13, 17, 23, 27, 28], "lead": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 20, 23, 26, 27, 28], "leaf": 9, "leaki": 1, "leakyrelu": 4, "lear": [13, 28], "learn": [3, 4, 5, 6, 7, 8, 9, 10, 12, 20, 24, 25], "learnabl": 3, "learner": 10, "learnig": 26, "learning_r": [8, 10], "learning_rate_init": [0, 1, 26], "learning_schedul": 13, "learnt": 21, "least": [0, 7, 8, 10, 11, 17, 18, 19, 20, 23], "leat": 13, "leav": [0, 1, 3, 5, 6, 9, 11, 26, 28], "lectur": [0, 1, 5, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27], "lecturenot": [0, 19, 21, 25, 26], "left": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 20, 21, 23, 26, 27, 28], "leftarrow": [8, 12], "legend": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 15, 26, 27, 28], "leinonen": 26, "len": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 16, 17, 20, 26, 27, 28], "length": [0, 1, 3, 4, 8, 9, 13, 16, 19, 26, 27, 28], "length_of_sequ": 4, "leq": [0, 5, 7, 8, 13, 14, 23, 26, 27, 28], "less": [0, 1, 3, 4, 5, 6, 8, 9, 13, 19, 23, 26, 27, 28], "lessen": 1, "let": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 23, 26, 27, 28], "letter": [0, 16, 20, 23, 26, 27], "level": [0, 1, 5, 6, 9, 19, 20, 21, 22, 24, 26], "li": [8, 11], "lib": [], "liblinear": 10, "librari": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 20, 21, 23, 25, 27, 28], "licens": [0, 1, 19, 21, 26], "lie": [0, 6, 11, 23, 26, 27], "life": [0, 1, 8, 12, 26], "lifetim": 13, "like": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 19, 20, 21, 23, 26, 27, 28], "likelihood": [0, 1, 5, 9, 26, 27], "lim_": 23, "limit": [0, 5, 6, 8, 12, 20, 21, 26, 27], "lin_clf": 8, "lin_model": [], "lin_reg": 9, "linalg": [0, 2, 5, 6, 8, 11, 13, 17, 20, 23, 26, 27, 28], "line": [0, 3, 6, 8, 11, 13, 15, 16, 26, 28], "line1": 8, "line2": 8, "line3": 8, "line_model": 15, "line_ms": 15, "line_predict": 15, "linear": [1, 3, 5, 6, 7, 9, 10, 11, 12, 16, 17, 18, 19, 21, 23], "linear_model": [0, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 26, 27, 28], "linear_regress": 6, "linearli": [5, 27, 28], "linearloc": [6, 13, 28], "linearregress": [0, 6, 7, 9, 15, 16, 26, 27], "linearsvc": 8, "lineat": 28, "liner": [1, 3], "linerar": 10, "linewidth": [0, 2, 4, 6, 8, 9, 10], "link": [0, 4, 9, 12, 15, 19, 21, 22, 24, 26], "linlag": 5, "linpack": [20, 26], "linreg": [0, 26], "linspac": [0, 2, 3, 4, 6, 8, 9, 10, 13, 16, 17, 20, 23, 26, 27], "linu": 4, "linux": [0, 1, 19, 21, 26], "liquid": [0, 26], "list": [1, 2, 3, 4, 9, 15, 19, 21, 26], "listedcolormap": [9, 10], "literatur": [1, 7, 14, 25], "littl": [1, 3, 9, 12], "live": [8, 16], "ll": [0, 18, 23, 26, 27], "lle": [0, 27], "lloyd": [4, 14], "lmb": [0, 2, 5, 6, 27, 28], "lmbd": [0, 1, 3, 26], "lmbd_val": [0, 1, 3, 26], "lmbda": [13, 28], "ln": [1, 13, 28], "load": [1, 4, 6, 7, 9, 10], "load_boston": [], "load_breast_canc": [1, 7, 9, 10, 11], "load_data": [3, 4], "load_digit": [1, 3], "load_iri": [8, 9], "loc": [3, 6, 7, 8, 9, 10, 26], "local": [0, 1, 3, 7, 12, 13, 15, 27, 28], "locat": [2, 3, 8, 15], "log": [0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 13, 15, 20, 21, 26], "log10": [0, 5, 6, 27, 28], "log_": [0, 26], "log_clf": 10, "logarithm": [0, 5, 7, 17, 20, 26], "logbook": 21, "logic": [0, 1, 9, 26], "login": 15, "logist": [0, 1, 2, 8, 9, 10, 11, 12, 13, 19, 27, 28], "logisticregress": [7, 9, 10, 11], "logit": 7, "logreg": [7, 9, 10, 11], "logspac": [0, 1, 3, 5, 6, 26, 27, 28], "long": [0, 1, 3, 4, 12, 13, 26, 28], "longer": [2, 3, 8, 10, 14, 20, 23, 26], "loocv": 6, "look": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 20, 21, 23, 26, 27, 28], "loop": [1, 4, 6, 10, 12, 14, 16, 17, 18, 19, 20, 26], "lose": 1, "loss": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13, 18, 20, 21, 26], "loss_fil": 4, "lossfil": 4, "lost": 4, "lot": [1, 4, 6, 16], "low": [0, 6, 9, 10, 11, 21, 26, 27], "lower": [0, 1, 3, 6, 9, 10, 16, 20, 27], "lowercas": [20, 26], "lowest": [9, 13, 23], "lr": [1, 3, 4, 10], "lstat": [], "lstm": 4, "lstm_2layer": 4, "lstsq": [0, 26, 27], "lt": 6, "lu": [0, 5, 26, 27, 28], "lubksb": 20, "luckili": 2, "ludcmp": 20, "lux": 20, "lvert": 1, "lw": [0, 26], "m": [0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 15, 18, 20, 23, 24, 25, 26, 27, 28], "m_": [9, 12], "m_1": 14, "m_h": [0, 26], "m_k": 14, "m_l": 12, "m_n": [0, 26], "m_p": [0, 26], "m_t": 13, "ma": 11, "machin": [1, 3, 4, 5, 6, 7, 9, 10, 11, 12, 15, 16, 20, 25, 27], "machinelearn": [0, 6, 16, 19, 21, 22, 24, 25, 26, 27, 28], "mackai": 25, "made": [0, 1, 3, 4, 5, 6, 7, 9, 11, 12, 21, 26, 27], "mae": [0, 26], "magic": 4, "magnitud": [1, 6, 7, 13, 27], "mai": [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 19, 20, 21, 23, 26, 27, 28], "mail": [22, 24], "main": [0, 1, 3, 4, 5, 6, 7, 9, 20, 21, 25, 27, 28], "mainli": [0, 5, 6, 7, 9, 26, 27], "maintain": 6, "major": [1, 6, 9, 10, 13, 20, 26, 28], "make": [1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 18, 19, 20, 21, 23, 25, 26, 28], "make_axes_locat": 6, "make_moon": [8, 9, 10], "make_pipelin": [0, 6, 10, 27], "makedir": [0, 6, 7, 9, 26], "malcondit": 20, "malign": [1, 7, 9], "mammographi": 5, "manag": [0, 2, 3, 15, 19, 21, 26], "mandatori": [24, 26], "mani": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28], "manifold": 11, "manner": 3, "manual": [6, 27], "map": [0, 1, 2, 6, 7, 8, 11, 12, 14, 23, 26], "marc": 27, "margin": [0, 5, 8], "marit": [0, 26], "mark": 26, "marker": [7, 20, 26], "markov": [19, 26], "marsaglia": 23, "mass": [0, 1, 5, 13, 27, 28], "massag": [0, 26], "masses2016": [0, 26], "masses2016ol": [0, 26], "masses2016tre": 0, "masseval2016": [0, 26], "master": [22, 24], "mat": [19, 26], "mat1100": [19, 26], "mat1110": [19, 26], "mat1120": [19, 26], "match": [1, 4, 5, 13, 14, 15, 27, 28], "materi": [4, 5, 7, 13, 15, 20, 22, 24], "math": [3, 7, 12, 13, 20, 23, 25, 26], "mathbb": [0, 4, 5, 6, 7, 8, 11, 12, 13, 14, 17, 20, 21, 23, 26, 27, 28], "mathbf": [0, 5, 6, 7, 8, 13, 20, 21, 26, 27, 28], "mathcal": [1, 5, 6, 7, 13, 21], "matheemat": 3, "mathemat": [0, 6, 11, 12, 13, 19, 20, 23, 25, 26], "mathemati": 26, "mathrm": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 21, 23, 26, 27, 28], "matmul": [1, 2, 5], "matnat": 25, "matplotlib": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 20, 21, 23, 26, 27, 28], "matric": [0, 1, 3, 4, 6, 7, 8, 11, 13, 16, 17, 19, 27, 28], "matrix": [0, 2, 3, 4, 6, 7, 8, 10, 13, 17, 18, 21, 23], "matshow": 1, "matter": [2, 3, 13, 27, 28], "max": [0, 1, 2, 3, 4, 9, 10, 12, 13, 24, 26, 28], "max_depth": [0, 9, 10], "max_diff": 2, "max_diff1": 2, "max_diff2": 2, "max_it": [0, 1, 8, 13, 26], "max_iter": 14, "max_leaf_nod": 10, "max_sampl": 10, "maxdegre": [0, 6, 10, 27], "maxdepth": 10, "maxim": [1, 4, 5, 7, 8, 11], "maximum": [0, 2, 3, 5, 7, 8, 9, 10, 13, 14, 26, 27, 28], "maxpolydegre": [5, 6, 27, 28], "maxpooling2d": 3, "mbox": [5, 6, 27, 28], "mcculloch": 12, "md": 11, "mdoel": 4, "mean": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 23, 26], "mean_absolute_error": [0, 26], "mean_divisor": 14, "mean_i": 23, "mean_matrix": 14, "mean_squared_error": [0, 4, 6, 7, 10, 15, 26, 27], "mean_squared_log_error": [0, 26], "mean_vector": 14, "mean_x": 23, "meaning": [0, 4, 7, 26], "meansquarederror": [0, 26], "meant": [3, 7, 10, 13], "measur": [0, 1, 2, 5, 6, 9, 11, 12, 14, 16, 18, 21, 23, 26, 27], "mechan": [0, 4, 23, 26], "median": [0, 26, 27], "medicin": 12, "medium": [4, 8, 13], "medv": [], "meet": [0, 24], "mehta": [0, 26, 27, 28], "memori": [3, 4, 11, 12, 13, 18, 20], "mention": [0, 12, 13, 21, 23, 26, 28], "mere": [0, 21], "meshgrid": [2, 5, 6, 8, 9, 10, 11], "mess": 15, "messag": [5, 13], "messi": 2, "met": [0, 3, 8, 27], "meteorolog": 9, "meter": [6, 27], "method": [0, 1, 2, 3, 4, 5, 7, 8, 11, 12, 14, 15, 16, 17, 18, 19, 20, 23, 25, 27], "metion": 6, "metric": [0, 1, 3, 6, 7, 9, 10, 14, 15, 26, 27], "metropoli": [19, 26], "mev": [0, 23, 26], "mgd": 13, "mglearn": [19, 26], "mgrid": 13, "mhjensen": [], "mi": 10, "mia": [24, 26], "microsoft": 25, "mid": 1, "midel": 4, "midnight": 15, "midpoint": 9, "might": [0, 1, 2, 4, 6, 9, 13, 15, 17, 18, 27, 28], "migth": 17, "mild": 9, "millimet": [6, 27], "million": [0, 26, 27], "mimic": 12, "min": [0, 2, 5, 8, 9, 28], "min_": [0, 2, 5, 14, 17, 26, 27, 28], "min_samples_leaf": 9, "mind": [0, 6, 13, 15, 18, 26, 27, 28], "mindboard": 4, "mine": [19, 26], "mini": [1, 11, 12, 13, 28], "minibatch": [1, 11, 13], "minibathc": 13, "miniforge3": [], "minim": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 27, 28], "minima": [0, 1, 7, 13, 26, 28], "minimum": [0, 1, 2, 6, 8, 9, 11, 13, 27, 28], "minmaxscal": [0, 27], "minor": 23, "minst": 1, "minu": 7, "mirjalili": 26, "mirror": 9, "misc": 6, "misclassif": [8, 9, 10], "misclassifi": [8, 10], "miser": 0, "mismatch": 1, "miss": [7, 10], "mistak": 4, "mit": 25, "mix": [1, 2, 26], "mixtur": 13, "mk": [9, 20], "mkdir": [0, 6, 7, 9, 26], "ml": [0, 1, 10, 13, 20, 21, 27, 28], "mlab": 23, "mle": [5, 7], "mlp": 1, "mlpclassifi": 1, "mlpregressor": [0, 26], "mm": 20, "mml": 27, "mn": [12, 23], "mnist": [1, 11], "mod": 23, "mode": [22, 24, 26], "model": [2, 3, 5, 7, 8, 9, 10, 11, 13, 14, 16, 18, 19, 21, 23, 25, 27, 28], "model_select": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 26, 27, 28], "moder": 10, "modern": [0, 6, 7, 19, 26], "modif": [2, 12, 13], "modifi": [0, 1, 3, 5, 7, 8, 10, 12, 13, 26, 27, 28], "modul": [0, 16, 20, 26], "modular": 23, "modulo": 23, "moe": [11, 27], "moment": [5, 6, 13, 23], "mondai": [24, 26], "monitor": [13, 18], "monoton": [5, 12, 23], "mont": [0, 6, 19, 23, 25, 26], "montli": 16, "moor": [5, 6], "more": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 23], "moreov": [0, 3], "morten": [24, 26, 27, 28], "mortenhj": 26, "most": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 19, 21, 23, 26, 27, 28], "mostli": [1, 11, 18], "motion": [0, 13], "motiv": [1, 4], "move": [0, 4, 5, 6, 7, 9, 12, 13, 14, 15, 16, 21, 23, 27, 28], "mpl": [7, 26], "mpl_toolkit": [2, 6, 13, 28], "mplot3d": [2, 6, 13, 28], "mplregressor": 1, "mse": [0, 4, 5, 6, 9, 10, 15, 16, 17, 18, 21, 26, 27, 28], "mse_simpletre": 10, "mselassopredict": [5, 28], "mselassotrain": [5, 28], "mseownridgepredict": [6, 27, 28], "msepredict": [5, 28], "mseridgepredict": [0, 5, 6, 27, 28], "msetrain": [5, 28], "msle": [0, 26], "mt": [7, 12], "mu": [0, 6, 11, 13, 23, 26], "mu0": 23, "mu1": 23, "mu2": 23, "mu_": [6, 23, 27], "mu_i": [6, 27], "mu_n": 11, "mu_x": 23, "much": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 20, 21, 23, 26, 27, 28], "multi": [0, 1, 3, 7, 19, 26], "multiclass": [1, 7], "multidimension": [11, 12, 26], "multilay": 1, "multinomi": 7, "multipl": [2, 4, 5, 6, 7, 12, 13, 15, 23, 27, 28], "multipli": [3, 5, 6, 11, 13, 18, 20, 23, 27, 28], "multiplum": 8, "multivari": [0, 2, 10, 11, 19, 23, 26], "multivariate_norm": [11, 14], "multpli": 16, "murphi": [11, 25, 26], "must": [1, 2, 5, 6, 8, 10, 12, 13, 14, 15, 23, 27, 28], "mutat": 7, "mutual": [1, 3, 6, 13], "mx_": 23, "my": 26, "myenv": [], "myriad": [0, 19, 26], "mz1": 23, "mz2": 23, "m\u00f8svatn": 6, "n": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 23, 26, 27, 28], "n1": 20, "n2": 20, "n_": [1, 2, 3, 8, 12, 23], "n_0": [12, 23], "n_boostrap": [6, 10], "n_bootstrap": 6, "n_categori": [1, 3], "n_cluster": 14, "n_compon": 11, "n_epoch": 13, "n_estim": 10, "n_examples_to_gener": 4, "n_featur": [1, 18], "n_filter": 3, "n_hidden": 2, "n_hidden_neuron": [0, 1, 26], "n_i": 23, "n_input": [0, 1, 3, 27], "n_instanc": 9, "n_job": 10, "n_k": 14, "n_l": [12, 23], "n_layer": 1, "n_m": 9, "n_neuron": 1, "n_neurons_connect": 3, "n_neurons_layer1": 1, "n_neurons_layer2": 1, "n_point": 14, "n_sampl": [6, 8, 9, 10, 14, 18], "n_split": 6, "n_step": 4, "n_t": 2, "n_x": 2, "nabla": [1, 13, 28], "nabla_": [2, 13, 28], "nabla_w": 13, "nag": 13, "naimi": [0, 26], "naiv": 7, "naive_kmean": 14, "name": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 19, 20, 21, 23, 24, 26, 27, 28], "narrow": 13, "nation": [1, 5], "nativ": [19, 26], "natur": [0, 1, 4, 8, 9, 12, 13, 21, 23, 25, 26, 28], "navier": 12, "navig": 15, "nb": 23, "nb_": 20, "nbconvert": 26, "nd": 14, "ndarrai": 6, "ne": [9, 10, 20, 23, 27, 28], "nearest": [1, 3, 6, 11], "nearli": [13, 28], "neat": 26, "neccesari": 6, "necess": 2, "necessari": [0, 1, 3, 4, 8, 14, 18, 26], "necessarili": [0, 4, 11, 23, 26], "necesserali": 5, "neck": 7, "need": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 23, 27, 28], "neg": [0, 1, 3, 5, 6, 7, 10, 13, 20, 23, 26, 28], "neg_mean_squared_error": 6, "neglect": 23, "neglig": 23, "neighbor": [3, 6, 11], "neither": [4, 13], "neq": [13, 14, 23, 28], "nervou": 12, "nest": [9, 12], "nesterov": 13, "net": [2, 4, 12], "netlib": [20, 26], "network": [0, 9, 13, 19, 25, 27], "neural": [0, 13, 19, 25, 27], "neural_network": [0, 1, 2, 26], "neuralnetwork": 1, "neuron": [1, 2, 3, 4, 12], "neutral": [0, 26], "neutron": [0, 26], "never": [1, 4, 6, 9, 23], "new": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 20, 26, 27, 28], "new_chang": 13, "new_hobbit": 26, "newaxi": [0, 3, 6, 9], "newli": [0, 26], "newton": [1, 7, 8, 13, 23], "next": [0, 1, 2, 3, 4, 5, 6, 8, 9, 13, 14, 15, 16, 26, 27, 28], "next_guess": 13, "next_input": 4, "ng": 1, "ni": 14, "nice": [0, 1, 5, 11, 26, 27, 28], "nicer": 18, "niter": [13, 28], "nitric": [], "nlambda": [0, 5, 6, 27, 28], "nlp": 25, "nm": 23, "nm_n": [0, 26], "nmse": 6, "nn": [2, 5, 6, 12, 20, 26], "nn_model": 1, "nnmin": 2, "node": [1, 3, 9, 10, 12], "nois": [0, 4, 5, 6, 8, 9, 10, 13, 18, 21, 26, 27, 28], "noise_dimens": 4, "noisi": [1, 6, 21], "non": [0, 1, 3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 18, 20, 23, 26, 27, 28], "none": [0, 1, 2, 4, 5, 9, 10, 13, 23, 26, 27], "nonlinear": [3, 6, 8, 9, 11, 12], "nonneg": [6, 9, 13, 28], "nonparametr": 6, "nonsens": 23, "nonsingular": 20, "nonumb": [3, 7, 8, 13, 20], "nor": [1, 4, 13], "norm": [0, 1, 5, 6, 8, 11, 13, 18, 26, 27, 28], "normal": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 20, 21, 23, 26, 27, 28], "normali": [20, 26], "norwai": [6, 21, 26, 28], "notat": [0, 2, 5, 6, 13, 14, 23, 26, 27, 28], "note": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 19, 20, 23, 25, 26], "notebook": [0, 1, 3, 9, 15, 16, 19, 21, 26], "noth": [1, 2, 5, 8, 12, 14, 23, 27, 28], "notic": [4, 5, 12, 13, 20, 23, 26], "notion": 3, "novel": [3, 6, 10, 26], "novemb": [1, 24, 26], "now": [0, 2, 4, 5, 6, 7, 8, 10, 11, 12, 14, 15, 16, 19, 20, 21, 23, 26, 27], "nowadai": [0, 1, 3, 9, 19, 26], "nox": [], "np": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 20, 23, 26, 27, 28], "npr": 2, "nsampl": 6, "nt": 2, "nu": 23, "nuclear": [5, 27, 28], "nuclei": [0, 23, 26], "nucleon": [0, 26], "nucleu": [0, 26], "num": 4, "num_coordin": 2, "num_hidden_neuron": 2, "num_it": [2, 18], "num_neuron": 2, "num_neurons_hidden": 2, "num_point": 2, "num_tre": 10, "num_valu": 2, "number": [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 26, 28], "numberid": 7, "numberparamet": 3, "numer": [0, 5, 6, 9, 10, 11, 12, 13, 19, 20, 25, 26, 27, 28], "numpi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 27, 28], "nunmpi": [5, 27], "nve_frngahw": 28, "nx": 2, "ny": 23, "o": [0, 1, 4, 5, 6, 7, 8, 9, 11, 20, 24, 25, 26, 27, 28], "obei": [6, 11, 13, 27], "object": [0, 1, 4, 8, 10, 15, 20, 26], "obliqu": [5, 27, 28], "observ": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 23, 26, 28], "obtain": [0, 1, 5, 6, 7, 8, 9, 10, 12, 13, 14, 17, 20, 21, 23, 26, 27, 28], "obviou": [5, 6, 11, 23, 27, 28], "obviouli": 26, "obvious": [0, 4, 5, 6, 20, 26], "oc": [27, 28], "occupi": [], "occur": [0, 6, 8, 9, 20, 23, 26], "octob": [24, 26], "od": 0, "odd": [0, 3, 7, 26, 27], "odenum": 2, "odesi": 2, "oen": 0, "off": [1, 3, 4, 5, 9, 13, 23], "offer": [6, 11, 19, 20, 22, 24, 26], "offic": [24, 26], "offici": [22, 26], "often": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 23, 26, 27, 28], "ofter": [20, 26], "ol": [0, 13, 17, 27], "old": [1, 5, 10, 13, 15, 18], "ols_paramet": 16, "ols_sk": 6, "ols_svd": 6, "olsbeta": 28, "olstheta": [0, 5], "omega": [2, 3, 6], "omega_0": 3, "omit": [0, 5, 26, 27, 28], "onc": [1, 6, 9, 11, 13], "one": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 19, 20, 21, 23, 24, 26, 27], "onehot": 1, "onehot_vector": 1, "onehotencod": 9, "ones": [0, 2, 5, 6, 8, 9, 10, 11, 13, 16, 18, 20, 21, 26, 27, 28], "ones_lik": 4, "ong": 27, "onl": 3, "onli": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 20, 21, 23, 26, 27, 28], "onlin": [11, 15, 22], "onto": [5, 11, 27, 28], "open": [0, 1, 4, 6, 7, 9, 15, 19, 21, 22, 24, 26], "oper": [0, 1, 3, 5, 6, 10, 11, 12, 13, 15, 16, 19, 23, 26, 27, 28], "operation": 23, "oplu": 23, "opmiz": 13, "opportun": 0, "oppos": [6, 13], "opposit": [1, 5, 8, 27, 28], "opt": [1, 5, 21, 26, 28], "optim": [0, 2, 3, 4, 5, 6, 7, 9, 10, 11, 14, 16, 17, 21], "optimis": [1, 3], "option": [0, 1, 3, 5, 6, 8, 11, 15, 18, 20, 27], "optmiz": [1, 8, 13, 27], "oral": 26, "orang": 0, "order": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 15, 20, 21, 23, 26, 27, 28], "ordinari": [0, 2, 3, 7, 11, 13, 17, 18, 19], "oreilli": [25, 26], "org": [0, 3, 4, 16, 19, 20, 21, 25, 26, 27, 28], "organ": [6, 7, 10, 20], "orient": [1, 5, 23, 27, 28], "origin": [0, 3, 5, 6, 8, 11, 12, 13, 15, 20, 26, 27, 28], "orthogn": [5, 27, 28], "orthogon": [0, 5, 6, 8, 11, 13, 20, 26, 27, 28], "orthonorm": [5, 27, 28], "os": [24, 26], "oscar": 1, "oscil": [3, 13], "oskar": 26, "oskarlei": 26, "osl": 18, "oslo": [0, 19, 21, 22, 24, 26, 27, 28], "osx": [0, 19, 21, 26], "other": [0, 1, 2, 3, 5, 6, 7, 8, 10, 13, 14, 16, 19, 22, 23, 24, 25, 27, 28], "otherwis": [0, 1, 4, 7, 13, 20, 26], "ouput": [5, 7, 12], "our": [1, 2, 3, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 23], "ourmodel": 0, "ourselv": [0, 5, 6, 8, 11, 13, 26, 27, 28], "out": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 19, 20, 21, 23, 26, 27], "out_fil": 9, "outcom": [0, 7, 9, 10, 12, 23, 27], "outdoor": 9, "outer": [6, 12, 13], "outfil": 4, "outlier": [0, 8, 26, 27], "outlin": [6, 10, 11], "outlook": 9, "outperform": 10, "output": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 20, 21, 23, 26, 27, 28], "output_bia": 1, "output_bias_gradi": 1, "output_shap": 4, "output_weight": 1, "output_weights_gradi": 1, "outputlayer1": 12, "outputlayer2": 12, "outsid": 4, "over": [0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 15, 16, 20, 21, 26, 27, 28], "over1": 13, "overal": [1, 10], "overcast": 9, "overcom": [12, 13], "overdetermin": [0, 26], "overfit": [0, 1, 3, 6, 9, 10, 13], "overflow": 5, "overhead": 12, "overlap": [3, 7, 8, 9], "overlin": [0, 5, 6, 9, 10, 11, 14, 20, 26, 27], "overst": 0, "overtrain": 4, "overview": 3, "own": [4, 5, 6, 8, 12, 13, 16, 18, 19, 20, 28], "owner": [], "ownmsepredict": 0, "ownmsetrain": 0, "ownridgebeta": 27, "ownridgetheta": [0, 6, 27, 28], "ownypredictridg": 0, "ownytilderidg": 0, "oxid": [], "p": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 20, 23, 26, 27, 28], "p0": 2, "p1": 2, "p_": [2, 4, 8, 9], "p_hidden": 2, "p_i": [5, 23], "p_j": 23, "p_n": 23, "p_output": 2, "p_x": 23, "pack": [0, 26], "packag": [0, 1, 3, 4, 5, 8, 11, 13, 15, 19, 21, 23, 27, 28], "packtpub": 26, "packtpublish": 26, "pad": [3, 4], "page": [0, 19, 26, 28], "pai": [0, 1, 9, 13, 15], "pair": [0, 2, 3, 9, 19, 23, 26], "paltform": 15, "panda": [0, 4, 5, 6, 7, 9, 11, 19, 21, 28], "panel": 26, "paper": 1, "paradigm": [0, 26], "parallel": [10, 13, 19, 20, 26], "param": 2, "paramat": 2, "paramet": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 16, 17, 18, 21, 23, 28], "parameter": [0, 6, 10, 26, 27], "parametr": [0, 6, 26, 27], "paramt": [3, 5], "part": [0, 1, 3, 5, 6, 10, 17, 20, 22, 23, 24, 26, 27], "partial": [0, 1, 5, 6, 7, 8, 10, 11, 12, 13, 16, 23, 26, 27, 28], "particip": [15, 19, 22, 24, 26], "particl": [0, 4, 13, 23, 26], "particular": [0, 1, 2, 3, 5, 6, 9, 10, 11, 12, 13, 16, 21, 23, 25, 26, 27, 28], "particularli": [5, 6, 8, 11, 13, 23, 27, 28], "partit": [1, 4, 9], "partli": [6, 26], "partner": 15, "pass": [2, 3, 12, 14], "password": 21, "past": [10, 23], "patch": [6, 23], "path": [0, 4, 6, 7, 9, 19, 26], "pathcollect": 17, "patient": 7, "patter": 4, "pattern": [0, 3, 4, 12, 25, 26], "pauli": [0, 26], "pc": [11, 15, 19], "pca": [0, 7, 19, 26, 27], "pd": [0, 4, 5, 6, 7, 9, 11, 26, 27, 28], "pde": 2, "pdf": [0, 3, 4, 5, 6, 9, 15, 16, 21, 25, 26], "pedagog": [0, 26, 27], "penal": [6, 18, 27], "penalti": [6, 13, 18, 21, 27], "penros": [5, 6], "pentagon": [13, 28], "peopl": [1, 9, 13, 19], "per": [0, 1, 6, 22, 24, 26], "percentag": [10, 11, 24], "perceptron": [0, 1, 7, 26], "peregrin": 26, "perfect": [0, 1, 13, 26], "perfectli": [4, 6], "perform": [0, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 23, 26, 27, 28], "performac": 4, "perhap": [0, 5, 13, 26, 27, 28], "perimet": 1, "period": [1, 4, 23], "permiss": 15, "permut": 11, "persist": 13, "person": [5, 6, 7, 16, 22, 24, 26, 27], "perspect": 25, "pertin": [12, 26], "petal": [8, 9], "peter": [25, 27], "phantom": 23, "phase": [6, 12], "phenomena": 23, "phi": 8, "phi_k": 8, "philosophi": 13, "phone": [24, 26], "photo": [4, 26], "php": 21, "phrase": [0, 26], "physic": [0, 1, 4, 7, 12, 13, 23, 24, 25, 26, 27, 28], "pi": [2, 3, 5, 6, 7, 9, 12, 13, 23], "pick": [1, 9, 10, 11, 13, 14], "pickl": 1, "pictur": [0, 26], "pie": [19, 26], "piec": [11, 14], "pillow": [0, 19, 21, 26], "pinv": [5, 6, 13, 21, 27, 28], "pip": [0, 1, 15, 19, 21, 26], "pip3": [0, 1, 21, 26], "pipelin": [0, 6, 8, 10, 27], "pippin": 26, "pit": 4, "pitfal": [6, 27], "pitt": 12, "pixel": [1, 3, 4, 26], "pixel_height": [1, 3], "pixel_width": [1, 3], "place": [0, 4, 6, 8, 13, 15, 20, 21, 26, 28], "plai": [0, 3, 4, 5, 6, 8, 11, 18, 19, 21, 26, 27, 28], "plain": [8, 10, 12, 13, 14, 28], "plan": [6, 9, 24, 25, 26], "plane": [8, 9], "plateau": [5, 28], "platform": [19, 26], "plausibl": 12, "pleas": [13, 21, 24, 26], "plenti": 1, "plethora": [3, 12], "plot": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 26, 27, 28], "plot_all_sc": [21, 27], "plot_confusion_matrix": [7, 10], "plot_count": 6, "plot_cumulative_gain": [7, 10], "plot_data": 1, "plot_dataset": 8, "plot_decision_boundari": [9, 10], "plot_import": 10, "plot_max": 4, "plot_min": 4, "plot_model": 4, "plot_numb": 4, "plot_predict": 8, "plot_regression_predict": 9, "plot_result": 4, "plot_roc": [7, 10], "plot_surfac": [2, 6, 13], "plot_train": 9, "plot_tre": [9, 10], "plt": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 20, 23, 26, 27, 28], "plu": [0, 3, 5, 7, 18, 26, 27], "pm": 8, "pmatrix": 2, "pml": 25, "pn": 3, "png": [0, 4, 6, 7, 9, 26], "point": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 20, 21, 23, 24, 26, 27, 28], "point_1": 4, "point_2": 4, "poisson": [19, 23, 26], "poli": [6, 8], "poly100_kernel_svm_clf": 8, "poly3": 0, "poly3_plot": 0, "poly_featur": [8, 9, 15], "poly_features10": 9, "poly_fit": 9, "poly_fit10": 9, "poly_kernel_svm_clf": 8, "poly_model": 15, "poly_ms": 15, "poly_predict": 15, "polydegre": [0, 5, 6, 10, 27], "polygon": [13, 28], "polym": 12, "polymi": 21, "polynomi": [0, 5, 6, 7, 8, 9, 10, 11, 15, 17, 21, 26, 27], "polynomial_featur": [6, 15, 16, 17], "polynomial_svm_clf": 8, "polynomialfeatur": [0, 6, 8, 9, 15, 16, 27], "polytrop": [0, 6], "pool": 3, "pool_siz": 3, "poor": [1, 13, 28], "poorli": [0, 27], "popul": [0, 5, 26, 27], "popular": [0, 1, 3, 6, 7, 8, 9, 11, 12, 15, 19, 20, 21, 23, 27], "popularli": [0, 26], "portabl": 10, "portion": [11, 13], "pose": [0, 4, 5, 6, 11, 23, 26], "posit": [0, 1, 2, 3, 5, 7, 8, 10, 11, 13, 14, 20, 23, 26, 27, 28], "possibl": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 19, 20, 21, 23, 24, 26, 27, 28], "possibli": [6, 8, 13, 21], "posterior": 5, "postpon": [0, 27], "postscript": 21, "postul": 5, "potenti": [0, 3, 5, 6, 12, 13, 27], "pott": 12, "power": [0, 1, 5, 6, 8, 9, 12, 13, 26, 27, 28], "pp": [5, 6], "practic": [0, 5, 6, 7, 8, 16, 18, 21, 23, 27], "practition": [0, 1, 3, 26], "pre": 26, "preced": [1, 11, 12, 23], "preceed": 4, "preceq": 8, "precis": [0, 2, 5, 11, 13, 20, 21, 23, 26, 27], "pred": 6, "predicit": 0, "predict": [0, 1, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 21, 25, 26, 27, 28], "predict_prob": 1, "predict_proba": [7, 10], "predictor": [0, 5, 6, 7, 9, 10, 11, 26, 27], "prefer": [0, 1, 6, 8, 9, 11, 13, 15, 19, 21, 26], "prepar": [0, 6, 20, 21, 26, 27], "preprocess": [0, 4, 6, 7, 8, 9, 10, 11, 15, 16, 17, 18, 21], "prerequisit": 0, "prescript": 21, "presenc": 13, "present": [0, 5, 6, 7, 9, 12, 13, 20, 21, 23, 26, 27, 28], "preserv": [3, 11, 20], "press": [13, 15, 25, 28], "pretrain": [1, 4], "pretti": [0, 4, 8, 9, 19, 21, 26], "prev_centroid": 14, "prevent": [13, 23], "previou": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 20, 21, 23, 27, 28], "previous": [2, 3, 9, 10, 23], "price": [0, 4, 9, 13], "primal": 8, "primari": [0, 7, 26], "prime": 23, "princip": [0, 5, 7, 19, 26, 27, 28], "principl": [0, 6, 7, 8, 14, 26], "print": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 20, 23, 26, 27, 28], "print_funct": [8, 9], "printout": [0, 26], "prior": [0, 5, 6, 26], "privat": 0, "prob": [1, 23], "probabilist": [0, 25, 26, 27], "probabl": [0, 1, 3, 4, 6, 7, 10, 13, 19, 26, 27], "problem": [0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 17, 19, 20, 21, 23], "probml": 25, "proce": [0, 5, 6, 7, 8, 9, 10, 11, 13, 20, 26, 27], "procedur": [2, 4, 5, 6, 8, 10, 11, 13, 27, 28], "proceed": 20, "process": [0, 2, 4, 6, 9, 10, 12, 13, 19, 20, 21, 23, 25, 26, 28], "prod": 25, "prod_": [1, 5, 7], "produc": [0, 3, 4, 5, 6, 9, 10, 11, 12, 13, 18, 19, 20, 23, 26, 27], "product": [0, 1, 3, 5, 6, 7, 8, 12, 13, 16, 17, 19, 20, 26, 27], "profess": [0, 26], "program": [0, 1, 4, 5, 6, 8, 12, 14, 15, 19, 20, 22, 23, 24, 26, 27], "programm": 20, "progress": [1, 4, 14], "prohibit": 6, "project": [0, 1, 2, 3, 5, 11, 13, 15, 19, 22, 27, 28], "project_root_dir": [0, 6, 7, 9, 26], "promin": 12, "promis": 8, "promot": [24, 26], "prone": [9, 15], "pronounc": [13, 19, 26], "proof": [0, 11, 12, 13, 26, 28], "propag": [2, 3, 13], "proper": [0, 2, 6, 7], "properli": [1, 6, 8, 10, 13, 18, 21], "properti": [0, 1, 3, 12, 13, 16, 20, 26], "proport": [0, 1, 5, 9, 11, 13, 23, 26, 27], "propos": [1, 4, 6, 10, 21, 26], "propto": [5, 13, 28], "proton": [0, 26], "prove": [3, 13, 28], "provid": [0, 1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 19, 20, 21, 23, 26, 27, 28], "proxi": [1, 13], "prune": 9, "pseudo": [20, 23], "pseudocod": 21, "pseudoinv": 5, "pseudoinvers": [5, 6, 21], "pseudorandom": [6, 23], "psychologi": [0, 26], "pt": 13, "public": [0, 15, 19, 26], "pull": 15, "punish": [0, 1, 26], "pure": [3, 9, 23], "purest": 9, "puriti": 9, "purpos": [0, 3, 10, 12, 14, 26], "push": 15, "put": 1, "py": 5, "pycod": 26, "pydata": 19, "pydot": 9, "pyhton2": 26, "pylab": [7, 26], "pypi": 19, "pyplot": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 20, 23, 26, 27, 28], "pythagora": 5, "python": [1, 2, 3, 5, 6, 8, 11, 12, 13, 14, 21, 23, 27], "python2": [0, 21], "python3": [0, 19, 21, 26], "pytorch": [0, 19, 21, 26], "q": [5, 6, 8, 11, 23], "qp": 8, "qquad": [2, 11, 13, 20], "qr": [5, 6, 20, 27, 28], "quad": [1, 13, 20], "quadrat": [0, 8, 9, 13, 26], "qualit": [4, 9, 21, 23], "qualiti": [0, 9, 19, 26, 27], "quantifi": 1, "quantil": 10, "quantit": [0, 6, 9, 21, 26], "quantiti": [0, 2, 5, 6, 7, 9, 10, 11, 12, 14, 16, 20, 23, 26, 27, 28], "quantum": [4, 12, 25, 26], "quartil": [0, 27], "quench": 5, "queri": 9, "question": [0, 5, 6, 9, 11, 12, 13, 21, 24, 26, 27], "qugan": 4, "quick": [4, 23], "quickli": [1, 3, 9, 11, 13, 28], "quit": [1, 5, 6, 9, 10, 12, 15, 27, 28], "quot": 4, "r": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 20, 21, 23, 27, 28], "r2": [0, 5, 6, 26, 27, 28], "r2_score": [0, 26], "r2score": [0, 26], "r_1": 9, "r_2": 9, "r_j": 9, "r_m": 9, "rad": [], "rade": [], "radial": [8, 12], "radioact": 23, "radiu": [0, 1, 27], "rain": 9, "ramp": 1, "ran0": 23, "ran1": 23, "ran2": 23, "ran3": 23, "rand": [0, 4, 5, 6, 9, 10, 13, 15, 20, 26, 27, 28], "randint": [6, 9, 13], "randn": [0, 1, 2, 5, 6, 9, 11, 13, 15, 18, 26, 27, 28], "random": [0, 1, 2, 3, 4, 5, 6, 8, 9, 13, 14, 15, 16, 17, 18, 19, 20, 26, 27, 28], "random_forest_model": 10, "random_index": 13, "random_indic": [1, 3], "random_st": [7, 8, 9, 10, 11], "randomforestclassifi": 10, "randomli": [1, 6, 9, 13, 14, 18, 28], "rang": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 18, 20, 23, 26, 27, 28], "rangl": [0, 6, 11, 23, 26, 27], "rangle_x": 23, "rank": [5, 27, 28], "rankdir": 4, "raphson": [1, 8, 13], "rapidli": 0, "rare": [1, 13], "raschka": [26, 27], "rasckha": 26, "rashcka": 28, "rate": [1, 2, 3, 4, 8, 9, 10, 12, 13, 18, 28], "rather": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 23, 26, 27, 28], "ratio": [4, 7, 9, 10, 11], "rational": [0, 26], "ravel": [5, 6, 7, 8, 9, 10, 11, 13, 20], "raw": 3, "rbf": [8, 11, 12], "rbf_kernel_svm_clf": 8, "rbf_pca": 11, "rc": 23, "rcond": [0, 26, 27], "rcparam": [1, 3, 7, 8, 9, 10, 23, 26], "re": [2, 4, 13, 15, 28], "reach": [1, 4, 5, 6, 9, 10, 12, 13, 14, 28], "read": [0, 2, 3, 4, 5, 6, 7, 8, 11, 12, 16, 17, 20, 21, 23, 25, 28], "read_csv": [0, 6, 7, 9], "read_fwf": [0, 26], "reader": [0, 6, 20, 23, 26, 27], "readi": [0, 1, 5, 6, 8, 10, 11, 12, 20, 26], "readili": 1, "readm": 15, "readthedoc": 19, "real": [0, 1, 4, 7, 10, 11, 12, 16, 18, 20, 27], "real_loss": 4, "real_output": 4, "realist": [8, 26], "realiti": 23, "realiz": [1, 12], "realli": [0, 1, 26], "rearrang": 13, "reason": [0, 1, 3, 4, 10, 13, 25, 26, 28], "reassign": 1, "recal": [5, 6, 9, 10, 11, 12, 20, 23, 26, 27, 28], "recast": 3, "receiv": [1, 3, 10, 12, 23], "recent": [0, 6, 13, 25], "recept": [3, 12], "receptive_field": 3, "recip": [0, 6, 7, 20, 21, 26, 27], "reciproc": 5, "recogn": [0, 4, 5, 10, 26], "recognit": [0, 1, 3, 12, 25, 26], "recommen": 26, "recommend": [0, 2, 3, 4, 5, 6, 8, 13, 15, 19, 20, 21, 25, 28], "reconsid": 9, "reconstruct": 11, "record": [10, 21, 22, 24, 26], "recreat": 15, "rectangl": [9, 13, 28], "rectangular": [5, 27, 28], "rectifi": [1, 3, 12], "recur": [0, 19, 26], "recurr": [0, 1, 19, 26], "recurs": [9, 19, 20, 26], "red": [0, 3, 4, 6, 8, 9], "redefin": [0, 10, 26, 27, 28], "redefinit": 28, "reduc": [1, 3, 5, 6, 9, 10, 11, 13, 26, 28], "reduct": [0, 10, 11, 19, 23, 26, 27], "reegress": 21, "refer": [0, 1, 2, 3, 5, 6, 11, 12, 13, 14, 20, 25, 26, 27, 28], "referenc": 2, "refin": 12, "refit": 6, "reflect": [0, 1, 4, 5, 21, 23, 26], "refresh": [19, 26], "refreshprogrammingskil": 26, "reg": [10, 11], "regard": [1, 9, 13], "regardless": [12, 16], "region": [3, 4, 6, 9, 12, 21], "regist": [6, 23], "reglasso": [5, 28], "regr_1": [0, 9], "regr_2": [0, 9], "regr_3": [0, 9], "regress": [1, 8, 11, 12, 16, 19, 20], "regressor": [0, 7, 10], "regridg": [0, 5, 6, 27, 28], "regular": [0, 3, 4, 5, 6, 7, 9, 13, 17, 18, 24, 26, 27, 28], "regularli": 15, "reilli": [0, 25, 26], "reinforc": [0, 8, 19, 26], "reiter": 1, "reject": 7, "rel": [0, 4, 6, 7, 9, 12, 13, 23, 26, 27], "relat": [0, 1, 3, 4, 5, 11, 13, 14, 20, 23, 26, 27, 28], "relationship": [0, 4, 9, 18, 26], "relativeerror": [0, 26, 27], "releas": [1, 19, 26], "relev": [0, 1, 5, 7, 11, 19, 21, 23, 26, 28], "reli": [0, 6, 8], "reliabilti": 21, "reliabl": [7, 23], "relu": [3, 4, 26], "remain": [1, 2, 4, 6, 12, 20, 23, 27], "remaind": 23, "reman": 2, "remark": 1, "rememb": [0, 8, 13, 20, 21, 26], "remind": [0, 5, 11, 13, 20, 23], "remot": 15, "remov": [4, 5, 6, 18, 27, 28], "renam": 15, "render": [0, 26, 27], "reorder": [5, 7, 27, 28], "reorgan": [0, 26], "repeat": [0, 1, 3, 4, 5, 6, 9, 10, 11, 13, 14, 20, 21, 23, 26, 27, 28], "repeated": 26, "repeatedli": [0, 6, 10, 13], "repet": 3, "repetit": [6, 26, 27], "rephras": [13, 28], "replac": [0, 1, 3, 4, 5, 6, 10, 12, 14, 19, 21, 26, 27, 28], "replica": 6, "repo": [15, 21], "report": 26, "repositori": [4, 21, 26], "repres": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 21, 23, 26, 27, 28], "represent": [0, 1, 3, 6, 23, 26], "representd": 3, "reproduc": [0, 5, 6, 9, 12, 15, 16, 18, 19, 23, 26, 27], "repuls": [0, 26], "request": [0, 13], "requir": [0, 1, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 17, 18, 20, 26, 27, 28], "res1": 2, "res2": 2, "res3": 2, "res_analyt": 2, "res_analytical1": 2, "res_analytical2": 2, "res_analytical3": 2, "resaml": 6, "resampl": [0, 7, 10, 19, 26, 27], "rescal": [0, 11, 12], "rescu": 5, "reseach": 6, "research": [0, 4, 13, 19, 25, 26], "resembl": [6, 23], "reserv": [1, 5, 6, 23], "reshap": [0, 1, 2, 3, 4, 6, 8, 9, 10, 20, 26, 27], "residenti": [], "residu": [0, 5, 13, 26], "resiz": [5, 27, 28], "resourc": 26, "respect": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 21, 23, 26, 27, 28], "respond": 12, "respons": [0, 7, 9, 12, 26, 27], "rest": [0, 5, 18, 27, 28], "restat": [0, 12, 26], "restor": 4, "restored_discrimin": 4, "restored_gener": 4, "restrict": [0, 3, 9, 12, 26], "result": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 26], "retail": [], "retain": [5, 6, 27, 28], "return": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 20, 23, 26, 27, 28], "return_data": 14, "return_sequ": 4, "return_x_i": 9, "reus": [1, 3, 6, 21], "reveal": [0, 12, 26], "revers": [1, 20], "review": [19, 20], "revisit": 14, "revolut": 26, "reward": [0, 4, 26], "rewrit": [0, 3, 5, 6, 7, 8, 10, 11, 12, 13, 16, 20, 21, 23, 28], "rewritten": [2, 6, 8, 10, 23], "rewrot": 13, "rf": 10, "rgb": 3, "rgoj5yh7evk": 19, "rh": 6, "rho": [0, 10, 13], "rho_1": 10, "rho_2": 10, "rho_m": 10, "rich": [0, 26], "ride": 9, "rideclass": 9, "ridedata": 9, "ridg": [7, 11, 13, 19, 26], "ridge_paramet": 17, "ridge_sk": 6, "ridgebeta": 28, "ridgetheta": 5, "right": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 20, 21, 23, 26, 27, 28], "right_sid": 2, "rightarrow": [0, 1, 5, 6, 8, 11, 12, 13, 23, 26, 27, 28], "rigor": [0, 26, 27, 28], "ring": 6, "rise": [0, 26], "risk": [0, 13, 26, 28], "rival": 4, "river": [], "rlm": 26, "rm": 23, "rmse": [], "rmsporp": 13, "rmsprop": [1, 3, 4, 13, 21], "rnd_clf": 10, "rng": 23, "rnn": [4, 12], "rnn1": 4, "rnn2": 4, "rnn_2layer": 4, "rnn_input": 4, "rnn_output": 4, "rnn_train": 4, "rntrick1": 23, "rntrick2": 23, "rntrick3": 23, "rntrick4": 23, "ro": [0, 13, 26, 28], "robert": [21, 25], "robust": [0, 26], "robustscal": [0, 27], "roc": [7, 10], "role": [0, 2, 5, 6, 8, 18, 19, 21, 26, 27, 28], "roll": 6, "room": [0, 24, 26], "root": [0, 5, 9, 13, 15, 23, 27, 28], "rot": 26, "rotat": [1, 8, 9, 10], "rotation_matrix": 9, "roughli": [1, 3, 18], "round": [7, 9, 13], "routin": [13, 20, 26, 28], "row": [0, 1, 2, 5, 6, 9, 11, 16, 20, 26, 27, 28], "rr": [5, 27, 28], "rrr": [5, 27, 28], "rudg": [], "rug": [13, 28], "rule": [0, 1, 5, 6, 13, 21, 26, 27, 28], "run": [0, 1, 2, 4, 5, 6, 8, 9, 11, 13, 15, 19, 21, 26, 27, 28], "runtim": [1, 6, 14, 15], "rust": [0, 19, 20, 26], "rvert": 1, "rvert_2": 1, "s_": [3, 6], "s_1": 6, "s_i": [6, 7], "s_j": 6, "s_k": 6, "s_phenomenon": 21, "saddl": [13, 28], "safeguard": 18, "sai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 20, 21, 23, 26, 27, 28], "said": [6, 9, 13, 28], "sake": [0, 5, 7, 11, 26, 27, 28], "sale": [0, 26], "sam": 26, "same": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 14, 15, 16, 18, 20, 21, 23, 26, 27, 28], "samm": 10, "sampl": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 18, 19, 20, 21, 23, 26, 27], "sample_vari": 14, "sampleexptvari": 23, "samwis": 26, "sastri": 11, "satisfactori": [0, 26], "satisfi": [1, 2, 3, 6, 8, 13, 20, 23, 28], "satur": [1, 6], "save": [0, 4, 6, 7, 9, 13, 26], "save_fig": [0, 6, 7, 9, 10, 26], "savefig": [0, 4, 6, 7, 9, 23, 26], "savetxt": 4, "saw": [5, 27], "scalabl": 10, "scalar": [2, 5, 6, 10, 27], "scale": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 19, 20, 21, 24, 26, 28], "scale_mean": 4, "scale_std": 4, "scaler": [0, 7, 8, 9, 10, 11, 17, 21, 27], "scan": [5, 7], "scari": 5, "scatter": [0, 1, 6, 7, 8, 9, 14, 15, 17, 26, 27], "scenario": [6, 13, 28], "schedul": 13, "scheme": [1, 13, 28], "schrage": 23, "sch\u00f8yen": [6, 27], "scienc": [0, 1, 10, 12, 13, 19, 22, 23, 24, 25, 28], "scientif": [0, 19, 21, 26], "scientist": [0, 26], "scikit": [3, 5, 6, 8, 9, 10, 13, 15, 16, 19, 20, 21, 25], "scikit_learn": 0, "scikitlearn": 26, "scikitplot": [7, 10], "scipi": [0, 3, 5, 6, 13, 19, 20, 21, 26, 27, 28], "scl": 6, "scm": 15, "score": [0, 1, 3, 6, 7, 9, 10, 11, 15, 16, 21, 24, 26, 27], "scores_kfold": 6, "scratch": [1, 13, 16], "sdg": 13, "sdv4f4s2sb8": 28, "seaborn": [0, 1, 3, 6, 7, 26], "seamless": [0, 19, 21, 26], "search": [0, 1, 3, 5, 9, 13, 15, 26, 28], "sebastian": 26, "sebastianraschka": 26, "sec": 6, "second": [0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 15, 16, 19, 20, 23, 24, 26, 27, 28], "secondeigvector": 11, "secondli": 12, "section": [4, 11, 16, 20, 21, 23, 27], "sector": 0, "see": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 23, 26, 27, 28], "seed": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 18, 23, 26, 27, 28], "seed_imag": 4, "seek": [1, 2, 8], "seem": [1, 3, 4], "seemingli": [0, 26], "seen": [0, 1, 3, 5, 10, 12, 23], "segment": [13, 28], "seismic": 6, "seldomli": [0, 26], "select": [1, 5, 6, 8, 9, 10, 11, 15, 21, 22, 23, 24, 25, 26, 27, 28], "selevet": 15, "self": [1, 5, 27], "sell": 4, "semest": [7, 22], "semi": [8, 13, 28], "semilogx": 6, "send": [5, 12, 13, 24, 26], "senior": [22, 24], "sens": [0, 4, 6, 8, 26], "sensibl": 3, "sensit": [0, 5, 6, 9, 13, 26, 27], "sent": 2, "sentenc": [4, 12], "separ": [0, 1, 2, 4, 6, 8, 9, 12, 14, 18, 19, 21, 23, 26], "septemb": [18, 21, 26], "sequenc": [3, 4, 7, 9, 10, 12, 13, 19, 20, 23, 26, 28], "sequenti": [1, 3, 4, 10, 12, 23], "seri": [0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 13, 20, 26, 27, 28], "serif": [7, 23, 26], "serv": [0, 1, 2, 3, 5, 7, 13, 25, 26, 27, 28], "servic": 21, "session": [1, 15, 21, 22, 24, 26], "set": [1, 4, 5, 6, 7, 8, 10, 11, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24], "set_major_formatt": 6, "set_major_loc": 6, "set_tick": [1, 8], "set_ticklabel": 1, "set_titl": [0, 1, 2, 3, 7, 12, 14, 26], "set_xlabel": [0, 1, 2, 3, 7, 12, 26], "set_xlim": [7, 12], "set_xticklabel": 1, "set_ylabel": [0, 1, 2, 3, 7, 26], "set_ylim": [7, 12], "set_ytick": 7, "set_yticklabel": [1, 6], "set_zlim": 6, "seth": 4, "setminu": 6, "setosa": [8, 9], "setosa_or_versicolor": 8, "setp": 6, "setup": [1, 4, 6, 8, 19, 26, 27, 28], "sever": [0, 3, 5, 6, 7, 8, 9, 11, 12, 13, 16, 19, 20, 21, 23, 26, 27, 28], "sgd": [1, 3, 28], "sgd_clf": 8, "sgdclassifi": 8, "sgdreg": 13, "sgdregressor": 13, "sgn": [5, 27, 28], "shallow": 13, "shape": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 20, 26, 27, 28], "share": [1, 3, 15, 26], "shareabl": 15, "she": 7, "shift": [1, 6, 12, 15, 18, 23, 27], "ship": 3, "shire": 26, "short": [4, 5, 21], "shortcom": [13, 28], "shorten": 4, "shorter": 23, "shorthand": 26, "shortli": [20, 26], "should": [0, 2, 3, 5, 6, 8, 9, 11, 12, 15, 18, 20, 21, 23, 26, 27], "show": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 20, 21, 23, 26, 27, 28], "show_shap": 4, "shown": [0, 4, 5, 8, 12, 13, 20, 27, 28], "shrink": [3, 5, 6, 8, 11, 27, 28], "shrinkag": [5, 6, 27, 28], "shrunk": 11, "shuffl": [0, 1, 4, 6, 13, 27], "side": [0, 2, 5, 8, 12, 13, 20, 21, 26, 28], "sigh": [19, 26], "sigma": [0, 1, 5, 6, 7, 10, 11, 12, 13, 20, 21, 23, 26, 27, 28], "sigma0": 23, "sigma1": 23, "sigma2": 23, "sigma_": [5, 20, 26, 27, 28], "sigma_0": [5, 27, 28], "sigma_1": [5, 27, 28], "sigma_2": [5, 27, 28], "sigma_fn": [7, 12], "sigma_i": [0, 5, 26, 27, 28], "sigma_j": [5, 27, 28], "sigma_m": [6, 23], "sigma_n": [11, 23], "sigma_t": 13, "sigma_x": 23, "sigmoid": [1, 2, 4, 7, 8, 10, 12], "sigmundson": [6, 27], "sign": [1, 2, 7, 8, 10, 23, 24], "signal": [1, 3, 10, 12], "signifi": 4, "signific": 1, "significantli": [1, 13, 18, 23, 28], "sim": [4, 5, 6, 13, 23], "similar": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 19, 20, 21, 26, 28], "similarli": [0, 1, 3, 5, 8, 10, 13, 23, 26, 27, 28], "simpl": [1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 14, 16, 17, 19, 20, 23], "simplepredict": 10, "simpler": [0, 1, 5, 6, 7, 13, 16, 19, 21, 26, 28], "simplernn": 4, "simplest": [0, 1, 3, 4, 9, 10, 12, 14, 21, 26], "simpletre": 10, "simpli": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, 19, 20, 21, 23, 26, 27, 28], "simplic": [2, 5, 6, 7, 8, 9, 10, 11, 12, 14, 27, 28], "simplicti": [5, 27, 28], "simplifi": [0, 6, 9, 18, 19, 21, 26, 27], "simplist": [3, 6, 23], "simul": [6, 18], "simultan": 6, "sin": [0, 1, 2, 3, 4, 9, 12, 13, 20, 26], "sinc": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 16, 18, 20, 21, 23, 25, 26, 27, 28], "sine": [3, 12], "singl": [0, 1, 2, 3, 5, 6, 7, 8, 9, 12, 13, 18, 20, 23, 26, 27, 28], "singular": [0, 6, 13, 20, 26], "sinusoid": 3, "site": [0, 21, 22, 27], "situat": [0, 4, 5, 7, 13, 23, 26, 27, 28], "six": [3, 23], "size": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 13, 18, 20, 21, 23, 26], "sketch": 10, "ski": 9, "skill": 0, "skip": 11, "skl": [0, 6, 26, 27], "sklearn": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 26, 27, 28], "skplt": [7, 10], "sl": [6, 27], "slack": 8, "slice": [2, 20, 26], "slide": [0, 3, 16, 21, 23, 26, 27, 28], "slight": [6, 13], "slightli": [1, 2, 3, 5, 6, 7, 10, 23, 27, 28], "slope": [8, 11, 12], "slow": [0, 2, 8, 13, 18, 27, 28], "slower": [5, 20, 26, 27, 28], "slowest": 20, "slowli": 12, "slp": 1, "small": [0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 19, 20, 23, 26, 27, 28], "smaller": [0, 1, 2, 5, 6, 8, 9, 11, 13, 23, 26, 27, 28], "smallest": [0, 4, 14, 26], "smallest_row_index": 14, "smooth": [0, 3, 6, 13, 21, 26, 28], "sn": [0, 1, 3, 6, 7, 26], "sne": 11, "so": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 24, 26, 27, 28], "soar": 6, "social": 0, "soft": [1, 7, 10, 12], "soften": 8, "softmax": [3, 7], "softwar": [0, 8, 19, 20], "sol": 8, "sole": [0, 6, 26], "solid": [0, 7], "solut": [0, 1, 2, 3, 5, 6, 8, 10, 11, 13, 18, 20, 21, 23, 26, 27, 28], "soluton": 2, "solv": [0, 1, 3, 5, 6, 8, 10, 11, 12, 13, 16, 20, 21, 26, 27], "solve_expdec": 2, "solve_ode_deep_neural_network": 2, "solve_ode_neural_network": 2, "solve_pde_deep_neural_network": 2, "solveod": 2, "solveode_popul": 2, "solver": [2, 7, 8, 9, 10, 20, 26], "some": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 21, 23, 26], "some_model": [6, 27], "somehow": 4, "someon": 16, "someth": [0, 1, 3, 4, 7, 9, 11, 15, 21, 23, 26, 27], "sometim": [0, 1, 11, 12, 13, 14, 27], "soon": [20, 24, 27], "sophist": [0, 26], "sopt": 13, "sort": [5, 6, 9, 11, 23], "sound": [3, 5], "sourc": [0, 1, 3, 6, 19, 20, 21, 23, 26], "space": [0, 1, 4, 5, 8, 9, 11, 12, 13, 14, 23, 27, 28], "span": [0, 3, 5, 9, 11, 20, 26, 27, 28], "spare": 1, "spars": [3, 6, 18, 20, 26], "sparse_mtx": [20, 26], "sparsecategoricalcrossentropi": 3, "sparsiti": [10, 18], "spatial": [1, 2, 3, 12], "speak": 23, "special": [6, 7, 10, 12, 13, 20, 23, 26, 27, 28], "specif": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 15, 16, 19, 20, 21, 23, 25, 26, 27, 28], "specifi": [0, 3, 5, 6, 7, 9, 11, 13, 14, 23, 26, 28], "specifici": [0, 10, 26], "spectacular": 3, "spectral": 1, "speech": [0, 1, 3, 4, 12], "speed": [1, 2, 4, 13], "spend": [16, 23], "spent": 21, "sphere": [0, 27], "spin": 6, "spite": 0, "spline": 8, "split": [1, 3, 4, 5, 6, 8, 9, 10, 11, 14, 16, 17, 21, 23, 26, 28], "splite": 0, "splitter": [1, 10], "spontan": 23, "spot": 3, "spread": [0, 11, 23, 26, 27], "springer": [21, 25, 26], "spuriou": 13, "sqquar": 28, "sqrsignal": 3, "sqrt": [3, 4, 5, 6, 8, 10, 11, 13, 23, 27, 28], "squar": [1, 2, 3, 4, 7, 8, 9, 11, 13, 14, 15, 17, 18, 19, 20, 23], "squarederror": 10, "squaredeuclidean": 14, "squash": 12, "srtm": 6, "srtm_data_norway_1": 6, "stabil": [5, 21], "stabl": [0, 4, 5, 6, 9, 16, 19, 21, 26, 27, 28], "stack": [3, 4], "stage": [5, 13, 15, 21], "stai": [0, 2, 4, 5, 11, 26, 27], "stand": [0, 5, 9, 12, 26, 27, 28], "standard": [0, 1, 4, 5, 6, 7, 8, 10, 12, 17, 18, 20, 21, 23, 26, 28], "standardscal": [0, 6, 7, 8, 9, 10, 11, 17, 27], "stanford": [13, 28], "start": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 23, 24, 26, 27, 28], "start_tim": 14, "stat": 6, "state": [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 19, 23, 26, 27, 28], "statement": [0, 7, 20, 26], "stationari": 28, "statist": [0, 1, 3, 4, 7, 9, 10, 11, 12, 13, 14, 20, 21, 25, 27, 28], "statu": [0, 7, 15, 26], "stavang": 6, "std": [0, 4, 6, 18, 26, 27], "steep": [13, 28], "step": [0, 1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 15, 18, 20, 21, 26, 28], "step_fn": [7, 12], "step_length": 13, "steps_list": 9, "stereo": 3, "still": [0, 2, 3, 5, 6, 11, 13, 23, 27, 28], "stimuli": 12, "stk": [25, 26], "stk2100": [25, 26], "stk3155": [15, 21, 22, 24], "stk4021": [25, 26], "stk4051": [25, 26], "stk4155": [22, 24], "stk5000": 25, "stochast": [0, 1, 5, 6, 8, 11, 12, 28], "stock": 4, "stoke": 12, "stone": [0, 7], "stop": [1, 4, 9, 13, 14, 18, 28], "storag": [5, 27, 28], "store": [0, 1, 2, 3, 6, 11, 13, 18, 23, 26], "storehaug": [24, 26], "str": [1, 3, 4], "straight": [0, 6, 8, 13, 26, 28], "straightforward": [0, 2, 3, 5, 6, 8, 9, 10, 13, 20, 26, 27, 28], "strategi": [0, 1, 9, 26], "stratifi": 6, "strength": [0, 5, 14, 27, 28], "stretch": 11, "strict": [8, 13, 28], "strictli": [8, 13, 28], "stride": [4, 20], "strike": 6, "string": 1, "stroke": 7, "strong": [3, 6, 9, 10, 12, 20, 23], "strongli": [0, 8, 15, 19, 20], "stronli": [], "structur": [0, 1, 2, 3, 6, 9, 10, 12, 19, 26], "stuck": [1, 13, 28], "student": [0, 15, 21, 22, 24, 25, 26], "studi": [0, 3, 4, 5, 6, 7, 8, 11, 12, 13, 19, 21, 25, 26, 27, 28], "studier": 25, "style": [7, 9, 20, 26], "st\u00f8land": 24, "sub": [9, 12], "subdivid": [0, 20, 26], "subfield": 0, "subject": [6, 8, 23], "submit": 26, "subplot": [0, 1, 3, 4, 6, 7, 8, 9, 10, 14, 26], "subplots_adjust": [8, 23], "subprogram": [20, 26], "subract": [0, 27], "subroutin": [0, 26], "subscript": 1, "subsequ": [1, 4, 5, 6, 12, 20, 23, 27, 28], "subset": [1, 6, 9, 12, 13, 19, 26, 28], "subspac": [0, 8, 11, 27], "substanti": [9, 10], "substep": 11, "substitut": [3, 6, 12, 16, 20], "subsubset": 9, "subtask": 6, "subtl": 1, "subtract": [0, 4, 5, 6, 11, 13, 18, 20, 21, 23, 27], "subtre": 9, "succeed": [0, 4, 26], "success": [3, 7, 9, 13, 23], "successfulli": [4, 9], "sudo": [0, 19, 21, 26], "suffer": [0, 1, 2, 5, 10, 26, 27, 28], "suffici": [1, 6, 8, 11, 13, 28], "suggest": [1, 13, 21, 25, 28], "suit": [8, 12], "suitabl": [0, 15, 23, 27], "sum": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 23, 26, 27, 28], "sum_": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 20, 21, 23, 26, 27, 28], "sum_i": [0, 2, 5, 6, 8, 13, 21, 27, 28], "sum_j": [6, 18], "sum_ja_": 0, "sum_k": [6, 8, 12, 20], "sum_logist": 13, "sum_m": 3, "sum_n": 3, "sum_nx_": 3, "summar": [5, 6, 9], "summari": [1, 3, 4, 10, 22, 28], "summat": [0, 3, 16, 27, 28], "sunni": 9, "super": [5, 27, 28], "superfici": 3, "superscript": [1, 12], "supervis": [0, 5, 6, 7, 9, 12, 19, 26, 27, 28], "supplement": [7, 21], "support": [0, 1, 9, 10, 11, 13, 19, 26, 27], "suppos": [0, 5, 6, 7, 8, 10, 11, 12, 13, 20, 26, 27, 28], "suppress": [5, 13, 28], "sure": [0, 1, 4, 6, 16, 21], "surf": 6, "surfac": [0, 6, 26], "surpass": 6, "surpris": [0, 26], "surround": [3, 19], "survei": [0, 5, 6, 26, 27], "svc": [8, 9, 10], "svd": [0, 6, 11, 26], "svdinv": 5, "svm": [8, 9, 10, 11], "svm_clf": [8, 10], "swath": [5, 27, 28], "switch": 0, "sy": [13, 28], "symbol": [1, 5, 11, 13, 19, 23, 26, 27, 28], "symmeteri": 1, "symmetr": [0, 5, 8, 11, 12, 13, 20, 26, 27], "symmetri": 6, "sympi": [0, 19, 21, 26], "synonim": 23, "syntax": 13, "system": [0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 19, 20, 21, 26, 28], "systemat": [4, 6], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 26, 28], "t0": [3, 6, 13], "t1": [2, 13], "t2": 2, "t3": 2, "t_": 2, "t_0": [2, 9, 13], "t_1": 13, "t_b": 10, "t_i": [1, 2, 5, 12, 27, 28], "t_j": 12, "t_k": 9, "tabl": [9, 21, 23, 24, 26], "tabul": [0, 26], "tabular": 26, "tackl": 4, "tag": [2, 3, 4, 5, 6, 7, 12, 13, 14, 20, 23, 27, 28], "taht": [0, 26], "tail": 23, "tailor": [2, 8, 11, 26], "taiwan": [0, 26], "take": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 19, 20, 23, 26, 27, 28], "taken": [0, 1, 3, 6, 10, 13, 20], "tan": 3, "tangent": [1, 4, 12, 13, 28], "tanh": [1, 4, 7, 8, 12], "target": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 18, 26, 27, 28], "target_nam": 9, "task": [0, 1, 3, 6, 9, 11, 12, 14, 21, 26], "tau": [3, 5, 23], "taught": 26, "tax": [], "taylor": [2, 13, 28], "taylornr": [13, 28], "tc": 8, "teach": [15, 22, 26], "team": 1, "teaser": 0, "technic": [0, 5, 6, 13, 21, 28], "techniqu": [0, 1, 8, 10, 13, 19, 23, 25, 26, 27], "technologi": [0, 1], "tell": [0, 4, 6, 10, 11, 13, 16, 23], "temp": 1, "temp1": 1, "temp2": 1, "temperatur": [0, 9, 26], "templat": 18, "temporarili": 1, "ten": [3, 26], "tend": [3, 5, 6, 8, 9, 10, 12, 13, 14, 27], "tendenc": [0, 26], "tension": 6, "tensor": 3, "tensorflow": [0, 2, 4, 8, 14, 19, 20, 21, 25, 26, 27], "term": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 21, 23, 26, 27, 28], "term1": [5, 6, 11], "term2": [5, 6, 11], "term3": [5, 6, 11], "term4": [5, 6, 11], "termin": [0, 4, 5, 9, 10, 13, 15, 27, 28], "terminarl": 15, "terrain": 6, "terrain1": 6, "test": [3, 4, 5, 6, 7, 8, 9, 10, 13, 16, 20, 21, 23, 26, 28], "test_acc": 3, "test_accuraci": [1, 3], "test_error": 6, "test_imag": [3, 4], "test_ind": 6, "test_input": 4, "test_label": [3, 4], "test_loss": 3, "test_pr": 1, "test_predict": 1, "test_rnn": 4, "test_scor": [7, 10], "test_siz": [0, 1, 3, 5, 6, 10, 15, 17, 27, 28], "test_split": 9, "testerror": [0, 6, 27], "testi": 4, "testpredict": 4, "testx": 4, "text": [0, 1, 2, 4, 5, 8, 9, 11, 13, 15, 18, 20, 23, 25, 27, 28], "textbook": [16, 21, 27, 28], "textual": 9, "textur": 1, "tf": [1, 3, 4, 13, 14, 28], "th": [0, 1, 2, 5, 6, 7, 9, 12, 13, 14, 20, 21, 23, 26, 27], "than": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 17, 19, 23, 26, 27], "thank": [4, 6, 27], "theano": [1, 19, 26], "thei": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 18, 20, 21, 23, 26, 27, 28], "them": [0, 1, 3, 4, 6, 8, 9, 10, 11, 12, 13, 18, 20, 21, 26, 27], "theme": [0, 15, 26], "themselv": [0, 21, 23, 26], "thenc": 6, "theorem": [2, 6, 7, 27, 28], "theoret": [0, 4, 10], "theori": [0, 1, 3, 8, 9, 12, 13, 19, 21, 25, 26], "thereaft": [0, 5, 6, 11, 12, 20, 21, 26], "therebi": [0, 5, 7, 11, 21, 26, 27, 28], "therefor": [0, 1, 2, 3, 4, 6, 7, 8, 11, 13, 23, 26, 27, 28], "therein": 11, "thereof": [0, 6, 13, 26], "theta": [0, 1, 4, 5, 6, 7, 13, 16, 21, 23, 26, 27, 28], "theta_": [0, 1, 6, 7, 13, 26, 27, 28], "theta_0": [0, 5, 6, 7, 16, 26, 27, 28], "theta_0x_": [0, 26, 27], "theta_1": [0, 5, 6, 7, 26, 27, 28], "theta_1x_": [0, 26, 27], "theta_1x_0": [0, 26], "theta_1x_1": [0, 7, 26], "theta_1x_2": [0, 26], "theta_1x_i": [7, 27, 28], "theta_2": [0, 26, 27], "theta_2x_": [0, 26, 27], "theta_2x_0": [0, 26], "theta_2x_1": [0, 26], "theta_2x_2": [0, 7, 26], "theta_2x_i": 27, "theta_3x_i": 27, "theta_4x_i": 27, "theta_closed_form": 18, "theta_closed_formol": 18, "theta_closed_formridg": 18, "theta_gdol": 18, "theta_gdridg": 18, "theta_i": [0, 1, 5, 26, 27, 28], "theta_j": [0, 5, 6, 18, 26, 27], "theta_k": 28, "theta_linreg": [13, 28], "theta_ol": 18, "theta_p": 7, "theta_px_p": 7, "theta_ridg": 18, "theta_t": 13, "theta_tru": 18, "thetavalu": 5, "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 27, 28], "thing": [0, 1, 2, 4, 5, 7, 9, 15, 16, 18, 23, 26], "think": [0, 1, 3, 4, 6, 9, 12, 13, 14, 23, 26, 27, 28], "third": [0, 3, 6, 13, 24, 26, 28], "thirti": 7, "thorughout": 26, "those": [0, 3, 5, 6, 8, 9, 10, 11, 20, 21, 26, 27, 28], "though": [1, 2, 3, 4, 13, 16, 17, 20, 23], "thought": [6, 14, 21, 23], "thousand": [0, 1, 21, 27], "three": [0, 1, 3, 5, 6, 8, 9, 12, 20, 21, 22, 23, 24, 26, 27, 28], "threshold": [1, 3, 9, 10, 11, 12, 13], "through": [0, 1, 2, 3, 4, 5, 6, 8, 11, 12, 13, 14, 15, 19, 20, 21, 23, 26, 27, 28], "throughout": [0, 4, 5, 14, 15, 19, 20, 23, 26], "throw": [3, 6, 23], "thu": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 24, 26, 27, 28], "thumb": [0, 6, 21, 27], "thursdai": [], "tibshirani": [6, 21, 25, 26], "tick_param": 6, "ticker": [6, 13, 23, 28], "tif": 6, "tight_layout": [1, 7], "tightli": 11, "tild": [0, 5, 6, 7, 11, 21, 23, 26, 27, 28], "till": [0, 4, 7, 8, 9, 10, 12, 20, 26, 27], "time": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 26, 27, 28], "timeit": 4, "timer": 4, "tini": 1, "tip": 3, "titl": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 13, 15, 23, 26, 28], "tmp": 13, "tn": [2, 3, 7], "to_categor": [1, 3, 4], "to_categorical_numpi": 1, "to_numer": [0, 6, 26], "todai": 3, "togeth": [0, 3, 6, 8, 11, 13, 19, 26], "toi": 14, "told": 13, "toler": [2, 14], "tolist": 4, "tomographi": 12, "too": [0, 2, 4, 5, 6, 9, 11, 13, 17, 18, 23, 25, 27, 28], "took": [8, 26], "tool": [0, 1, 3, 6, 13, 15, 19, 27], "toolbox": 8, "top": [0, 3, 5, 6, 9, 10, 19, 26], "topic": [0, 5, 6, 7, 8, 19, 21, 27, 28], "topolog": [3, 12], "topologi": [1, 12], "torkjellsdatt": [24, 26], "toss": [10, 23], "total": [0, 1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 13, 14, 20, 23, 24, 26, 27, 28], "total_loss": 4, "totalclustervari": 14, "totalscatt": 14, "toward": [1, 2, 7, 12, 13, 15, 28], "town": [], "tp": [4, 7], "tpng": 9, "tpu": [13, 19, 26], "tqdm": 6, "tr": [], "track": [3, 13, 14, 15, 20, 27, 28], "tract": [], "tractabl": [0, 26, 27], "trade": [5, 9], "tradeoff": [0, 5, 21, 26, 27, 28], "tradit": [0, 1, 4, 6, 26], "train": [2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 16, 17, 21, 28], "train_accuraci": [0, 1, 3, 26], "train_dataset": 4, "train_end": [0, 1, 27], "train_error": 6, "train_imag": [3, 4], "train_ind": 6, "train_label": [3, 4], "train_pr": 1, "train_siz": [0, 1, 3, 27], "train_step": 4, "train_test_split": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 26, 27, 28], "train_test_split_numpi": [0, 1, 27], "trainable_vari": 4, "trained_model": [6, 27], "trainerror": [0, 27], "traini": 4, "training_checkpoint": 4, "training_dataset": 4, "training_gradi": 13, "trainingerror": 6, "trainpredict": 4, "trainscor": 4, "trainx": 4, "trait": [0, 26], "trajectori": 4, "transfer": [9, 26], "transform": [0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 19, 20, 26, 27, 28], "transit": [6, 12], "translat": [1, 4, 6, 10, 26, 27], "transpos": [1, 5, 11, 20, 27, 28], "travers": [0, 5], "treat": [0, 1, 3, 6, 12, 13, 18, 23, 26, 27, 28], "tree": [0, 1, 19, 26], "tree_clf": [9, 10], "tree_clf_": 9, "tree_clf_sr": 9, "tree_reg": 9, "tree_reg1": 9, "tree_reg2": 9, "trend": 23, "treue": 7, "trevor": [21, 25], "tri": [2, 3, 4, 9, 13, 16], "triain": 0, "trial": [0, 2, 4, 6, 13, 23, 26, 28], "triangl": [13, 28], "triangular": 20, "trick": [3, 4, 8, 11, 13, 23], "trickier": 23, "tridiagon": 20, "trillion": 19, "trivial": [0, 1, 5, 11, 23, 26, 28], "troubl": [0, 8, 12, 15, 27], "truck": 3, "true": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 23, 26, 27, 28], "true_beta": 27, "true_fun": 6, "true_theta": 6, "truli": 26, "try": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 18, 19, 20, 21, 23, 26, 27, 28], "tucker": 8, "tuesdai": [24, 26], "tumor": [7, 9], "tumour": 7, "tunabl": 1, "tune": [4, 9, 13, 20, 26], "turn": [0, 1, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 23, 26, 27, 28], "tutori": [1, 4], "tv": 2, "tveito": 2, "tweak": [1, 4, 10, 23], "twice": [13, 28], "twist": 11, "two": [0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 17, 20, 21, 22, 23, 25, 26, 27, 28], "tx": [13, 28], "tx_1": [13, 28], "txt": [4, 15], "ty": [13, 28], "type": [0, 1, 3, 6, 8, 10, 13, 20, 23, 27, 28], "typic": [0, 1, 2, 3, 4, 5, 7, 9, 10, 12, 13, 15, 16, 23, 26, 27, 28], "typo": 21, "u": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 21, 23, 25, 26, 27, 28], "u_": 20, "u_i": 12, "u_m": 10, "ua": [0, 26], "ubuntu": [0, 19, 21, 26], "uci": 21, "uio": [15, 21, 24, 25], "un": 14, "unabl": 15, "unari": [20, 26], "unbalanc": [6, 9], "unbias": [0, 5, 6, 26], "uncent": [6, 27], "uncertainti": [0, 5, 26], "uncertitud": 23, "unchang": [1, 3], "uncorrel": [10, 23], "undefin": [5, 27, 28], "under": [0, 1, 5, 6, 10, 13, 19, 21, 26, 27, 28], "underdetermin": [0, 26], "underfit": [1, 6], "underflowproblem": 5, "undergo": 5, "undergradu": [22, 24], "underli": [0, 1, 9, 13, 18, 23, 26], "underset": [4, 14], "understand": [0, 1, 3, 5, 6, 10, 13, 14, 15, 19, 26, 27, 28], "understood": [8, 13], "undesir": 8, "undetermin": [5, 8], "undo": 4, "unexpect": 6, "unexpected": 23, "unexplain": 18, "unfair": [6, 27], "unfortun": [1, 8, 9, 10], "unicode_liter": [8, 9], "uniform": [0, 1, 5, 6, 11, 13, 21, 23, 26, 28], "uniformli": [13, 23, 28], "unifrompdf": 23, "unimport": [13, 28], "union": [5, 6], "uniqu": [0, 2, 6, 13, 14, 20, 26], "unique_cluster_label": 14, "unit": [0, 1, 3, 4, 5, 10, 12, 18, 23, 26, 27, 28], "unitari": [5, 6, 20, 27, 28], "unitarili": [20, 26], "uniti": 23, "univari": 23, "univers": [0, 1, 2, 13, 19, 21, 22, 24, 26, 27, 28], "unix": 1, "unknow": [0, 20, 26], "unknown": [0, 1, 3, 4, 5, 6, 8, 10, 13, 20, 21, 26, 27, 28], "unknowwn": 12, "unlabel": 1, "unless": [0, 3, 6, 11, 13, 21, 26, 28], "unlik": [1, 3, 8, 13, 28], "unnecessarili": 9, "unord": 3, "unravel": 1, "unrol": [3, 11], "unseen": [0, 7, 9, 15], "unstabl": 1, "unsupervis": [0, 1, 4, 12, 19, 26], "unsymmetr": [20, 26], "until": [1, 2, 4, 9, 12, 13, 14, 28], "untouch": 0, "unusu": 12, "up": [1, 3, 4, 5, 6, 8, 10, 11, 13, 14, 16, 18, 19, 20, 21, 23, 24], "updat": [1, 2, 10, 12, 13, 14, 15, 18], "uploa": 26, "upload": [15, 19, 21, 25], "upon": [0, 1, 6, 7, 11, 20], "upper": [0, 8, 9, 16, 20, 27], "uppercas": [20, 26], "upsampl": 4, "upscal": 4, "url": [26, 27], "us": [4, 5, 6, 8, 9, 10, 11, 12, 14, 15, 17, 20, 23, 25], "usag": [0, 8, 19, 26, 27], "usd": [], "usd10000": [], "use_bia": 4, "usecol": [0, 26], "useless": 1, "user": [0, 1, 2, 4, 6, 7, 15, 19, 20, 21, 26, 27], "usernam": [15, 21], "usetex": 23, "usg": 6, "usr": 23, "usual": [0, 3, 4, 7, 12, 13, 14, 26], "ut": 5, "util": [1, 3, 4, 6, 7, 10, 14, 26], "ux": 20, "v": [2, 4, 5, 6, 11, 13, 15, 19, 27, 28], "v0": 23, "v1": 23, "v2": 23, "v_0": 11, "va": 1, "vahid": 26, "val": 13, "val_accuraci": 3, "val_loss": 4, "vale": 2, "valid": [0, 1, 4, 7, 9, 10, 13, 19, 23, 26, 27], "validation_data": 3, "validation_split": 4, "valu": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 26], "valuat": 9, "valy": 4, "van": [0, 21, 26, 27, 28], "vandenbergh": [8, 13, 28], "vandermond": [0, 26], "vanilla": [0, 6, 11, 14, 27], "vanish": [1, 4, 13, 23, 28], "var": [5, 6, 10, 11, 21, 23, 27], "var_x": 23, "varabl": 8, "varepsilon": [5, 6], "varepsilon_": [5, 6], "varepsilon_i": [5, 6], "vari": [0, 1, 3, 5, 6, 10, 26], "variabl": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 14, 20, 26, 27], "varianc": [0, 1, 5, 7, 9, 10, 11, 13, 14, 18, 19, 20, 23, 26, 27, 28], "variance_i": [5, 11, 27], "variance_x": [5, 11, 27], "variant": [0, 1, 6, 8, 12, 13, 26, 27, 28], "variat": [3, 4, 11, 26], "varieti": [0, 3, 12, 19, 21, 26], "variou": [1, 3, 5, 6, 7, 8, 9, 11, 12, 13, 16, 19, 20, 21, 23, 26, 27, 28], "varydimens": 4, "vastli": 3, "vaue": 1, "vault": 0, "vdot": [2, 13, 28], "ve": 21, "vec": 6, "vector": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 14, 17, 18, 19, 28], "vector_mean": 14, "ventur": [0, 8, 19, 26], "venv": 15, "verbos": [1, 3, 4], "veri": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 21, 23, 25, 26, 27, 28], "verifi": [3, 11, 20, 26], "versatil": [8, 26], "versicolor": [8, 9], "version": [0, 3, 10, 13, 14, 15, 19, 20, 21, 23, 26], "versu": 1, "vert": [0, 1, 5, 6, 7, 8, 9, 11, 13, 16, 17, 26, 27, 28], "vert_1": [5, 6, 27, 28], "vert_2": [5, 6, 11, 17, 27, 28], "via": [0, 5, 6, 7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 23, 24, 26, 27, 28], "vidal": 11, "video": [0, 1, 12, 19, 22, 24, 26, 27, 28], "view": [1, 3, 5, 6, 12, 13, 23, 25, 26, 28], "violat": 8, "virginica": 9, "viridi": [0, 1, 2, 3, 26], "virtual": 1, "viscos": 13, "viscou": 13, "visibl": 15, "vision": [0, 3], "visual": [0, 3, 11, 12, 18, 19, 26, 27], "visualis": 1, "visualstudio": [15, 16], "viz": [6, 8, 23], "vmap": 13, "vmax": [1, 6], "vmin": [1, 6], "voic": 3, "volum": [0, 3, 26], "vote": [10, 26], "voting_clf": 10, "votingclassifi": 10, "votingsimpl": 10, "vstack": [5, 11, 20, 23, 26, 27], "vt": [5, 27, 28], "w": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 20, 23, 26, 27, 28], "w1": 8, "w2": [8, 11], "w3": 8, "w_": [1, 12], "w_1": [8, 20], "w_1x_": 8, "w_1x_1": 8, "w_2": [8, 20], "w_2x_": 8, "w_2x_2": 8, "w_3": 20, "w_4": 20, "w_hidden": 2, "w_i": [1, 2, 10], "w_ix_i": 12, "w_j": 20, "w_m": 20, "w_output": 2, "w_px_": 8, "w_px_p": 8, "wa": [1, 3, 4, 5, 6, 7, 10, 11, 12, 14, 17, 20, 26, 27], "wai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 18, 20, 23, 26, 27, 28], "walk": 9, "walker": 23, "wang": [0, 26], "want": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 21, 23, 26, 27, 28], "warn": 4, "warrant": 6, "wast": 3, "watch": [19, 28], "wave": 3, "wavelet": 8, "we": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 27, 28], "weak": [9, 10, 14], "weather": [1, 12], "web": [19, 22, 24, 26], "webpag": 26, "websit": [6, 20, 21, 22, 26], "wedg": [8, 23], "wednesdai": [24, 26], "wee": 11, "week": [0, 5, 6, 7, 21, 22, 24], "weekli": [15, 16, 19, 21, 22, 24, 25, 26], "weight": [1, 2, 3, 6, 7, 9, 10, 12, 13, 18, 23], "weigth": 2, "welcom": [8, 15, 19], "well": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 19, 20, 21, 23, 25, 26, 27, 28], "went": 8, "were": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 14, 23, 26], "wessel": [0, 21, 26, 27, 28], "what": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 23], "whatev": 3, "when": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 23, 26, 27, 28], "whenev": [13, 15, 23], "where": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 26, 27, 28], "wherea": [6, 23], "wherefrom": 21, "wherein": [1, 12], "whether": [0, 3, 5, 7, 9, 21, 23, 26], "which": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 27, 28], "whichev": [1, 3], "while": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 23, 26, 27, 28], "white": 9, "whiteboard": [27, 28], "who": [0, 15], "whole": [1, 3, 4, 5, 9, 11, 13], "whose": [0, 6, 10, 23, 27], "whow": [11, 27], "why": [0, 1, 3, 6, 13, 15, 16, 17, 21, 27, 28], "wide": [0, 1, 3, 6, 7, 12, 19, 20, 21, 26], "widehat": 6, "width": [0, 3, 8, 9, 26], "wieringen": [0, 21, 26, 27, 28], "wiki": 21, "wikipedia": 21, "win": 10, "wind": 9, "wing": [24, 26], "winther": 2, "wiothout": 6, "wiscons": 7, "wisconsin": 10, "wisdom": [6, 27], "wise": [1, 5, 12, 13, 27, 28], "wish": [0, 2, 5, 7, 8, 11, 13, 14, 18, 20, 21, 26, 27, 28], "with_std": [0, 27], "wither": 6, "within": [0, 2, 3, 4, 7, 9, 12, 13, 14, 23, 25, 26, 28], "withinclust": 14, "without": [0, 1, 5, 6, 8, 9, 11, 12, 13, 15, 21, 26, 27, 28], "won": [0, 15, 26], "wonder": 8, "word": [0, 1, 3, 4, 5, 6, 7, 14, 23, 26, 27, 28], "work": [0, 1, 4, 6, 7, 8, 9, 13, 15, 16, 18, 19, 21, 22, 23, 24, 26, 27], "workshop": 26, "world": [0, 8, 16, 27], "worldwid": [0, 26], "worri": 15, "wors": [0, 1, 3, 4, 6, 26], "worth": 9, "worthi": 21, "would": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 18, 20, 21, 23, 26, 27, 28], "wrap": [6, 20, 26], "write": [0, 1, 2, 3, 5, 6, 7, 8, 12, 13, 15, 16, 20, 26, 27], "written": [0, 2, 3, 5, 11, 12, 13, 16, 19, 20, 21, 23, 26, 27, 28], "wrong": [1, 8, 15], "wrongli": 10, "wrote": [5, 11, 27], "wrt": [10, 13], "wth": [10, 13], "www": [19, 20, 21, 25, 26, 28], "wx_1": 8, "x": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 23, 26, 28], "x0": 8, "x1": [4, 8, 9, 10, 13], "x1_exampl": 8, "x1d": 8, "x2": [8, 9, 10, 13], "x2d": [8, 11], "x2d_train": 11, "x2dsl": 11, "x3": 8, "x_": [0, 2, 3, 5, 6, 8, 10, 11, 13, 14, 20, 23, 26, 27, 28], "x_0": [0, 5, 11, 18, 20, 26, 27], "x_1": [0, 2, 5, 6, 7, 8, 9, 10, 11, 13, 18, 20, 23, 26, 27, 28], "x_2": [0, 2, 5, 6, 7, 8, 9, 10, 11, 13, 20, 23, 26, 27, 28], "x_3": [8, 20, 23], "x_4": 20, "x_6": 18, "x_center": 11, "x_data": 1, "x_data_ful": 1, "x_hidden": 2, "x_i": [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 23, 26, 27, 28], "x_input": 2, "x_ix_": [0, 26], "x_iy_i": 8, "x_j": [0, 2, 8, 9, 12, 16, 23, 27], "x_jy_j": 8, "x_k": [12, 14, 20, 23, 27], "x_l": 23, "x_m": [6, 12, 20, 23], "x_mean": 18, "x_n": [0, 2, 3, 6, 8, 11, 12, 13, 20, 23, 26, 28], "x_new": [9, 10], "x_norm": 18, "x_offset": [6, 27], "x_output": 2, "x_p": [3, 7, 9], "x_poli": 9, "x_poly10": 9, "x_pred": 4, "x_prev": 2, "x_reduc": 11, "x_scale": 8, "x_small": 13, "x_std": 18, "x_test": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 27, 28], "x_test_": 17, "x_test_own": 6, "x_test_scal": [0, 6, 7, 9, 10, 11, 27], "x_tot": 4, "x_train": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 26, 27, 28], "x_train_": 17, "x_train_mean": [6, 27], "x_train_own": 6, "x_train_scal": [0, 6, 7, 9, 10, 11, 27], "x_val": 1, "xarrai": [19, 26], "xavier": 1, "xbnew": [13, 28], "xcode": [0, 19, 21, 26], "xdclassiffierconfus": 10, "xdclassiffierroc": 10, "xg_clf": 10, "xgb": 10, "xgbclassifi": 10, "xgboost": 9, "xgboot": 10, "xgbregressor": 10, "xgparam": 10, "xgtree": 10, "xi": [8, 13], "xi_": 8, "xi_1": 8, "xi_i": 8, "xk": 8, "xla": [13, 19, 26], "xlabel": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 23, 26, 27, 28], "xlim": [6, 10], "xm": 9, "xmesh": 13, "xnew": [0, 13, 26, 28], "xp": 23, "xpanda": [0, 27], "xpd": [5, 11, 27], "xplot": 0, "xscale": [0, 27], "xsr": 9, "xt_x": [13, 28], "xtest": 6, "xtick": [3, 6, 8, 9], "xtrain": 6, "xu": [0, 26], "xx": [0, 20, 26], "xy": [0, 6, 8, 20, 26], "xytext": 8, "xz": [20, 26], "y": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 23, 26, 27, 28], "y1": 4, "y2": 4, "y3": 4, "y_": [0, 1, 5, 6, 10, 11, 20, 26, 27], "y_0": [0, 5, 11, 20, 26, 27], "y_1": [0, 5, 8, 9, 11, 13, 20, 26, 27, 28], "y_1y_1": 8, "y_1y_1k": 8, "y_1y_2": 8, "y_1y_2k": 8, "y_1y_n": 8, "y_1y_nk": 8, "y_2": [0, 5, 8, 9, 11, 20, 26, 27], "y_2y_1": 8, "y_2y_1k": 8, "y_2y_2": 8, "y_2y_2k": 8, "y_3": [0, 9, 20], "y_4": 20, "y_center": 18, "y_data": [0, 1, 5, 6, 26, 27, 28], "y_data_ful": 1, "y_decis": 8, "y_fit": [0, 27], "y_i": [0, 1, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 26, 27, 28], "y_if_": 10, "y_ix_": [0, 26], "y_ix_i": [7, 8, 13, 27, 28], "y_iy_jk": 8, "y_j": [6, 8, 12, 21], "y_k": 12, "y_m": 20, "y_mean": 18, "y_model": [0, 4, 5, 6, 26, 27, 28], "y_n": [8, 13, 28], "y_ny_1": 8, "y_ny_1k": 8, "y_ny_2": 8, "y_ny_2k": 8, "y_ny_n": 8, "y_ny_nk": 8, "y_offset": [6, 17, 27], "y_plot": 9, "y_pred": [0, 1, 4, 6, 7, 8, 9, 10, 27], "y_pred1": 9, "y_pred2": 9, "y_pred_rf": 10, "y_pred_tre": 10, "y_proba": [7, 10], "y_scaler": [6, 27], "y_test": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 27, 28], "y_test_onehot": 1, "y_test_predict": [], "y_tot": 4, "y_train": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 26, 27, 28], "y_train_mean": [6, 27], "y_train_onehot": 1, "y_train_predict": [], "y_train_scal": [6, 27], "y_val": 1, "ye": [3, 6, 7], "year": [0, 19, 26], "yet": [0, 1, 6, 8, 11, 13, 26], "yi": 13, "yield": [0, 2, 5, 6, 8, 10, 12, 13, 14, 20, 23, 26, 28], "yk": 8, "ylabel": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 23, 26, 27, 28], "ylim": [3, 6], "ym": 9, "ymesh": 13, "yn": 0, "yo": [8, 9, 10], "yoshua": [1, 25], "you": [0, 1, 3, 4, 5, 6, 8, 9, 10, 11, 13, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28], "young": 0, "your": [1, 2, 4, 5, 6, 8, 11, 13, 15, 17, 19, 20, 26, 28], "your_model_object": 16, "yourself": [11, 13, 26, 28], "youtu": [27, 28], "youtub": [19, 28], "ypred": 6, "ypredict": [0, 13, 26, 27, 28], "ypredict2": [13, 28], "ypredictlasso": [5, 28], "ypredictol": [0, 5, 28], "ypredictown": [6, 27], "ypredictownridg": [6, 27, 28], "ypredictridg": [0, 5, 6, 27, 28], "ypredictskl": [6, 27], "ytest": 6, "ytick": [3, 6, 8, 9], "ytild": [0, 6, 26, 27], "ytildelasso": [5, 28], "ytildenp": [0, 26, 27], "ytildeol": [0, 5, 28], "ytildeownridg": [6, 27, 28], "ytilderidg": [5, 6, 27, 28], "ytrain": 6, "yuxi": 26, "yx": [20, 26], "yy": [20, 26], "yz": [20, 26], "z": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 20, 23, 26, 27], "z_": [1, 2, 12, 20, 26], "z_0": [20, 26], "z_1": [20, 26], "z_2": [20, 26], "z_c": 1, "z_h": 1, "z_hidden": 2, "z_i": [1, 12], "z_j": [1, 12], "z_k": [12, 27], "z_m": 1, "z_mod": 9, "z_o": 1, "z_output": 2, "zaman": 23, "zaxi": 6, "zero": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 21, 23, 26, 27, 28], "zeros_lik": 4, "zeroth": 27, "zfill": 4, "zip": [4, 6], "zm_h": [0, 26], "zn": [], "zone": [], "zoom": 26, "zx": [20, 26], "zy": [20, 26], "zz": [20, 26], "\u00f8yvind": [6, 27]}, "titles": ["3. Linear Regression", "14. Building a Feed Forward Neural Network", "15. Solving Differential Equations with Deep Learning", "16. Convolutional Neural Networks", "17. Recurrent neural networks: Overarching view", "4. Ridge and Lasso Regression", "5. Resampling Methods", "6. Logistic Regression", "8. Support Vector Machines, overarching aims", "9. Decision trees, overarching aims", "10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods", "11. Basic ideas of the Principal Component Analysis (PCA)", "13. Neural networks", "7. Optimization, the central part of any Machine Learning algortithm", "12. Clustering and Unsupervised Learning", "Exercises week 34", "Exercises week 35", "Exercises week 36", "Exercises week 37", "Applied Data Analysis and Machine Learning", "2. Linear Algebra, Handling of Arrays and more Python Features", "Project 1 on Machine Learning, deadline October 6 (midnight), 2025", "Course setting", "1. Elements of Probability Theory and Statistical Data Analysis", "Teachers and Grading", "Textbooks", "Week 34: Introduction to the course, Logistics and Practicalities", "Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression", "Week 36: Linear Regression and Gradient descent"], "titleterms": {"": [8, 10, 28], "1": [0, 15, 16, 17, 18, 21, 27], "1a": 18, "2": [0, 15, 16, 17, 18, 26, 27, 28], "2023": 24, "2025": 21, "2a": [], "2b": [], "3": [0, 15, 16, 17, 18, 27], "34": [15, 26], "35": [16, 27], "36": [17, 28], "37": 18, "3a": 18, "3b": 18, "4": [0, 15, 16, 17, 18, 27], "4a": 18, "4b": 18, "5": [0, 16, 18], "6": 21, "A": [0, 1, 4, 8, 9, 26], "And": [26, 27], "In": 24, "Ising": 6, "The": [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 15, 19, 26, 27, 28], "To": 26, "With": [4, 28], "about": [26, 27, 28], "abov": 28, "activ": [1, 12], "ad": [0, 6, 21, 26, 27], "adaboost": 10, "adagrad": 13, "adam": 13, "adapt": 10, "adjust": 1, "advanc": 21, "adversari": 4, "again": [3, 9], "ai": [21, 26], "aim": [8, 9, 26], "aka": 26, "algebra": [20, 26], "algorithm": [9, 10, 11, 12, 26, 27, 28], "algortithm": [13, 28], "all": 8, "an": [0, 4, 10, 15, 26], "analys": [5, 27, 28], "analysi": [0, 5, 6, 11, 19, 21, 23, 26, 27, 28], "analyt": [0, 16, 18], "ani": [13, 28], "anoth": [9, 28], "appli": 19, "approach": [0, 8, 14, 26], "approxim": 12, "architectur": 1, "arrai": [20, 26], "assist": 24, "autocorrel": 23, "autograd": [2, 13], "automat": 13, "b": 21, "back": [1, 11, 12, 27, 28], "background": [19, 21], "bag": 10, "base": 13, "basic": [0, 5, 7, 9, 10, 11, 20, 27, 28], "batch": 1, "bay": 5, "befor": 11, "better": 8, "bia": [6, 21], "binari": 1, "bind": 26, "bird": 10, "boldsymbol": [18, 27], "boost": 10, "bootstrap": [6, 10], "boston": [], "breast": 1, "brief": 26, "bring": 12, "build": [1, 3, 9], "c": [21, 26], "calcul": [18, 27, 28], "can": 26, "cancer": [1, 7, 9, 11], "cart": 9, "case": [8, 10, 23, 27, 28], "central": [13, 19, 23, 28], "chain": 12, "chang": 10, "channel": 26, "chi": [0, 26], "choic": 17, "choos": 1, "cifar01": 3, "classic": 11, "classif": [1, 9, 10], "classifi": 8, "clip": 1, "cluster": 14, "cnn": 3, "code": [1, 2, 5, 9, 11, 12, 13, 14, 15, 16, 21, 26, 27, 28], "collect": [1, 3], "commun": 26, "compar": [2, 10, 16], "comparison": 28, "complet": 27, "complex": [0, 6, 21, 27], "complic": 6, "compon": 11, "comput": 9, "computerlab": 26, "con": 9, "concept": 23, "condit": 28, "conjug": 13, "contn": 26, "convex": [8, 13, 28], "convolut": [3, 12], "correl": [11, 27], "correspond": [], "cost": [1, 10, 27, 28], "cours": [19, 22, 25, 26], "covari": [5, 11, 23, 27], "cover": 26, "creat": 16, "cross": [6, 21], "cython": 26, "d": 21, "data": [0, 1, 3, 6, 7, 9, 11, 15, 17, 18, 19, 23, 26, 27], "dataset": [1, 3, 18], "david": 26, "deadlin": [21, 26], "deadllin": 24, "decai": 2, "decis": [9, 10], "decomposit": [5, 11, 20, 27, 28], "deeep": [], "deep": [1, 2, 26], "defin": [1, 26], "degre": [0, 17, 27], "deliver": [15, 16], "deliveri": 21, "dens": 0, "deriv": [5, 12, 16, 17, 27, 28], "descent": [2, 10, 13, 18, 21, 28], "design": 27, "detail": [3, 26], "develop": 1, "diagon": 11, "differ": 8, "differenti": [2, 13], "diffus": 2, "dimension": [2, 3, 8, 18], "disadvantag": 9, "discret": 23, "discrimin": 26, "distribut": [5, 23], "do": 1, "doe": [27, 28], "domain": 23, "down": 1, "dropout": 1, "e": 21, "economi": [27, 28], "electron": 21, "element": [0, 23, 26], "elimin": 20, "energi": 26, "ensembl": 10, "entropi": 9, "environ": [0, 15], "equat": [0, 2, 12, 27, 28], "error": [0, 10, 26, 27, 28], "essenti": 26, "etc": 26, "euler": 2, "evalu": 1, "exampl": [1, 2, 3, 4, 6, 7, 8, 9, 10, 26, 27, 28], "exercis": [0, 6, 15, 16, 17, 18, 27], "expect": 23, "experi": 23, "explor": 0, "exponenti": 2, "express": [16, 17, 27], "extend": 28, "extrapol": 4, "extrem": [10, 26], "ey": 10, "f": 21, "fall": 24, "famili": [1, 26], "famou": 20, "fantast": [27, 28], "featur": [9, 16, 20, 27], "feed": [1, 12], "final": [12, 27], "find": [16, 18], "fine": 1, "first": [4, 12, 26, 28], "fit": [0, 10, 15, 16, 26, 28], "fix": [27, 28], "forc": 3, "forest": 10, "form": 18, "format": [21, 26], "formula": 18, "forward": [1, 2, 12], "foster": 26, "fourier": 3, "frank": 6, "freedom": [0, 17, 27], "frequent": 27, "frequentist": [0, 26], "from": [5, 10, 12, 26, 27, 28], "full": 2, "function": [0, 1, 6, 7, 8, 10, 11, 12, 13, 21, 23, 26, 27, 28], "further": [3, 5, 27, 28], "g": 21, "gan": 4, "gaussian": 20, "gd": 13, "gener": [4, 9, 26], "geometr": [11, 28], "gini": 9, "github": 15, "goal": [15, 16, 17, 18], "good": [0, 26], "grade": [24, 26], "gradient": [1, 2, 10, 13, 18, 21, 28], "growth": 2, "h": 21, "ha": 19, "handl": [20, 26], "hessian": [27, 28], "hidden": 2, "hous": [], "how": 16, "hyperparamet": [1, 17], "hyperplan": 8, "i": [0, 1, 26], "id3": 9, "idea": 11, "ideal": 28, "ii": 26, "illustr": 28, "implement": [1, 16, 17, 18], "implic": [5, 27, 28], "import": [5, 20, 26, 27, 28], "improv": 1, "includ": [13, 21], "increment": 11, "index": 9, "inform": 24, "input": 2, "instal": [19, 21, 26], "instructor": 24, "interpret": [5, 11, 26, 27, 28], "introduc": [11, 13, 27], "introduct": [0, 6, 19, 20, 21, 26], "invers": [5, 20], "invert": [27, 28], "iter": 10, "its": 27, "jacobian": 27, "jax": 13, "julia": 26, "jungl": 10, "kera": [1, 3], "kernel": [8, 11], "lab": 28, "lagrangian": 8, "lasso": [5, 6, 21, 27, 28], "last": 27, "later": [5, 27, 28], "layer": [1, 2, 3, 12], "learn": [0, 1, 2, 11, 13, 14, 15, 16, 17, 18, 19, 21, 26, 27, 28], "least": [5, 6, 16, 21, 26, 27, 28], "lectur": [26, 28], "level": 10, "librari": [19, 26], "likelihood": 7, "limit": [1, 13, 23, 28], "linear": [0, 8, 13, 15, 20, 26, 27, 28], "link": [5, 11, 25, 27], "literatur": 21, "logist": [7, 26], "loss": [27, 28], "lu": 20, "machin": [0, 8, 13, 19, 21, 26, 28], "main": [23, 26], "make": [0, 9, 10, 27], "mani": [10, 12], "mass": 26, "materi": [21, 26, 27, 28], "math": [5, 27, 28], "mathemat": [3, 5, 8, 27, 28], "matric": [5, 20, 26], "matrix": [1, 5, 11, 12, 16, 20, 26, 27, 28], "matter": 0, "max": 27, "mean": [0, 27, 28], "meet": [5, 10, 23, 26, 27], "mercer": 8, "method": [6, 9, 10, 13, 21, 26, 28], "midnight": 21, "min": 27, "minim": 26, "ml": 26, "mlp": 12, "mnist": [3, 4], "model": [0, 1, 4, 6, 12, 15, 17, 26], "momentum": [13, 21], "mondai": 28, "moon": [8, 9], "more": [3, 6, 20, 21, 26, 27, 28], "multilay": 12, "multipl": [1, 3, 17], "multipli": 8, "need": [21, 26], "network": [1, 2, 3, 4, 7, 12, 26], "neural": [1, 2, 3, 4, 7, 12, 26], "new": [4, 18], "newton": 28, "non": 8, "normal": [0, 1], "notat": 12, "note": [21, 27, 28], "now": [1, 9, 13, 28], "nuclear": [0, 26], "numba": 26, "number": [0, 2, 23, 27], "numer": [2, 21, 23], "numpi": [20, 26], "object": 3, "obtain": 11, "octob": 21, "od": 2, "off": [6, 21], "ol": [5, 6, 15, 16, 18, 21, 28], "one": [2, 12, 18, 28], "oper": 20, "optim": [1, 8, 13, 18, 19, 26, 27, 28], "order": [13, 18], "ordinari": [5, 6, 16, 21, 26, 27, 28], "organ": [0, 26], "oslo": 25, "other": [4, 9, 11, 12, 20, 21, 26], "our": [0, 4, 5, 11, 13, 21, 26, 27, 28], "outcom": [19, 26], "output": 2, "overarch": [0, 4, 8, 9, 26, 27], "overview": [10, 26], "own": [0, 10, 11, 21, 26, 27], "packag": [20, 26], "panda": [26, 27], "paramet": [26, 27], "paramt": 18, "part": [13, 19, 21, 28], "partial": 2, "pass": 1, "pca": 11, "pdf": 23, "perceptron": 12, "perform": [1, 9], "period": 3, "perspect": 1, "plan": [27, 28], "plethora": 26, "point": 4, "poisson": 2, "polynomi": [3, 16, 18, 28], "popul": 2, "popular": 26, "practic": [13, 24, 26], "pre": [1, 3], "preambl": 21, "predict": 4, "preprocess": 27, "prerequisit": [3, 19, 26], "princip": 11, "principl": 3, "pro": 9, "probabl": [5, 23], "problem": [1, 2, 13, 26, 27, 28], "procedur": [9, 26], "process": [1, 3], "program": [2, 13, 21, 28], "project": [6, 21, 24, 26], "prop": 13, "propag": [1, 12], "properti": [5, 23, 27, 28], "python": [0, 9, 15, 19, 20, 26], "quick": 8, "r": 26, "random": [10, 11, 23], "raphson": 28, "rate": 21, "read": [9, 26, 27], "real": [6, 26], "recommend": [26, 27], "recurr": [4, 12], "reduc": [0, 27], "reduct": 3, "refer": 21, "reformul": 2, "regress": [0, 5, 6, 7, 9, 10, 13, 15, 17, 18, 21, 26, 27, 28], "regular": 1, "relat": [], "relev": [25, 27], "relu": 1, "remark": 3, "remind": [6, 8, 26, 27, 28], "replac": 13, "report": 21, "repositori": 15, "requir": [2, 19], "resampl": [6, 21], "rescal": [6, 27], "residu": [27, 28], "resourc": 2, "result": [27, 28], "revisit": [13, 28], "rewrit": [26, 27], "ridg": [0, 5, 6, 17, 18, 21, 27, 28], "rm": 13, "rule": 12, "rung": 21, "same": 13, "sampl": 11, "scale": [17, 18, 27], "schedul": 26, "schemat": 9, "scheme": 2, "scienc": 26, "scikit": [0, 1, 11, 26, 27, 28], "second": [13, 18], "semest": 24, "sensit": 28, "septemb": 28, "session": 28, "set": [0, 2, 3, 9, 12, 15, 22, 26, 27, 28], "setup": 15, "sgd": 13, "should": 1, "similar": 13, "simpl": [0, 4, 9, 13, 18, 26, 27, 28], "simplest": 18, "singl": 10, "singular": [5, 11, 27, 28], "size": [27, 28], "sklearn": 16, "soft": 8, "softmax": 1, "softwar": [21, 26], "solv": [2, 28], "solver": 13, "some": [13, 20, 27, 28], "specifi": 2, "split": [0, 15, 27], "squar": [0, 5, 6, 10, 16, 21, 26, 27, 28], "standard": [13, 27], "state": 0, "statist": [5, 6, 19, 23, 26], "steepest": [10, 13, 28], "stochast": [13, 21, 23], "strongli": 26, "suggest": 26, "summari": [24, 26], "superposit": 3, "supervis": 1, "support": 8, "svd": [5, 27, 28], "synthet": 18, "systemat": 3, "t": 27, "take": 16, "taken": 26, "teach": 24, "teacher": [24, 26], "technic": 27, "techniqu": [6, 11, 21], "technologi": 19, "tensorflow": [1, 3], "tent": [24, 26], "test": [0, 1, 15, 17, 27], "text": 26, "textbook": [25, 26], "than": 28, "theorem": [5, 8, 11, 12, 23], "theori": 23, "theta": 18, "thi": 26, "tip": 13, "togeth": 12, "tool": [21, 26], "top": 1, "topic": 26, "toward": 11, "trade": [6, 21], "tradeoff": 6, "train": [0, 1, 4, 15, 26, 27], "transform": 3, "tree": [9, 10], "tuesdai": 28, "tune": 1, "two": [3, 8, 19], "type": [2, 4, 12, 26], "uio": 26, "univers": [12, 25], "unsupervis": 14, "up": [0, 2, 9, 12, 15, 26, 27, 28], "updat": 21, "us": [0, 1, 2, 3, 7, 13, 16, 18, 19, 21, 26, 27, 28], "v": 3, "valid": [6, 21], "valu": [5, 11, 23, 27, 28], "variabl": [23, 28], "varianc": [6, 21], "variou": 0, "vector": [8, 12, 16, 20, 26, 27], "versu": 26, "view": [0, 4, 10, 27], "virtual": 15, "visual": [1, 9], "wai": [9, 21], "wave": 2, "we": 26, "wednesdai": 28, "week": [15, 16, 17, 18, 26, 27, 28], "weekli": [], "what": [0, 26, 27, 28], "which": 1, "why": 26, "wisconsin": 7, "write": [4, 11, 21, 28], "x": 27, "xgboost": 10, "yet": 28, "your": [0, 10, 16, 18, 21, 27]}}) \ No newline at end of file +Search.setIndex({"alltitles": {"1a)": [[18, "a"]], "3a)": [[18, "id1"]], "3b)": [[18, "b"]], "4a)": [[18, "id2"]], "4b)": [[18, "id3"]], "A Classification Tree": [[9, "a-classification-tree"]], "A Frequentist approach to data analysis": [[0, "a-frequentist-approach-to-data-analysis"], [27, "a-frequentist-approach-to-data-analysis"]], "A better approach": [[8, "a-better-approach"]], "A first summary": [[27, "a-first-summary"]], "A quick Reminder on Lagrangian Multipliers": [[8, "a-quick-reminder-on-lagrangian-multipliers"]], "A simple example": [[4, "a-simple-example"]], "A soft classifier": [[8, "a-soft-classifier"]], "A top-down perspective on Neural networks": [[1, "a-top-down-perspective-on-neural-networks"]], "ADAM algorithm, taken from Goodfellow et al": [[30, "adam-algorithm-taken-from-goodfellow-et-al"]], "ADAM optimizer": [[13, "adam-optimizer"], [30, "id2"]], "Accuracy": [[30, "accuracy"]], "Activation functions": [[12, "activation-functions"]], "AdaGrad Properties": [[30, "adagrad-properties"]], "AdaGrad Update Rule Derivation": [[30, "adagrad-update-rule-derivation"]], "AdaGrad algorithm, taken from Goodfellow et al": [[30, "adagrad-algorithm-taken-from-goodfellow-et-al"]], "Adam Optimizer": [[30, "adam-optimizer"]], "Adaptive boosting: AdaBoost, Basic Algorithm": [[10, "adaptive-boosting-adaboost-basic-algorithm"]], "Adding error analysis and training set up": [[27, "adding-error-analysis-and-training-set-up"], [28, "adding-error-analysis-and-training-set-up"]], "Adjust hyperparameters": [[1, "adjust-hyperparameters"]], "Algorithms and codes for Adagrad, RMSprop and Adam": [[30, "algorithms-and-codes-for-adagrad-rmsprop-and-adam"]], "Algorithms for Setting up Decision Trees": [[9, "algorithms-for-setting-up-decision-trees"]], "An Overview of Ensemble Methods": [[10, "an-overview-of-ensemble-methods"]], "An extrapolation example": [[4, "an-extrapolation-example"]], "An optimization/minimization problem": [[27, "an-optimization-minimization-problem"]], "And finally \\boldsymbol{X}\\boldsymbol{X}^T": [[28, "and-finally-boldsymbol-x-boldsymbol-x-t"]], "And finally ADAM": [[30, "and-finally-adam"]], "And what about using neural networks?": [[27, "and-what-about-using-neural-networks"]], "Another Example, now with a polynomial fit": [[29, "another-example-now-with-a-polynomial-fit"]], "Another example, the moons again": [[9, "another-example-the-moons-again"]], "Applied Data Analysis and Machine Learning": [[20, null]], "Autocorrelation function": [[24, "autocorrelation-function"]], "Automatic differentiation": [[13, "automatic-differentiation"]], "Back to Ridge and LASSO Regression": [[28, "back-to-ridge-and-lasso-regression"], [29, "back-to-ridge-and-lasso-regression"]], "Back to the Cancer Data": [[11, "back-to-the-cancer-data"]], "Background literature": [[22, "background-literature"]], "Bagging": [[10, "bagging"]], "Bagging Examples": [[10, "bagging-examples"]], "Basic Matrix Features": [[21, "basic-matrix-features"]], "Basic ideas of the Principal Component Analysis (PCA)": [[11, null]], "Basic math of the SVD": [[5, "basic-math-of-the-svd"], [28, "basic-math-of-the-svd"], [29, "basic-math-of-the-svd"]], "Basics": [[7, "basics"]], "Basics of a tree": [[9, "basics-of-a-tree"]], "Batch Normalization": [[1, "batch-normalization"]], "Batches and mini-batches": [[30, "batches-and-mini-batches"]], "Bayes\u2019 Theorem and Ridge and Lasso Regression": [[5, "bayes-theorem-and-ridge-and-lasso-regression"]], "Boosting, a Bird\u2019s Eye View": [[10, "boosting-a-bird-s-eye-view"]], "Bootstrap": [[6, "bootstrap"]], "Bringing it together, first back propagation equation": [[12, "bringing-it-together-first-back-propagation-equation"]], "Building a Feed Forward Neural Network": [[1, null]], "Building a tree, regression": [[9, "building-a-tree-regression"]], "Building neural networks in Tensorflow and Keras": [[1, "building-neural-networks-in-tensorflow-and-keras"]], "But none of these can compete with Newton\u2019s method": [[30, "but-none-of-these-can-compete-with-newton-s-method"]], "CNNs in more detail, building convolutional neural networks in Tensorflow and Keras": [[3, "cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras"]], "Cancer Data again now with Decision Trees and other Methods": [[9, "cancer-data-again-now-with-decision-trees-and-other-methods"]], "Challenge: Choosing a Fixed Learning Rate": [[30, "challenge-choosing-a-fixed-learning-rate"]], "Choose cost function and optimizer": [[1, "choose-cost-function-and-optimizer"]], "Classical PCA Theorem": [[11, "classical-pca-theorem"]], "Clustering and Unsupervised Learning": [[14, null]], "Code for SVD and Inversion of Matrices": [[5, "code-for-svd-and-inversion-of-matrices"]], "Code with a Number of Minibatches which varies": [[30, "code-with-a-number-of-minibatches-which-varies"]], "Codes and Approaches": [[14, "codes-and-approaches"]], "Codes for the SVD": [[5, "codes-for-the-svd"], [28, "codes-for-the-svd"], [29, "codes-for-the-svd"]], "Coding Setup and Linear Regression": [[15, "coding-setup-and-linear-regression"]], "Collect and pre-process data": [[1, "collect-and-pre-process-data"]], "Communication channels": [[27, "communication-channels"]], "Compare Bagging on Trees with Random Forests": [[10, "compare-bagging-on-trees-with-random-forests"]], "Comparing with a numerical scheme": [[2, "comparing-with-a-numerical-scheme"]], "Comparison with OLS": [[29, "comparison-with-ols"]], "Computation of gradients": [[30, "computation-of-gradients"]], "Computing the Gini index": [[9, "computing-the-gini-index"]], "Conditions on convex functions": [[29, "conditions-on-convex-functions"]], "Conjugate gradient method": [[13, "conjugate-gradient-method"]], "Convergence rates": [[30, "convergence-rates"]], "Convex function": [[29, "convex-function"]], "Convex functions": [[13, "convex-functions"], [29, "convex-functions"]], "Convolution Examples: Polynomial multiplication": [[3, "convolution-examples-polynomial-multiplication"]], "Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)": [[3, "convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms"]], "Convolutional Neural Network": [[12, "convolutional-neural-network"]], "Convolutional Neural Networks": [[3, null]], "Correlation Function and Design/Feature Matrix": [[28, "correlation-function-and-design-feature-matrix"]], "Correlation Matrix": [[11, "correlation-matrix"], [28, "correlation-matrix"]], "Correlation Matrix with Pandas": [[28, "correlation-matrix-with-pandas"]], "Course Format": [[27, "course-format"]], "Course setting": [[23, null]], "Covariance Matrix Examples": [[28, "covariance-matrix-examples"]], "Covariance and Correlation Matrix": [[28, "covariance-and-correlation-matrix"]], "Cross-validation": [[6, "cross-validation"]], "Deadlines for projects (tentative)": [[27, "deadlines-for-projects-tentative"]], "Decision trees, overarching aims": [[9, null]], "Deep learning methods": [[27, "deep-learning-methods"]], "Define model and architecture": [[1, "define-model-and-architecture"]], "Defining the cost function": [[1, "defining-the-cost-function"]], "Deliverables": [[15, "deliverables"], [16, "deliverables"]], "Derivation of the AdaGrad Algorithm": [[30, "derivation-of-the-adagrad-algorithm"]], "Derivatives and the chain rule": [[12, "derivatives-and-the-chain-rule"]], "Derivatives, example 1": [[28, "derivatives-example-1"]], "Deriving OLS from a probability distribution": [[5, "deriving-ols-from-a-probability-distribution"]], "Deriving and Implementing Ordinary Least Squares": [[16, "deriving-and-implementing-ordinary-least-squares"]], "Deriving and Implementing Ridge Regression": [[17, "deriving-and-implementing-ridge-regression"]], "Deriving the Lasso Regression Equations": [[28, "deriving-the-lasso-regression-equations"], [29, "deriving-the-lasso-regression-equations"], [29, "id6"]], "Deriving the Ridge Regression Equations": [[28, "deriving-the-ridge-regression-equations"], [29, "deriving-the-ridge-regression-equations"], [29, "id3"]], "Deriving the back propagation code for a multilayer perceptron model": [[12, "deriving-the-back-propagation-code-for-a-multilayer-perceptron-model"]], "Developing a code for doing neural networks with back propagation": [[1, "developing-a-code-for-doing-neural-networks-with-back-propagation"]], "Diagonalize the sample covariance matrix to obtain the principal components": [[11, "diagonalize-the-sample-covariance-matrix-to-obtain-the-principal-components"]], "Different kernels and Mercer\u2019s theorem": [[8, "different-kernels-and-mercer-s-theorem"]], "Disadvantages": [[9, "disadvantages"]], "Discriminative Modeling": [[27, "discriminative-modeling"]], "Domains and probabilities": [[24, "domains-and-probabilities"]], "Dropout": [[1, "dropout"]], "Economy-size SVD": [[28, "economy-size-svd"], [29, "economy-size-svd"]], "Elements of Probability Theory and Statistical Data Analysis": [[24, null]], "Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods": [[10, null]], "Entropy and the ID3 algorithm": [[9, "entropy-and-the-id3-algorithm"]], "Essential elements of ML": [[27, "essential-elements-of-ml"]], "Evaluate model performance on test data": [[1, "evaluate-model-performance-on-test-data"]], "Example 2": [[28, "example-2"]], "Example 3": [[28, "example-3"]], "Example 4": [[28, "example-4"]], "Example Matrix": [[28, "example-matrix"], [29, "example-matrix"]], "Example of discriminative modeling, taken from Generative Deep Learning by David Foster": [[27, "example-of-discriminative-modeling-taken-from-generative-deep-learning-by-david-foster"]], "Example of generative modeling, taken from Generative Deep Learning by David Foster": [[27, "example-of-generative-modeling-taken-from-generative-deep-learning-by-david-foster"]], "Example of own Standard scaling": [[28, "example-of-own-standard-scaling"]], "Example relevant for the exercises": [[28, "example-relevant-for-the-exercises"]], "Example: Exponential decay": [[2, "example-exponential-decay"]], "Example: Population growth": [[2, "example-population-growth"]], "Example: The diffusion equation": [[2, "example-the-diffusion-equation"]], "Example: binary classification problem": [[1, "example-binary-classification-problem"]], "Examples": [[27, "examples"]], "Examples of likelihood functions used in logistic regression and neural networks": [[7, "examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks"]], "Exercise 1 - Choice of model and degrees of freedom": [[17, "exercise-1-choice-of-model-and-degrees-of-freedom"]], "Exercise 1 - Finding the derivative of Matrix-Vector expressions": [[16, "exercise-1-finding-the-derivative-of-matrix-vector-expressions"]], "Exercise 1 - Github Setup": [[15, "exercise-1-github-setup"]], "Exercise 1, scale your data": [[18, "exercise-1-scale-your-data"]], "Exercise 1: Expectation values for ordinary least squares expressions": [[19, "exercise-1-expectation-values-for-ordinary-least-squares-expressions"]], "Exercise 1: Setting up various Python environments": [[0, "exercise-1-setting-up-various-python-environments"]], "Exercise 2 - Deriving the expression for OLS": [[16, "exercise-2-deriving-the-expression-for-ols"]], "Exercise 2 - Deriving the expression for Ridge Regression": [[17, "exercise-2-deriving-the-expression-for-ridge-regression"]], "Exercise 2 - Setting up a Github repository": [[15, "exercise-2-setting-up-a-github-repository"]], "Exercise 2, calculate the gradients": [[18, "exercise-2-calculate-the-gradients"]], "Exercise 2: Expectation values for Ridge regression": [[19, "exercise-2-expectation-values-for-ridge-regression"]], "Exercise 2: making your own data and exploring scikit-learn": [[0, "exercise-2-making-your-own-data-and-exploring-scikit-learn"]], "Exercise 3 - Creating feature matrix and implementing OLS using the analytical expression": [[16, "exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression"]], "Exercise 3 - Fitting an OLS model to data": [[15, "exercise-3-fitting-an-ols-model-to-data"]], "Exercise 3 - Scaling data": [[17, "exercise-3-scaling-data"]], "Exercise 3 - Setting up a Python virtual environment": [[15, "exercise-3-setting-up-a-python-virtual-environment"]], "Exercise 3, using the analytical formulae for OLS and Ridge regression to find the optimal paramters \\boldsymbol{\\theta}": [[18, "exercise-3-using-the-analytical-formulae-for-ols-and-ridge-regression-to-find-the-optimal-paramters-boldsymbol-theta"]], "Exercise 3: Deriving the expression for the Bias-Variance Trade-off": [[19, "exercise-3-deriving-the-expression-for-the-bias-variance-trade-off"]], "Exercise 3: Normalizing our data": [[0, "exercise-3-normalizing-our-data"]], "Exercise 4 - Fitting a polynomial": [[16, "exercise-4-fitting-a-polynomial"]], "Exercise 4 - Implementing Ridge Regression": [[17, "exercise-4-implementing-ridge-regression"]], "Exercise 4 - Testing multiple hyperparameters": [[17, "exercise-4-testing-multiple-hyperparameters"]], "Exercise 4 - The train-test split": [[15, "exercise-4-the-train-test-split"]], "Exercise 4, Implementing the simplest form for gradient descent": [[18, "exercise-4-implementing-the-simplest-form-for-gradient-descent"]], "Exercise 4: Adding Ridge Regression": [[0, "exercise-4-adding-ridge-regression"]], "Exercise 4: Computing the Bias and Variance": [[19, "exercise-4-computing-the-bias-and-variance"]], "Exercise 5 - Comparing your code with sklearn": [[16, "exercise-5-comparing-your-code-with-sklearn"]], "Exercise 5, Ridge regression and a new Synthetic Dataset": [[18, "exercise-5-ridge-regression-and-a-new-synthetic-dataset"]], "Exercise 5: Analytical exercises": [[0, "exercise-5-analytical-exercises"]], "Exercise: Cross-validation as resampling techniques, adding more complexity": [[6, "exercise-cross-validation-as-resampling-techniques-adding-more-complexity"]], "Exercise: Analysis of real data": [[6, "exercise-analysis-of-real-data"]], "Exercise: Bias-variance trade-off and resampling techniques": [[6, "exercise-bias-variance-trade-off-and-resampling-techniques"]], "Exercise: Lasso Regression on the Franke function with resampling": [[6, "exercise-lasso-regression-on-the-franke-function-with-resampling"]], "Exercise: Ordinary Least Square (OLS) on the Franke function": [[6, "exercise-ordinary-least-square-ols-on-the-franke-function"]], "Exercise: Ridge Regression on the Franke function with resampling": [[6, "exercise-ridge-regression-on-the-franke-function-with-resampling"]], "Exercises": [[0, "exercises"]], "Exercises and Projects": [[6, "exercises-and-projects"]], "Exercises week 34": [[15, null]], "Exercises week 35": [[16, null]], "Exercises week 36": [[17, null]], "Exercises week 37": [[18, null], [19, null]], "Expectation values": [[24, "expectation-values"]], "Extending to more than one variable": [[29, "extending-to-more-than-one-variable"]], "Extremely useful tools, strongly recommended": [[27, "extremely-useful-tools-strongly-recommended"]], "Feed-forward neural networks": [[12, "feed-forward-neural-networks"]], "Feed-forward pass": [[1, "feed-forward-pass"]], "Final back propagating equation": [[12, "final-back-propagating-equation"]], "Fine-tuning neural network hyperparameters": [[1, "fine-tuning-neural-network-hyperparameters"]], "Fitting an Equation of State for Dense Nuclear Matter": [[0, "fitting-an-equation-of-state-for-dense-nuclear-matter"]], "Fixing the singularity": [[28, "fixing-the-singularity"], [29, "fixing-the-singularity"]], "Format for electronic delivery of report and programs": [[22, "format-for-electronic-delivery-of-report-and-programs"]], "Frequently used scaling functions": [[28, "frequently-used-scaling-functions"], [30, "frequently-used-scaling-functions"]], "From OLS to Ridge and Lasso": [[29, "from-ols-to-ridge-and-lasso"]], "From one to many layers, the universal approximation theorem": [[12, "from-one-to-many-layers-the-universal-approximation-theorem"]], "Functionality in Scikit-Learn": [[28, "functionality-in-scikit-learn"], [30, "functionality-in-scikit-learn"]], "Further Dimensionality Remarks": [[3, "further-dimensionality-remarks"]], "Further properties (important for our analyses later)": [[5, "further-properties-important-for-our-analyses-later"], [28, "further-properties-important-for-our-analyses-later"], [29, "further-properties-important-for-our-analyses-later"]], "Gaussian Elimination": [[21, "gaussian-elimination"]], "General Features": [[9, "general-features"]], "General linear models and linear algebra": [[27, "general-linear-models-and-linear-algebra"]], "Generalizing the fitting procedure as a linear algebra problem": [[27, "generalizing-the-fitting-procedure-as-a-linear-algebra-problem"], [27, "id1"]], "Generative Adversarial Networks": [[4, "generative-adversarial-networks"]], "Generative Models": [[4, "generative-models"]], "Generative Versus Discriminative Modeling": [[27, "generative-versus-discriminative-modeling"]], "Geometric Interpretation and link with Singular Value Decomposition": [[11, "geometric-interpretation-and-link-with-singular-value-decomposition"]], "Gradient Boosting, Classification Example": [[10, "gradient-boosting-classification-example"]], "Gradient Boosting, Examples of Regression": [[10, "gradient-boosting-examples-of-regression"]], "Gradient Clipping": [[1, "gradient-clipping"]], "Gradient Descent Example": [[29, "id1"], [30, "id1"]], "Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent": [[10, "gradient-boosting-basics-with-steepest-descent-functional-gradient-descent"]], "Gradient descent": [[2, "gradient-descent"]], "Gradient descent and Ridge": [[29, "gradient-descent-and-ridge"], [30, "gradient-descent-and-ridge"]], "Gradient descent and revisiting Ordinary Least Squares from last week": [[30, "gradient-descent-and-revisiting-ordinary-least-squares-from-last-week"]], "Gradient descent example": [[29, "gradient-descent-example"], [30, "gradient-descent-example"]], "Grading": [[25, "grading"], [25, "id2"], [27, "grading"]], "How to take derivatives of Matrix-Vector expressions": [[16, "how-to-take-derivatives-of-matrix-vector-expressions"]], "Hyperplanes and all that": [[8, "hyperplanes-and-all-that"]], "Important Matrix and vector handling packages": [[21, "important-matrix-and-vector-handling-packages"]], "Important technicalities: More on Rescaling data": [[28, "important-technicalities-more-on-rescaling-data"]], "Improving gradient descent with momentum": [[30, "improving-gradient-descent-with-momentum"]], "Improving performance": [[1, "improving-performance"]], "In summary": [[25, "in-summary"]], "Including Stochastic Gradient Descent with Autograd": [[13, "including-stochastic-gradient-descent-with-autograd"], [30, "including-stochastic-gradient-descent-with-autograd"]], "Incremental PCA": [[11, "incremental-pca"]], "Installing R, C++, cython or Julia": [[27, "installing-r-c-cython-or-julia"]], "Installing R, C++, cython, Numba etc": [[27, "installing-r-c-cython-numba-etc"]], "Instructor information": [[25, "instructor-information"]], "Interpretations and optimizing our parameters": [[27, "interpretations-and-optimizing-our-parameters"], [27, "id2"], [27, "id3"], [28, "interpretations-and-optimizing-our-parameters"], [28, "id1"], [28, "id2"]], "Interpreting the Ridge results": [[28, "interpreting-the-ridge-results"], [29, "interpreting-the-ridge-results"], [29, "id4"]], "Introducing JAX": [[13, "introducing-jax"]], "Introducing the Covariance and Correlation functions": [[11, "introducing-the-covariance-and-correlation-functions"], [28, "introducing-the-covariance-and-correlation-functions"]], "Introduction": [[0, "introduction"], [6, "introduction"], [20, "introduction"], [21, "introduction"]], "Introduction to numerical projects": [[22, "introduction-to-numerical-projects"]], "Iterative Fitting, Classification and AdaBoost": [[10, "iterative-fitting-classification-and-adaboost"]], "Iterative Fitting, Regression and Squared-error Cost Function": [[10, "iterative-fitting-regression-and-squared-error-cost-function"]], "Kernel PCA": [[11, "kernel-pca"]], "Kernels and non-linearity": [[8, "kernels-and-non-linearity"]], "LU Decomposition, the inverse of a matrix": [[21, "lu-decomposition-the-inverse-of-a-matrix"]], "Lasso Regression": [[29, "lasso-regression"]], "Lasso case": [[29, "lasso-case"]], "Layers": [[1, "layers"]], "Layers used to build CNNs": [[3, "layers-used-to-build-cnns"]], "Learning goals": [[15, "learning-goals"], [16, "learning-goals"], [17, "learning-goals"], [18, "learning-goals"], [19, "learning-goals"]], "Learning outcomes": [[20, "learning-outcomes"], [27, "learning-outcomes"]], "Lectures and ComputerLab": [[27, "lectures-and-computerlab"]], "Limitations of supervised learning with deep networks": [[1, "limitations-of-supervised-learning-with-deep-networks"]], "Linear Algebra, Handling of Arrays and more Python Features": [[21, null]], "Linear Regression": [[0, null]], "Linear Regression Problems": [[28, "linear-regression-problems"], [29, "linear-regression-problems"]], "Linear Regression and the SVD": [[29, "linear-regression-and-the-svd"]], "Linear Regression, basic elements": [[0, "linear-regression-basic-elements"]], "Linking Bayes\u2019 Theorem with Ridge and Lasso Regression": [[5, "linking-bayes-theorem-with-ridge-and-lasso-regression"]], "Linking the regression analysis with a statistical interpretation": [[5, "linking-the-regression-analysis-with-a-statistical-interpretation"]], "Linking with the SVD": [[5, "linking-with-the-svd"], [28, "linking-with-the-svd"]], "Links to relevant courses at the University of Oslo": [[26, "links-to-relevant-courses-at-the-university-of-oslo"]], "Logistic Regression": [[7, null], [7, "id1"]], "MNIST and GANs": [[4, "mnist-and-gans"]], "Machine Learning": [[27, "machine-learning"]], "Machine learning": [[20, "machine-learning"]], "Main textbooks": [[27, "main-textbooks"]], "Making a tree": [[9, "making-a-tree"]], "Making your own Bootstrap: Changing the Level of the Decision Tree": [[10, "making-your-own-bootstrap-changing-the-level-of-the-decision-tree"]], "Making your own test-train splitting": [[28, "making-your-own-test-train-splitting"]], "Material for exercises week 35": [[28, "material-for-exercises-week-35"]], "Material for lab sessions sessions Tuesday and Wednesday": [[29, "material-for-lab-sessions-sessions-tuesday-and-wednesday"]], "Material for lecture Monday September 2": [[29, "material-for-lecture-monday-september-2"]], "Material for lecture Monday September 8": [[30, "material-for-lecture-monday-september-8"]], "Material for the lab sessions": [[30, "material-for-the-lab-sessions"]], "Mathematical Interpretation of Ordinary Least Squares": [[5, "mathematical-interpretation-of-ordinary-least-squares"], [28, "mathematical-interpretation-of-ordinary-least-squares"], [29, "mathematical-interpretation-of-ordinary-least-squares"]], "Mathematical optimization of convex functions": [[8, "mathematical-optimization-of-convex-functions"]], "Mathematics of CNNs": [[3, "mathematics-of-cnns"]], "Mathematics of the SVD and implications": [[5, "mathematics-of-the-svd-and-implications"], [28, "mathematics-of-the-svd-and-implications"], [29, "mathematics-of-the-svd-and-implications"]], "Matrices in Python": [[27, "matrices-in-python"]], "Matrix multiplication": [[1, "matrix-multiplication"]], "Matrix-vector notation and activation": [[12, "matrix-vector-notation-and-activation"]], "Meet the covariance!": [[24, "meet-the-covariance"]], "Meet the Covariance Matrix": [[5, "meet-the-covariance-matrix"], [28, "meet-the-covariance-matrix"]], "Meet the Hessian Matrix": [[28, "meet-the-hessian-matrix"]], "Meet the Pandas": [[27, "meet-the-pandas"]], "Min-Max Scaling": [[28, "min-max-scaling"]], "Momentum based GD": [[13, "momentum-based-gd"], [30, "momentum-based-gd"]], "Momentum parameter": [[30, "momentum-parameter"]], "More complicated Example: The Ising model": [[6, "more-complicated-example-the-ising-model"]], "More interpretations": [[28, "more-interpretations"], [29, "more-interpretations"], [29, "id5"]], "More on Dimensionalities": [[3, "more-on-dimensionalities"]], "More on Rescaling data": [[6, "more-on-rescaling-data"]], "More on Steepest descent": [[29, "more-on-steepest-descent"]], "More on convex functions": [[29, "more-on-convex-functions"]], "More on momentum based approaches": [[30, "more-on-momentum-based-approaches"]], "More preprocessing": [[28, "more-preprocessing"], [30, "more-preprocessing"]], "Motivation for Adaptive Step Sizes": [[30, "motivation-for-adaptive-step-sizes"]], "Multilayer perceptrons": [[12, "multilayer-perceptrons"]], "Network requirements": [[2, "network-requirements"]], "Neural Networks vs CNNs": [[3, "neural-networks-vs-cnns"]], "Neural networks": [[12, null]], "Note about SVD Calculations": [[28, "note-about-svd-calculations"], [29, "note-about-svd-calculations"]], "Note on Scikit-Learn": [[29, "note-on-scikit-learn"]], "Numerical experiments and the covariance, central limit theorem": [[24, "numerical-experiments-and-the-covariance-central-limit-theorem"]], "Numpy and arrays": [[21, "numpy-and-arrays"], [27, "numpy-and-arrays"]], "Numpy examples and Important Matrix and vector handling packages": [[27, "numpy-examples-and-important-matrix-and-vector-handling-packages"]], "Optimization and gradient descent, the central part of any Machine Learning algortithm": [[29, "optimization-and-gradient-descent-the-central-part-of-any-machine-learning-algortithm"]], "Optimization, the central part of any Machine Learning algortithm": [[13, null]], "Optimizing our parameters": [[27, "optimizing-our-parameters"]], "Optimizing our parameters, more details": [[27, "optimizing-our-parameters-more-details"]], "Optimizing the cost function": [[1, "optimizing-the-cost-function"]], "Organizing our data": [[0, "organizing-our-data"], [27, "organizing-our-data"]], "Other Matrix and Vector Operations": [[21, "other-matrix-and-vector-operations"]], "Other Types of Recurrent Neural Networks": [[4, "other-types-of-recurrent-neural-networks"]], "Other courses on Data science and Machine Learning at UiO": [[27, "other-courses-on-data-science-and-machine-learning-at-uio"]], "Other courses on Data science and Machine Learning at UiO, contn": [[27, "other-courses-on-data-science-and-machine-learning-at-uio-contn"]], "Other popular texts": [[27, "other-popular-texts"]], "Other techniques": [[11, "other-techniques"]], "Other types of networks": [[12, "other-types-of-networks"]], "Other ways of visualizing the trees": [[9, "other-ways-of-visualizing-the-trees"]], "Our model for the nuclear binding energies": [[27, "our-model-for-the-nuclear-binding-energies"]], "Overview of first week": [[27, "overview-of-first-week"]], "Overview video on Stochastic Gradient Descent (SGD)": [[30, "overview-video-on-stochastic-gradient-descent-sgd"]], "Own code for Ordinary Least Squares": [[27, "own-code-for-ordinary-least-squares"], [28, "own-code-for-ordinary-least-squares"]], "PCA and scikit-learn": [[11, "pca-and-scikit-learn"]], "Pandas AI": [[27, "pandas-ai"]], "Part a : Ordinary Least Square (OLS) for the Runge function": [[22, "part-a-ordinary-least-square-ols-for-the-runge-function"]], "Part b: Adding Ridge regression for the Runge function": [[22, "part-b-adding-ridge-regression-for-the-runge-function"]], "Part c: Writing your own gradient descent code": [[22, "part-c-writing-your-own-gradient-descent-code"]], "Part d: Including momentum and more advanced ways to update the learning the rate": [[22, "part-d-including-momentum-and-more-advanced-ways-to-update-the-learning-the-rate"]], "Part e: Writing our own code for Lasso regression": [[22, "part-e-writing-our-own-code-for-lasso-regression"]], "Part f: Stochastic gradient descent": [[22, "part-f-stochastic-gradient-descent"]], "Part g: Bias-variance trade-off and resampling techniques": [[22, "part-g-bias-variance-trade-off-and-resampling-techniques"]], "Part h): Cross-validation as resampling techniques, adding more complexity": [[22, "part-h-cross-validation-as-resampling-techniques-adding-more-complexity"]], "Partial Differential Equations": [[2, "partial-differential-equations"]], "Plans for week 35": [[28, "plans-for-week-35"]], "Plans for week 36": [[29, "plans-for-week-36"]], "Plans for week 37, lecture Monday": [[30, "plans-for-week-37-lecture-monday"]], "Practical tips": [[13, "practical-tips"], [30, "practical-tips"]], "Practicalities": [[25, "practicalities"], [25, "id1"]], "Preamble: Note on writing reports, using reference material, AI and other tools": [[22, "preamble-note-on-writing-reports-using-reference-material-ai-and-other-tools"]], "Predicting New Points With A Trained Recurrent Neural Network": [[4, "predicting-new-points-with-a-trained-recurrent-neural-network"]], "Preprocessing our data": [[28, "preprocessing-our-data"]], "Prerequisites": [[27, "prerequisites"]], "Prerequisites and background": [[20, "prerequisites-and-background"]], "Prerequisites: Collect and pre-process data": [[3, "prerequisites-collect-and-pre-process-data"]], "Probability Distribution Functions": [[24, "probability-distribution-functions"]], "Program example for gradient descent with Ridge Regression": [[29, "program-example-for-gradient-descent-with-ridge-regression"], [30, "program-example-for-gradient-descent-with-ridge-regression"]], "Program for stochastic gradient": [[13, "program-for-stochastic-gradient"]], "Project 1 on Machine Learning, deadline October 6 (midnight), 2025": [[22, null]], "Properties of PDFs": [[24, "properties-of-pdfs"]], "Pros and cons": [[30, "pros-and-cons"]], "Pros and cons of trees, pros": [[9, "pros-and-cons-of-trees-pros"]], "Python installers": [[20, "python-installers"], [27, "python-installers"]], "RMS prop": [[13, "rms-prop"], [30, "rms-prop"]], "RMSProp algorithm, taken from Goodfellow et al": [[30, "rmsprop-algorithm-taken-from-goodfellow-et-al"]], "RMSProp: Adaptive Learning Rates": [[30, "rmsprop-adaptive-learning-rates"]], "RMSprop for adaptive learning rate with Stochastic Gradient Descent": [[30, "rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent"]], "Random Numbers": [[24, "random-numbers"]], "Random forests": [[10, "random-forests"]], "Randomized PCA": [[11, "randomized-pca"]], "Reading material": [[27, "reading-material"]], "Reading recommendations:": [[28, "reading-recommendations"]], "Reading suggestions week 34": [[27, "reading-suggestions-week-34"]], "Readings and Videos:": [[30, "readings-and-videos"]], "Recurrent neural networks": [[12, "recurrent-neural-networks"]], "Recurrent neural networks: Overarching view": [[4, null]], "Reducing the number of degrees of freedom, overarching view": [[0, "reducing-the-number-of-degrees-of-freedom-overarching-view"], [28, "reducing-the-number-of-degrees-of-freedom-overarching-view"]], "Reformulating the problem": [[2, "reformulating-the-problem"]], "Regression Case": [[10, "regression-case"]], "Regression analysis and resampling methods": [[22, "regression-analysis-and-resampling-methods"]], "Regression analysis, overarching aims": [[27, "regression-analysis-overarching-aims"]], "Regression analysis, overarching aims II": [[27, "regression-analysis-overarching-aims-ii"]], "Regularization": [[1, "regularization"]], "Reminder from last week": [[28, "reminder-from-last-week"]], "Reminder on Newton-Raphson\u2019s method": [[29, "reminder-on-newton-raphson-s-method"]], "Reminder on Statistics": [[6, "reminder-on-statistics"]], "Reminder on different scaling methods": [[30, "reminder-on-different-scaling-methods"]], "Replace or not": [[13, "replace-or-not"], [30, "replace-or-not"]], "Required Technologies": [[20, "required-technologies"]], "Resampling Methods": [[6, null]], "Resampling and the Bias-Variance Trade-off": [[19, "resampling-and-the-bias-variance-trade-off"]], "Resampling methods": [[6, "id1"]], "Residual Error": [[28, "residual-error"], [29, "residual-error"]], "Resources on differential equations and deep learning": [[2, "resources-on-differential-equations-and-deep-learning"]], "Revisiting Ordinary Least Squares": [[29, "revisiting-ordinary-least-squares"]], "Revisiting our Linear Regression Solvers": [[13, "revisiting-our-linear-regression-solvers"]], "Rewriting the Covariance and/or Correlation Matrix": [[28, "rewriting-the-covariance-and-or-correlation-matrix"]], "Rewriting the fitting procedure as a linear algebra problem": [[27, "rewriting-the-fitting-procedure-as-a-linear-algebra-problem"]], "Rewriting the fitting procedure as a linear algebra problem, more details": [[27, "rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details"]], "Ridge Regression": [[29, "ridge-regression"]], "Ridge and LASSO Regression": [[28, "ridge-and-lasso-regression"], [29, "ridge-and-lasso-regression"], [29, "id2"]], "Ridge and Lasso Regression": [[5, null], [5, "id1"]], "SGD example": [[30, "sgd-example"]], "SVD analysis": [[29, "svd-analysis"]], "Same code but now with momentum gradient descent": [[13, "same-code-but-now-with-momentum-gradient-descent"], [30, "same-code-but-now-with-momentum-gradient-descent"], [30, "id3"], [30, "id4"]], "Schedule first week": [[27, "schedule-first-week"]], "Schematic Regression Procedure": [[9, "schematic-regression-procedure"]], "Second moment of the gradient": [[30, "second-moment-of-the-gradient"]], "September 15-19": [[19, "september-15-19"]], "Setting up the Back propagation algorithm": [[12, "setting-up-the-back-propagation-algorithm"]], "Setting up the Matrix to be inverted": [[28, "setting-up-the-matrix-to-be-inverted"], [29, "setting-up-the-matrix-to-be-inverted"]], "Setting up the network using Autograd; The full program": [[2, "setting-up-the-network-using-autograd-the-full-program"]], "Similar (second order function now) problem but now with AdaGrad": [[13, "similar-second-order-function-now-problem-but-now-with-adagrad"], [30, "similar-second-order-function-now-problem-but-now-with-adagrad"]], "Simple Python Code to read in Data and perform Classification": [[9, "simple-python-code-to-read-in-data-and-perform-classification"]], "Simple case": [[28, "simple-case"], [29, "simple-case"]], "Simple code for solving the above problem": [[29, "simple-code-for-solving-the-above-problem"]], "Simple example code": [[30, "simple-example-code"]], "Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression": [[29, "simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression"]], "Simple geometric interpretation": [[29, "simple-geometric-interpretation"]], "Simple linear regression model using scikit-learn": [[0, "simple-linear-regression-model-using-scikit-learn"], [27, "simple-linear-regression-model-using-scikit-learn"]], "Simple one-dimensional second-order polynomial": [[18, "simple-one-dimensional-second-order-polynomial"]], "Simple program": [[29, "simple-program"], [30, "simple-program"]], "Slightly different approach": [[30, "slightly-different-approach"]], "Sneaking in automatic differentiation using Autograd": [[30, "sneaking-in-automatic-differentiation-using-autograd"]], "Software and needed installations": [[22, "software-and-needed-installations"], [27, "software-and-needed-installations"]], "Solving Differential Equations with Deep Learning": [[2, null]], "Solving the one dimensional Poisson equation": [[2, "solving-the-one-dimensional-poisson-equation"]], "Solving the wave equation with Neural Networks": [[2, "solving-the-wave-equation-with-neural-networks"]], "Some famous Matrices": [[21, "some-famous-matrices"]], "Some simple problems": [[13, "some-simple-problems"], [29, "some-simple-problems"]], "Some useful matrix and vector expressions": [[28, "some-useful-matrix-and-vector-expressions"]], "Splitting our Data in Training and Test data": [[0, "splitting-our-data-in-training-and-test-data"], [28, "splitting-our-data-in-training-and-test-data"]], "Standard steepest descent": [[13, "standard-steepest-descent"]], "Statistical analysis and optimization of data": [[20, "statistical-analysis-and-optimization-of-data"], [27, "statistical-analysis-and-optimization-of-data"]], "Steepest descent": [[13, "steepest-descent"], [29, "steepest-descent"]], "Stochastic Gradient Descent": [[30, "stochastic-gradient-descent"]], "Stochastic Gradient Descent (SGD)": [[13, "stochastic-gradient-descent-sgd"], [30, "stochastic-gradient-descent-sgd"]], "Stochastic variables and the main concepts, the discrete case": [[24, "stochastic-variables-and-the-main-concepts-the-discrete-case"]], "Support Vector Machines, overarching aims": [[8, null]], "Systematic reduction": [[3, "systematic-reduction"]], "Teachers": [[27, "teachers"]], "Teachers and Grading": [[25, null]], "Teaching Assistants Fall semester 2023": [[25, "teaching-assistants-fall-semester-2023"]], "Tentative deadllines for projects": [[25, "tentative-deadllines-for-projects"]], "Testing the Means Squared Error as function of Complexity": [[0, "testing-the-means-squared-error-as-function-of-complexity"], [28, "testing-the-means-squared-error-as-function-of-complexity"]], "Textbooks": [[26, null]], "The Algorithm before theorem": [[11, "the-algorithm-before-theorem"]], "The Breast Cancer Data, now with Keras": [[1, "the-breast-cancer-data-now-with-keras"]], "The CART algorithm for Classification": [[9, "the-cart-algorithm-for-classification"]], "The CART algorithm for Regression": [[9, "the-cart-algorithm-for-regression"]], "The CIFAR01 data set": [[3, "the-cifar01-data-set"]], "The Hessian matrix": [[29, "the-hessian-matrix"], [30, "the-hessian-matrix"]], "The Hessian matrix for Ridge Regression": [[29, "the-hessian-matrix-for-ridge-regression"], [30, "the-hessian-matrix-for-ridge-regression"]], "The Jacobian": [[28, "the-jacobian"]], "The MNIST dataset again": [[3, "the-mnist-dataset-again"]], "The OLS case": [[29, "the-ols-case"]], "The RELU function family": [[1, "the-relu-function-family"]], "The Ridge case": [[29, "the-ridge-case"]], "The SVD, a Fantastic Algorithm": [[28, "the-svd-a-fantastic-algorithm"], [29, "the-svd-a-fantastic-algorithm"]], "The Softmax function": [[1, "the-softmax-function"]], "The \\chi^2 function": [[0, "the-chi-2-function"], [27, "the-chi-2-function"], [27, "id4"], [27, "id5"], [27, "id6"], [27, "id7"], [27, "id8"]], "The bias-variance tradeoff": [[6, "the-bias-variance-tradeoff"]], "The code for solving the ODE": [[2, "the-code-for-solving-the-ode"]], "The complete code with a simple data set": [[28, "the-complete-code-with-a-simple-data-set"]], "The cost/loss function": [[28, "the-cost-loss-function"]], "The course has two central parts": [[20, "the-course-has-two-central-parts"]], "The derivative of the cost/loss function": [[29, "the-derivative-of-the-cost-loss-function"], [30, "the-derivative-of-the-cost-loss-function"]], "The equations": [[29, "the-equations"]], "The equations for ordinary least squares": [[28, "the-equations-for-ordinary-least-squares"]], "The first Case": [[29, "the-first-case"]], "The gradient step": [[30, "the-gradient-step"]], "The ideal": [[29, "the-ideal"]], "The logistic function": [[7, "the-logistic-function"]], "The mean squared error and its derivative": [[28, "the-mean-squared-error-and-its-derivative"]], "The moons example": [[8, "the-moons-example"]], "The multilayer perceptron (MLP)": [[12, "the-multilayer-perceptron-mlp"]], "The network with one input layer, specified number of hidden layers, and one output layer": [[2, "the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer"]], "The plethora of machine learning algorithms/methods": [[27, "the-plethora-of-machine-learning-algorithms-methods"]], "The sensitiveness of the gradient descent": [[29, "the-sensitiveness-of-the-gradient-descent"]], "The singular value decomposition": [[5, "the-singular-value-decomposition"], [28, "the-singular-value-decomposition"], [29, "the-singular-value-decomposition"]], "The two-dimensional case": [[8, "the-two-dimensional-case"]], "Time decay rate": [[30, "time-decay-rate"]], "To our real data: nuclear binding energies. Brief reminder on masses and binding energies": [[27, "to-our-real-data-nuclear-binding-energies-brief-reminder-on-masses-and-binding-energies"]], "Topics covered in this course: Statistical analysis and optimization of data": [[27, "topics-covered-in-this-course-statistical-analysis-and-optimization-of-data"]], "Towards the PCA theorem": [[11, "towards-the-pca-theorem"]], "Train and test datasets": [[1, "train-and-test-datasets"]], "Two-dimensional Objects": [[3, "two-dimensional-objects"]], "Type of problem": [[2, "type-of-problem"]], "Types of Machine Learning": [[27, "types-of-machine-learning"]], "Useful Python libraries": [[20, "useful-python-libraries"], [27, "useful-python-libraries"]], "Using Autograd": [[13, "using-autograd"]], "Using forward Euler to solve the ODE": [[2, "using-forward-euler-to-solve-the-ode"]], "Using gradient descent methods, limitations": [[13, "using-gradient-descent-methods-limitations"], [29, "using-gradient-descent-methods-limitations"], [30, "using-gradient-descent-methods-limitations"]], "Visualization": [[1, "visualization"], [1, "id1"]], "Visualizing the Tree, Classification": [[9, "visualizing-the-tree-classification"]], "Week 34: Introduction to the course, Logistics and Practicalities": [[27, null]], "Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression": [[28, null]], "Week 36: Linear Regression and Gradient descent": [[29, null]], "Week 37: Statistical interpretations and Resampling Methods": [[30, null]], "What Is Generative Modeling?": [[27, "what-is-generative-modeling"]], "What does it mean?": [[28, "what-does-it-mean"], [29, "what-does-it-mean"]], "What is Machine Learning?": [[0, "what-is-machine-learning"]], "What is a good model?": [[0, "what-is-a-good-model"], [27, "what-is-a-good-model"]], "What is a good model? Can we define it?": [[27, "what-is-a-good-model-can-we-define-it"]], "When do we stop?": [[30, "when-do-we-stop"]], "Which activation function should I use?": [[1, "which-activation-function-should-i-use"]], "Why Linear Regression (aka Ordinary Least Squares and family)": [[27, "why-linear-regression-aka-ordinary-least-squares-and-family"]], "Wisconsin Cancer Data": [[7, "wisconsin-cancer-data"]], "With Lasso Regression": [[29, "with-lasso-regression"]], "Writing Our First Generative Adversarial Network": [[4, "writing-our-first-generative-adversarial-network"]], "Writing our own PCA code": [[11, "writing-our-own-pca-code"]], "Writing the Cost Function": [[29, "writing-the-cost-function"]], "XGBoost: Extreme Gradient Boosting": [[10, "xgboost-extreme-gradient-boosting"]], "Yet another Example": [[29, "yet-another-example"]], "a) Expression for Ridge regression": [[17, "a-expression-for-ridge-regression"]], "scikit-learn implementation": [[1, "scikit-learn-implementation"]]}, "docnames": ["chapter1", "chapter10", "chapter11", "chapter12", "chapter13", "chapter2", "chapter3", "chapter4", "chapter5", "chapter6", "chapter7", "chapter8", "chapter9", "chapteroptimization", "clustering", "exercisesweek34", "exercisesweek35", "exercisesweek36", "exercisesweek37", "exercisesweek38", "intro", "linalg", "project1", "schedule", "statistics", "teachers", "textbooks", "week34", "week35", "week36", "week37"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1}, "filenames": ["chapter1.ipynb", "chapter10.ipynb", "chapter11.ipynb", "chapter12.ipynb", "chapter13.ipynb", "chapter2.ipynb", "chapter3.ipynb", "chapter4.ipynb", "chapter5.ipynb", "chapter6.ipynb", "chapter7.ipynb", "chapter8.ipynb", "chapter9.ipynb", "chapteroptimization.ipynb", "clustering.ipynb", "exercisesweek34.ipynb", "exercisesweek35.ipynb", "exercisesweek36.ipynb", "exercisesweek37.ipynb", "exercisesweek38.ipynb", "intro.md", "linalg.ipynb", "project1.ipynb", "schedule.md", "statistics.ipynb", "teachers.md", "textbooks.md", "week34.ipynb", "week35.ipynb", "week36.ipynb", "week37.ipynb"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 22, 24, 25, 27, 28], "0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29, 30], "00": [0, 1, 5, 11, 27, 28], "000": [1, 3], "00000000e": [], "001": [2, 8, 13, 29, 30], "004": 5, "004113634617443131": 28, "004113634617443139": 28, "00411363461744314": 28, "004113634617443147": 28, "00727646693": [0, 27], "0086649156": [0, 27], "01": [0, 1, 2, 5, 9, 11, 13, 17, 26, 27, 28, 30], "0110": 24, "01719003e": [], "02": [0, 4, 7, 12, 27], "02334824": [], "02857": 4, "02f": 6, "03077640549": 4, "03097597e": [], "031": 5, "04": 11, "0458": 9, "05": [4, 6], "062292565": 4, "062435": [], "06730814": [], "07": [], "0713": [0, 27], "07285": 3, "08": 24, "08078025e": [], "08336233266": 4, "08376632": 28, "083766322923899": 28, "0837663229239043": 28, "0917": 9, "0n": [0, 27], "0x113e21950": 17, "1": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 23, 24, 25, 26, 27, 29, 30], "10": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 21, 23, 24, 25, 27, 28, 29, 30], "100": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 27, 28, 29, 30], "1000": [0, 1, 2, 4, 5, 8, 11, 13, 14, 18, 19, 20, 24, 27, 29, 30], "10000": [2, 5, 6, 10, 11, 13, 24], "100000": 8, "10001": 10, "1001": 24, "1002": 24, "1003": 24, "1005": 24, "1009": 24, "101": 16, "1011": 24, "1013": 24, "1013904243": 24, "1015": 24, "102": 16, "1023": 24, "1024": 3, "1026": 24, "1027": 24, "103": 1, "1030": 24, "1037": 24, "1038": 24, "1040": 24, "1047": 24, "107": 16, "108": [], "10th": 9, "10x": [0, 27], "11": [0, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 21, 22, 24, 26, 27, 28, 29, 30], "110": [], "1100": 24, "1101": 24, "111": [1, 7, 12], "112": 16, "11340253": [], "11590451": [], "116": 16, "117": 16, "118": 16, "12": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 18, 21, 24, 26, 27, 28, 29, 30], "120": 3, "121": [8, 9, 10, 16], "1215pm": [25, 27], "122": [8, 9, 10], "124": [0, 27], "125": 16, "127": [4, 16], "128": [3, 4, 13, 30], "129": 16, "1298": 9, "12pm": [25, 27], "13": [0, 2, 9, 12, 21, 24, 27], "131": 16, "133": 7, "135": 16, "136": 16, "14": [0, 2, 4, 6, 8, 9, 10, 12, 21, 24, 26, 28], "141": 16, "1412": 30, "143": 16, "1446729567": 4, "149": 16, "14g": 6, "15": [0, 2, 4, 6, 7, 8, 9, 12, 13, 22, 24, 27, 29, 30], "150": [4, 8], "152": 16, "153760": [], "156": 16, "157": [], "158": [], "159": 16, "15g": 6, "15pm": 27, "16": [1, 2, 3, 4, 5, 8, 9, 10, 24, 27, 29], "160": 16, "1603": 3, "161": 16, "162": 16, "16231451": 4, "163": 16, "16384": 3, "164": 16, "167": 16, "17": [1, 2, 8, 24], "172": 16, "173": 16, "176": 16, "178": 16, "179": 16, "1797": 1, "18": [2, 6, 7, 8, 9, 10, 24, 27], "1807": 4, "18392847": [], "19": [2, 24, 27], "1940": [], "1943": 12, "19569961": 28, "1970": [21, 27], "1973": 9, "1979": 6, "1_1": 12, "1_2": 12, "1_3": 12, "1cm": [0, 8, 10, 24, 27], "1d": [1, 2, 3], "1e": [2, 4, 13, 14, 30], "1e10": 14, "1e4": 6, "1f": 1, "1k": 21, "1n": [0, 27], "1x": [0, 27], "2": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 22, 24, 26, 30], "20": [0, 1, 2, 6, 7, 8, 16, 17, 24, 25, 27, 28, 29, 30], "200": [0, 2, 3, 4, 8, 9, 10], "2000": [0, 28], "2004": [13, 29], "2006": 26, "20072279": [], "2008": 27, "2010": 1, "2011": 1, "2012": 30, "2014": [4, 30], "2015": 1, "2016": [0, 27], "2018": [0, 6, 28], "2021": [6, 14, 28, 30], "2022": 27, "2025": [18, 27, 28, 29, 30], "21": [0, 1, 5, 7, 9, 12, 21, 27, 28, 29], "2116753732": 4, "215pm": [25, 27], "2167072": [], "22": [0, 1, 5, 12, 13, 21, 27, 28, 29], "221": 8, "225": 4, "22948497": [], "23": [1, 12, 21], "24": [0, 1, 21, 27], "25": [2, 3, 4, 5, 6, 8, 9, 11, 28], "250": [2, 4, 7, 9], "25000": [], "250154": [], "253775": [], "255": 3, "256": 4, "25x": 22, "26": [], "26303845": [], "264": [], "265": [], "265109911": 4, "266": [], "269": [], "27": 1, "270": [], "278": [29, 30], "27n_": 24, "28": [1, 3, 4], "283": [29, 30], "2830637392": 4, "2861": 24, "2873": 9, "2882": 24, "2886": 24, "2890": [0, 27], "2892": 24, "29": 28, "2915": 24, "2931": 27, "29364655": [], "294399745619595": [], "296247": [], "2968": 27, "2980": 27, "298273": [], "298375": [], "2990": 27, "2_": 12, "2_1": 12, "2_2": 12, "2_3": 12, "2_i": 12, "2_m": [6, 24], "2_t": [13, 30], "2_x": 24, "2a": 17, "2b": 24, "2c8f433990d1": 30, "2cm": 8, "2d": [1, 3, 11, 12, 20, 27], "2e": 6, "2f": [0, 7, 9, 10, 11, 12, 27], "2g": 2, "2g_i": 2, "2k": 3, "2m": 6, "2mvizaqfst8": 28, "2n": [0, 2, 3, 27, 28], "2nd": 9, "2p": 24, "2pt": 4, "2x": [0, 3, 8, 13, 27], "2x_ix_jy_iy_j": 8, "2x_j": 8, "2y_i": 10, "2y_j": 8, "3": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 22, 23, 24, 25, 27, 29, 30], "30": [0, 1, 4, 6, 7, 10, 13, 25, 30], "30000": [0, 27], "3072": 3, "31": [12, 21, 24], "315": [6, 28, 30], "3155": [0, 5, 6, 28, 29, 30], "32": [3, 4, 6, 12, 13, 21, 24, 30], "3200": 1, "3250": 1, "3297": [], "33": [12, 21, 25], "3303": [], "3310": [], "332331": [], "333": 7, "3331": [], "3337": [], "34": 21, "3436": [0, 27], "3437": [0, 27], "35": [0, 6, 22, 27, 29, 30], "3581341341": 4, "359": [5, 29], "36": [0, 5, 6, 18, 22, 24], "37": [22, 29], "370782966": 4, "38": [22, 24], "39": [0, 25, 27], "3d": [2, 3, 4, 6, 13, 16], "3f": [1, 3, 9], "3n": 21, "3x": [2, 8], "3x_i": 2, "3y": 8, "4": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 22, 24, 27, 29, 30], "40": [1, 6, 25, 27], "400": 4, "4000": 27, "4050": [26, 27], "41": 21, "4155": [2, 15], "41589548": [], "42": [1, 4, 8, 9, 10, 21], "43": [0, 7, 21], "4310": 27, "436462435": 4, "44": [0, 21, 29, 30], "45": [25, 27], "46": [25, 27], "462": 7, "47": [25, 27], "479465113": 4, "47958494": [], "48": [], "48257387": [25, 27], "49": [5, 6, 11], "49152": 3, "4940954": [0, 27], "4990": 24, "4992": 24, "4997": 24, "4c4c7f": [9, 10], "4d": 3, "4f": 6, "4pm": [25, 27], "4y": 8, "4y_i": 10, "5": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 22, 24, 27, 28, 29, 30], "50": [1, 2, 3, 4, 6, 7, 8, 10, 13, 27, 28, 30], "500": [1, 3, 4, 6, 9, 10, 13, 30], "5018": 24, "506": [], "507d50": [9, 10], "50j": 13, "50x10": 1, "51": 10, "510": 1, "512132": [], "5177783846": 4, "53": 9, "54": [6, 24], "5411205": [], "54894451": [], "55": 1, "56": 1, "56536": [0, 27], "569": 1, "57": [0, 8, 25, 27], "571": [5, 29], "58": [10, 25, 27], "591317992": 4, "5cm": 24, "5f": [8, 30], "5x": [8, 18], "5y": 8, "6": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 18, 21, 24, 25, 27, 28, 29, 30], "60": [1, 3], "60000": 4, "6019067271": 4, "606439": [], "625": 7, "63": 1, "64": [1, 3, 4, 13, 21, 27, 30], "64x50": 1, "65": [1, 8, 9], "6887363571": 4, "69": [16, 24], "69069n_": 24, "691": [], "6980": 30, "6n_": 24, "7": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 21, 22, 24, 26, 27, 28, 30], "70": [1, 7], "70653767": 4, "71": 1, "724": 3, "73": [], "7304881": [], "75": [5, 6, 8, 11], "76": [25, 27], "765": 7, "77": [25, 27], "7718": 9, "7782028952": 4, "77893972": [], "78": [], "7d7d58": [9, 10], "8": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 18, 19, 21, 24, 25, 27, 29], "80": [0, 1, 5, 8, 17, 28], "800": [4, 7], "81": 1, "815am": [25, 27], "85": 1, "8702784034": 4, "88": 27, "8f": 6, "8g": 6, "8n": 21, "8x8": 1, "9": [0, 1, 2, 4, 5, 6, 7, 8, 9, 11, 12, 13, 21, 24, 27, 30], "90": 1, "9040": 9, "91": [25, 27], "92": [25, 27], "93": 16, "931": [0, 27], "933": [5, 29], "937": 24, "938": 24, "939": [0, 24, 27], "94": 24, "95": [1, 11], "954": 24, "955820c21e8b": 4, "96": 6, "960": 24, "961": 24, "962": 24, "9649652536": 4, "96611194e": [], "9780387310732": 26, "9780387848570": 26, "9781098134174": 27, "9781492032632": 26, "9781801819312": 27, "97898392": 28, "98": [0, 1, 16], "985": 24, "986": 24, "989": 24, "9898ff": [9, 10], "99": [13, 16, 30], "991": 24, "992": 24, "993": 24, "996": 5, "999": [9, 24, 30], "9x": 6, "9y": 6, "A": [2, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30], "AND": 2, "And": [0, 3, 4, 5, 6, 9, 13, 20, 22, 24, 29], "As": [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 21, 22, 24, 27, 28, 29, 30], "At": [0, 4, 6, 13, 27, 30], "BE": [0, 27], "Be": [2, 18, 20, 27], "Being": 13, "But": [0, 1, 2, 3, 5, 6, 9, 10, 16, 24, 28], "By": [0, 3, 5, 6, 12, 13, 17, 21, 27, 28, 29, 30], "For": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29, 30], "IF": [6, 28, 30], "IN": 26, "If": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 18, 20, 21, 22, 24, 27, 28, 29, 30], "In": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 24, 26, 27, 28, 29, 30], "Ising": [5, 12, 28, 29], "It": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 20, 21, 22, 24, 27, 28, 29, 30], "Its": [1, 2, 4, 11], "No": [6, 9, 27, 28, 30], "Not": [0, 1, 5, 6, 28, 29, 30], "OR": 24, "Of": 24, "On": [0, 3, 22, 24, 25, 26, 27], "One": [0, 1, 3, 4, 5, 6, 7, 8, 11, 12, 13, 17, 24, 28, 29, 30], "Or": [0, 1, 6, 27], "Such": [0, 6, 12, 16, 24, 30], "That": [0, 5, 7, 10, 11, 12, 14, 22, 24, 27], "The": [4, 10, 13, 14, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26], "Then": [0, 1, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 21, 27, 29, 30], "There": [0, 3, 4, 5, 6, 8, 9, 11, 12, 14, 15, 21, 22, 24, 25, 27, 28, 29, 30], "These": [0, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 17, 18, 21, 22, 24, 25, 27, 28, 29, 30], "To": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 21, 24, 28, 29, 30], "With": [0, 5, 6, 8, 9, 10, 11, 12, 14, 16, 19, 21, 22, 24, 27, 28], "_": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 21, 22, 27, 28, 29, 30], "_0": [5, 8, 10, 11, 13, 28, 29], "_1": [2, 5, 6, 8, 10, 11, 12, 13, 14, 21, 28, 29, 30], "_2": [2, 5, 8, 11, 12, 13, 21, 28, 30], "_3": 21, "_4": 21, "_9": [13, 30], "__class__": 10, "__doc__": 6, "__future__": [8, 9], "__init__": 1, "__main__": 2, "__name__": [2, 10], "_auto1": [2, 3, 4, 5, 6, 7, 12, 13, 21, 24, 28, 29, 30], "_auto10": [6, 12], "_auto11": 6, "_auto12": 6, "_auto2": [2, 3, 4, 5, 6, 12, 13, 21, 24, 30], "_auto3": [3, 4, 5, 6, 12, 13, 21, 30], "_auto4": [4, 6, 12, 13, 21, 30], "_auto5": [4, 6, 12, 13, 21, 30], "_auto6": [4, 6, 12, 21], "_auto7": [4, 6, 12, 21], "_auto8": [6, 12], "_auto9": [6, 12], "_build": [0, 20, 22, 26, 27], "_c": 1, "_center": [], "_compon": 11, "_depth": 9, "_export": [15, 16], "_fraction": 9, "_i": [0, 1, 2, 5, 6, 7, 8, 11, 12, 13, 19, 22, 27, 28, 29, 30], "_j": [0, 1, 2, 3, 5, 6, 8, 13, 19, 22, 28, 29, 30], "_k": [13, 29, 30], "_l": 12, "_lambda": 6, "_leaf": 9, "_m": 10, "_multilayer_perceptron": [], "_n": [2, 5, 8, 11, 13, 28, 29, 30], "_node": 9, "_norm": [], "_p": [5, 8, 28, 29], "_ratio": 11, "_sampl": 9, "_split": [6, 9, 22], "_t": [13, 30], "_test": [6, 22], "_varianc": 11, "_weight": 9, "a0": 3, "a0faa0": [9, 10], "a1": [0, 27], "a2": [0, 27], "a3": [0, 27], "a4": [0, 27], "a_": [0, 1, 16, 21, 27, 28], "a_0": [0, 27], "a_1a": [0, 27], "a_2a": [0, 27], "a_3": [0, 27], "a_3a": [0, 27], "a_4": [0, 27], "a_4a": [0, 27], "a_h": 1, "a_i": [0, 1, 2, 12, 27], "a_j": [1, 12], "a_k": [0, 1, 12], "aaron": 26, "ab": [0, 2, 5, 13, 14, 27, 28, 30], "ab_channel": 20, "abandon": 1, "abid": 24, "abil": [0, 10], "abl": [0, 1, 4, 5, 6, 7, 10, 12, 13, 16, 18, 19, 22, 28, 29, 30], "about": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 20, 21, 22, 25, 30], "abov": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 26, 27, 28, 30], "abovement": [6, 22, 27], "abscissa": [13, 29], "absolut": [0, 2, 5, 6, 13, 27, 28, 29], "absorb": [28, 29], "abstract": 1, "acceler": [13, 30], "accept": [0, 3, 6, 9, 22, 28], "access": [3, 11, 24, 27], "accid": [4, 6], "accompani": [0, 27, 28], "accomplish": [8, 9, 13, 30], "accord": [0, 1, 2, 5, 6, 9, 12, 13, 14, 24, 27, 29, 30], "accordingli": 11, "account": [0, 3, 5, 13, 15, 16, 24, 27, 30], "accumul": [12, 13, 24, 30], "accur": [0, 3, 4, 6, 10, 13, 30], "accuraci": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 12, 27, 28, 29], "accuracy_scor": [0, 1, 10, 27], "accuracy_score_numpi": 1, "achiev": [0, 1, 5, 6, 8, 12, 21, 27, 30], "aco": 24, "acquaint": 20, "acquir": [1, 20, 27], "acr": [], "across": [1, 3, 6, 9, 17, 20, 27], "act": [1, 3, 21], "action": 24, "activ": [0, 2, 3, 4, 9, 15, 23, 25, 27, 30], "actual": [0, 1, 4, 5, 6, 8, 11, 15, 16, 18, 21, 24, 27, 28, 29, 30], "ad": [1, 3, 4, 5, 8, 13, 15, 16, 21, 29, 30], "ada_clf": 10, "adaboostclassifi": 10, "adadelta": [13, 30], "adagrad": 22, "adam": [1, 3, 4, 22, 27], "adapt": [4, 6, 13, 17, 26, 29], "add": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 15, 16, 17, 18, 24, 25, 27, 28, 29, 30], "add_subplot": [1, 7, 12, 14], "addendum": 5, "addit": [0, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 20, 21, 22, 24, 25, 26, 27, 28, 30], "addition": [12, 13, 29, 30], "address": [1, 9, 11, 13, 27, 30], "adjac": [3, 12], "adjoint": [5, 28], "adjust": [0, 5, 12, 13, 29, 30], "admir": [0, 27], "advanc": [4, 6, 12, 26, 27, 30], "advantag": [1, 3, 5, 6, 10, 13, 21, 29, 30], "adversari": 27, "afecionado": 27, "affect": [3, 15], "affin": [0, 3, 8, 11, 28], "afford": 3, "aficionado": 27, "aforement": 14, "african": [], "after": [0, 1, 2, 4, 5, 6, 9, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "afterward": [0, 27], "ag": [0, 7, 27, 28], "ag_0": 2, "again": [0, 1, 4, 5, 6, 7, 8, 10, 11, 12, 13, 22, 24, 27, 28, 29, 30], "against": [1, 4, 7, 10], "agegroup": 7, "agegroupmean": 7, "aggreg": [9, 10], "agorithm": 10, "agre": [5, 6, 24, 28, 29, 30], "agreement": [13, 30], "ahead": 9, "ai": [0, 26], "aid": 11, "aim": [0, 1, 4, 6, 7, 11, 14, 16, 17, 19, 20, 21, 22, 28], "ainv": 5, "airplan": 3, "aka": 5, "al": [0, 2, 4, 16, 17, 26, 27, 28, 29], "alarm": [5, 7], "aldo": 28, "algebra": [0, 3, 5, 13, 20, 28, 29], "algorithm": [0, 1, 2, 4, 5, 6, 7, 8, 13, 14, 16, 20, 21, 22, 24, 26], "align": [0, 2, 5, 6, 7, 8, 13, 24, 27, 28, 29], "all": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "allevi": [1, 13, 29], "alloc": [3, 21], "allow": [0, 1, 2, 3, 5, 6, 8, 10, 13, 15, 20, 21, 22, 27, 28, 29, 30], "almost": [0, 1, 6, 8, 11, 13, 24, 29, 30], "alon": [2, 9], "along": [2, 3, 4, 5, 6, 9, 10, 11, 15, 20, 21, 27, 28, 29], "alpha": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 13, 14, 24, 27, 28, 29, 30], "alpha_": [10, 30], "alpha_0": 3, "alpha_1": 3, "alpha_2": 3, "alpha_i": [3, 13], "alpha_k": 13, "alpha_m": 10, "alpha_n": 3, "alpha_opt": 13, "alreadi": [2, 3, 4, 5, 6, 10, 12, 15, 20, 21, 24, 27, 28, 29], "also": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "alter": 1, "altern": [0, 1, 4, 5, 6, 8, 9, 11, 13, 15, 18, 21, 22, 27, 28, 30], "although": [0, 1, 5, 6, 8, 10, 13, 16, 27, 30], "alwai": [0, 3, 5, 6, 12, 13, 16, 22, 24, 27, 28, 29, 30], "am": 4, "ame2016": [0, 27], "american": [], "among": [0, 3, 5, 9, 10, 12, 21, 27, 28], "amongst": 5, "amount": [0, 1, 3, 4, 6, 8, 10, 14, 20], "an": [1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29, 30], "an_": 24, "anaconda": [0, 1, 20, 22, 27], "analogi": [13, 30], "analys": 6, "analysi": [1, 3, 4, 7, 14, 19, 21, 26], "analyt": [2, 3, 5, 6, 7, 12, 13, 17, 20, 22, 27, 28, 29, 30], "analyz": [0, 1, 3, 4, 5, 6, 16, 22, 24, 28, 29], "andrew": 1, "angl": [0, 3, 9, 28, 30], "anharmon": 3, "ani": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 19, 24, 27, 28, 30], "anim": [4, 12], "ann": 12, "annot": [0, 1, 3, 7, 8, 27], "announc": 27, "anonym": 18, "anoth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 21, 22, 24, 27, 28, 30], "ansatz": [0, 18, 27], "answer": [0, 1, 3, 5, 6, 21, 22, 25, 27], "antialias": [2, 6], "anticip": 4, "anymor": [1, 8], "anyon": [4, 8, 15], "anyth": [1, 15, 16, 24], "anytim": [25, 27], "apach": 1, "apart": [11, 13, 29], "api": [1, 20, 27], "appar": 2, "appear": [0, 1, 3, 13, 21, 24], "append": [1, 3, 4, 8, 9, 13, 19, 27, 30], "appendix": 22, "appli": [0, 1, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 18, 22, 24, 26, 27, 28, 30], "applic": [0, 1, 3, 4, 5, 6, 7, 9, 12, 13, 16, 21, 24, 26, 27, 28, 29, 30], "apply_gradi": 4, "approach": [1, 2, 4, 5, 6, 9, 10, 11, 12, 13, 15, 16, 18, 20, 22, 24, 26, 28, 29], "approch": 22, "appropri": [2, 6, 9, 12, 13, 17, 20, 24, 30], "approv": 27, "approx": [0, 2, 3, 6, 10, 11, 13, 18, 22, 24, 27, 29, 30], "approxim": [0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 13, 19, 22, 24, 27, 28, 29, 30], "apt": [0, 20, 22, 27], "aq": 24, "ar": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "aragorn": 27, "arang": [1, 3, 4, 6, 7, 9, 10, 12, 13, 27, 30], "arbitrari": [1, 4, 6, 8, 12, 13, 24, 29], "arbitrarili": [0, 1, 11, 27], "arc": 6, "architectur": [3, 4, 12], "archiv": 22, "area": [0, 3, 6, 26, 27], "argmax": [1, 11], "argmin": [4, 10, 14], "argsort": 11, "argu": [1, 13, 30], "argument": [0, 2, 3, 5, 11, 12, 13, 17, 27, 28, 30], "aris": [0, 6, 12, 13, 24, 27, 29], "arithmet": [0, 13, 21, 27], "arm": [6, 28, 30], "armadillo": 21, "around": [0, 1, 4, 5, 6, 11, 18, 22, 24, 27], "arrai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 16, 18, 20, 22, 24, 28, 29, 30], "arrang": [3, 27], "arraybox": 13, "arriv": [0, 6, 9, 11, 21, 24, 27], "arrow": 12, "arrowprop": 8, "art": [0, 1, 20], "articl": [0, 3, 4, 6, 10, 19, 27, 28, 29, 30], "artifici": [0, 2, 7, 12, 26, 27], "artificialneuron": 12, "arug": 13, "arxiv": [3, 4, 30], "asarrai": [0, 6, 9, 28, 30], "asid": 28, "ask": [5, 6, 11, 12, 15, 22], "aspect": [0, 6, 20, 27, 28], "assembl": 3, "assembli": [0, 27], "assert": 4, "assess": [0, 6, 22, 27, 28], "assici": 4, "assign": [0, 7, 8, 9, 12, 13, 14, 15, 23, 25, 26, 27], "associ": [0, 6, 9, 12, 14, 24, 27], "assum": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 19, 21, 22, 24, 27, 28, 29, 30], "assumpt": [0, 3, 5, 6, 9, 11, 24, 27, 28], "ast": [0, 5, 6, 27], "astyp": [4, 9, 10], "asymmetri": [0, 27], "asymptot": [4, 6], "atom": [0, 27], "attempt": [0, 4, 6, 7, 8, 10, 27, 28, 30], "attend": 27, "attent": [0, 21, 27], "attract": [0, 10, 27], "attribut": [0, 9, 27], "audi": [0, 27], "audio": [3, 4], "august": [27, 28], "aurelien": [0, 26, 27], "austfjel": 6, "auth": 15, "authent": 15, "author": [0, 1, 10, 24], "authour": 27, "auto": [9, 10, 24], "auto_exampl": [22, 28], "autocor": 24, "autocorrelation_tim": 24, "autocorrelform": 24, "autocovari": 24, "autoencod": [4, 20, 27], "autoencond": 20, "autograd": [20, 27], "autom": [0, 20, 26, 27], "automac": 21, "automag": 27, "automat": [0, 1, 2, 3, 4, 11, 16, 20, 21, 27], "automobil": 3, "autonom": 4, "avail": [0, 1, 4, 6, 10, 11, 20, 21, 22, 23, 25, 26, 27], "averag": [0, 1, 3, 6, 9, 10, 13, 14, 24, 25, 27, 28, 30], "avoid": [0, 4, 5, 6, 9, 11, 13, 18, 21, 28, 30], "awai": [2, 3, 6, 28, 30], "awar": [2, 10], "award": [25, 27], "ax": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 22, 27], "axes3d": [2, 6, 13, 29, 30], "axes_grid1": 6, "axhlin": 8, "axi": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18, 21, 24, 27, 28, 29, 30], "axiom": 5, "axvlin": [4, 8], "axvspan": 4, "b": [0, 1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 19, 24, 25, 27, 28, 29, 30], "b1": 8, "b2": 8, "b3": 8, "b_": [0, 1, 21], "b_0": 0, "b_1": [0, 2, 12, 13, 30], "b_2": [0, 13], "b_5": [13, 30], "b_group": 9, "b_i": [0, 1, 2, 12, 27], "b_ia_": [0, 27], "b_ia_i": 0, "b_index": 9, "b_j": [1, 12], "b_k": [0, 1, 12, 13, 30], "b_m": 12, "b_score": 9, "b_valu": 9, "ba": 30, "babcock": 27, "bachelor": [23, 25], "back": [0, 3, 4, 5, 6, 8, 9, 10, 15, 16, 21, 24, 27, 30], "backbon": 21, "backend": [1, 4], "background": [26, 27], "backpropag": 1, "backtrack": 9, "backup": 21, "backward": [1, 2, 4, 12, 21], "bad": [6, 17, 28], "badli": 24, "bag": [9, 20, 27], "bag_clf": 10, "baggin": 27, "baggingboot": 10, "baggingclassifi": 10, "baggingtre": 10, "balanc": [6, 30], "ballpark": 18, "band": 21, "bandwidth": 21, "bar": [0, 6, 11, 22, 27], "barber": 26, "bare": [4, 10], "base": [0, 1, 3, 4, 5, 7, 8, 9, 10, 14, 15, 16, 17, 20, 24, 25, 26, 27, 28, 29], "basi": [5, 7, 8, 10, 11, 12, 13, 21, 28, 29], "basic": [6, 8, 12, 13, 14, 15, 20, 22, 24, 27], "batch": [3, 4, 11, 12, 13, 29], "batch_shap": 4, "batch_siz": [1, 3, 4], "batchnorm": 4, "bay": 7, "bayesian": [5, 20, 26, 27], "becaus": [0, 1, 2, 3, 4, 5, 6, 8, 9, 12, 13, 14, 27, 28, 29, 30], "becom": [0, 1, 2, 5, 6, 7, 9, 12, 13, 24, 27, 28, 29, 30], "been": [0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 20, 21, 22, 27, 28, 30], "befor": [0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 27, 28, 30], "beforehand": [0, 24, 27], "begin": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 21, 24, 25, 27, 28, 29, 30], "behav": [1, 6, 13, 29], "behavior": [0, 1, 13, 27, 29, 30], "behaviour": [12, 30], "behind": [0, 1, 6, 8, 13, 27, 29], "being": [0, 1, 2, 3, 4, 5, 7, 8, 10, 11, 12, 13, 17, 24, 27, 28, 29, 30], "believ": [9, 21], "belong": [7, 8, 9, 13, 14, 29], "below": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 18, 21, 22, 24, 27, 28, 29, 30], "benchmark": 10, "benefici": [1, 13, 30], "benefit": [0, 1, 4, 11, 13, 20, 27, 29, 30], "bengio": [1, 26, 27, 28, 30], "benign": [1, 7], "besid": [4, 5, 29], "bessel": [5, 28], "best": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 18, 25, 27, 28, 29, 30], "beta": [1, 3, 10, 11, 13, 16, 17, 19, 27, 28, 29, 30], "beta1": 30, "beta2": 30, "beta_": [3, 13, 17, 28, 30], "beta_0": [1, 3, 13, 28], "beta_1": [1, 3, 10, 13, 28, 30], "beta_1x_i": 13, "beta_2": [3, 13, 30], "beta_3": 3, "beta_i": 3, "beta_j": [13, 28, 30], "beta_k": 13, "beta_linreg": [13, 30], "beta_m": 10, "beta_mg_m": 10, "beta_n": 3, "better": [0, 1, 2, 3, 4, 6, 9, 10, 11, 12, 13, 27, 28, 30], "between": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 27, 28, 29, 30], "beyond": [0, 1, 5, 6, 8, 13, 27, 28, 29, 30], "bf": [13, 14, 19, 21, 24, 29], "bg": 27, "bgd": [13, 30], "bia": [0, 1, 2, 3, 5, 8, 9, 10, 12, 13, 27, 28, 29, 30], "bias": [1, 2, 3, 5, 6, 9, 12, 19], "bibliographi": 22, "big": [0, 1, 2, 5, 6, 14, 19], "bigger": [1, 6, 28], "bigr": 12, "bike": 9, "bilbo": 27, "billion": [3, 12, 20], "bin": [7, 24], "binari": [0, 3, 5, 7, 9, 10, 12, 27], "binarycrossentropi": 4, "bind": 0, "binomi": [20, 24, 27], "binsboot": 6, "bioinformat": 0, "biolog": [1, 12], "bios1100": [20, 27], "bird": [0, 3], "birth": 27, "bishop": [26, 27], "bit": [1, 4, 21, 24, 27], "bitwis": 24, "bivari": 2, "bk": [13, 30], "bla": [21, 27], "black": [8, 9, 14], "block": [6, 10, 20, 21, 24, 27], "blog": 27, "blogpost": 4, "blue": [0, 3], "bm": [], "bmatrix": [0, 1, 3, 5, 7, 8, 11, 13, 21, 27, 28, 29, 30], "bmi": 1, "bodi": [0, 1, 4, 12], "bold": 1, "boldfac": [0, 5, 16, 28, 29], "boldsymbol": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 13, 14, 16, 17, 19, 22, 27, 29, 30], "boltzmann": [12, 20, 27], "book": [17, 22, 26, 27, 28], "book1": 26, "boolean": [4, 17], "boost": [1, 9, 20, 27], "boostrap": 10, "bootstrap": [1, 13, 19, 20, 22, 27, 30], "borrow": 27, "boston_dataset": [], "bot": 8, "both": [0, 1, 4, 5, 6, 8, 9, 10, 13, 14, 15, 16, 17, 20, 21, 22, 24, 25, 27, 28, 29, 30], "bottl": 7, "bound": [8, 12, 30], "boundari": [2, 4, 8, 11, 12], "box": [4, 9], "boyd": [8, 13, 29], "bracket": [4, 24], "brain": [1, 7, 12], "branch": [9, 27], "break": [0, 4, 6, 11, 14, 27], "breast": [5, 7, 11], "breviti": [13, 30], "brew": [0, 20, 22, 27], "brg": 8, "brief": [22, 28], "briefli": [0, 16, 27], "bring": [0, 5, 6, 10, 28, 30], "britt": [25, 27], "broad": 0, "broadli": 27, "brought": [13, 20, 27], "brownle": 4, "browser": [15, 27], "brute": [3, 5, 11, 28], "buffer_s": 4, "bui": 4, "build": [0, 4, 5, 6, 10, 16, 21, 24, 27], "built": [1, 3, 4, 6], "bunch": 11, "busi": [], "byte": [21, 27], "c": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 23, 24, 25, 26, 28, 29, 30], "c1": [8, 11], "c2": [8, 11], "c_": [8, 9, 10, 13, 24, 29, 30], "c_0": 24, "c_1": 12, "c_2": 12, "c_3": 12, "c_4": 12, "c_i": [12, 13, 30], "c_k": 24, "ca": [1, 27], "cach": 10, "cal": [0, 8, 10, 12, 13, 29, 30], "calcul": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 16, 19, 21, 24, 27, 30], "california": 22, "call": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29, 30], "calor": [0, 28], "cambridg": [13, 26, 29], "can": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29], "cancel": [0, 13, 27, 28], "cancer": [5, 10], "cancerpd": 7, "candid": [8, 9, 10, 30], "cannot": [0, 1, 4, 5, 6, 7, 8, 9, 22, 24, 28, 29], "canopi": [0, 20, 22, 27], "canva": [15, 16, 22, 27], "cap": 5, "capabl": [0, 1, 8, 13, 20, 27], "capac": [2, 25], "capita": [], "caption": 22, "captur": [4, 11, 12, 27], "car": [3, 4], "card": [0, 7, 27], "cardin": 1, "care": [11, 15], "carefulli": [13, 30], "carlo": [0, 6, 20, 24, 26, 27], "carri": [2, 6, 7, 22], "cart": 10, "case": [0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 20, 21, 22, 27, 30], "casella": 26, "cast": 1, "cat": [3, 4], "catch": 0, "categor": [0, 1, 3, 9, 11, 27], "categori": [0, 1, 3, 7, 10, 12, 14, 27], "categorical_crossentropi": [1, 3], "caus": [0, 5, 6, 24, 27, 28, 29, 30], "causal": 0, "causat": [0, 27], "cax": 1, "cb": [6, 27], "cbar": 1, "cc": [0, 1, 5, 13, 27, 28, 29], "ccc": [5, 12, 29], "cdf": 24, "cdot": [0, 2, 6, 12, 13, 14, 21, 24, 27, 29, 30], "celebr": [13, 29], "cell": 4, "center": [0, 1, 6, 7, 8, 9, 11, 14, 18, 22, 24, 27, 28, 30], "central": [0, 3, 5, 6, 8, 16, 21, 27, 28], "centroid": [14, 24], "centroid_differ": 14, "centuri": 3, "certain": [0, 3, 6, 7, 9, 24, 27, 28], "cg": 13, "cha": [], "chain": [0, 1, 13, 20, 24, 27], "challeng": 15, "chanc": [1, 5, 13, 24, 30], "chang": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 15, 16, 19, 21, 22, 24, 27, 28, 29, 30], "channel": 3, "chapter": [0, 6, 10, 11, 16, 17, 19, 21, 22, 26, 27, 28, 29, 30], "chapter3": [0, 22], "charact": [0, 3, 5, 27, 28, 29], "character": [8, 9, 10, 12, 24], "characterist": [0, 1, 3, 10, 13, 27, 30], "charg": [0, 27], "charl": [], "chase": 4, "chatgpt": [15, 22], "chd": 7, "chddata": 7, "cheap": [5, 28, 29], "cheaper": [1, 13, 30], "check": [1, 3, 4, 5, 11, 13, 15, 16, 19, 21, 27, 30], "checkmark": 3, "checkpoint": 4, "checkpoint_dir": 4, "checkpoint_prefix": 4, "chen": 10, "cheng": 28, "chiaramont": 2, "childcar": 16, "children": 16, "choic": [0, 1, 2, 3, 4, 6, 9, 12, 13, 14, 21, 27, 28, 29, 30], "choleski": [5, 21, 28, 29], "choos": [2, 3, 6, 9, 10, 11, 13, 14, 15, 18, 19, 22, 29], "chosen": [0, 1, 2, 6, 8, 9, 10, 13, 16, 24, 27, 29, 30], "chosen_datapoint": 1, "christian": 26, "christoph": [26, 27], "cifar": 3, "cifar10": 3, "circ": [1, 12, 30], "circl": [0, 8, 12, 28, 30], "circuit": 3, "circumfer": 9, "circumv": [1, 5, 13, 28, 29, 30], "cite": 22, "ckpt": 4, "clariti": 24, "class": [0, 1, 3, 4, 6, 7, 8, 9, 11, 12, 13, 24, 27], "class_nam": [3, 9], "class_val": 9, "class_valu": 9, "classic": [7, 9, 13, 30], "classif": [0, 3, 5, 6, 7, 8, 11, 12, 20, 22, 26, 27, 28], "classifi": [0, 1, 4, 7, 9, 10, 11, 27], "classificaton": 1, "classifii": 10, "clean": 1, "clear": [1, 5, 10, 12, 13, 30], "clearli": [0, 3, 5, 6, 7, 8, 24, 28, 29], "clever": [1, 10], "clf": [0, 6, 8, 9, 10, 27, 28], "clf3": 0, "clf_lasso": 6, "clf_ridg": 6, "cli": 15, "clip": [3, 24], "clone": [15, 25], "close": [0, 1, 2, 4, 6, 8, 9, 11, 12, 13, 14, 18, 24, 26, 27, 29, 30], "closer": [3, 5, 13, 28, 29], "closest": [8, 11, 13, 14], "closur": [20, 27], "cloud": [20, 27], "cluster": [0, 1, 4, 6, 11, 20, 27], "cluster_label": 14, "cm": [1, 2, 3, 6, 8, 13, 29, 30], "cmap": [0, 1, 2, 3, 4, 6, 8, 9, 10, 27], "cmap_arg": 6, "cmd": [9, 15], "cn_": 24, "cnn": 12, "cnn_kera": 3, "cntk": [20, 27], "co": [0, 2, 3, 6, 9, 13, 27], "code": [0, 3, 4, 6, 7, 8, 18, 20, 21, 24, 26], "coef": [0, 27], "coef0": 8, "coef_": [0, 5, 6, 8, 9, 13, 16, 27, 28, 29, 30], "coeff": 5, "coeffici": [0, 3, 5, 6, 7, 8, 9, 13, 18, 21, 27, 28, 30], "coerc": [0, 6, 27], "coin": [10, 24], "coin_toss": 10, "col": [0, 11, 27, 28], "colab": [20, 27], "cold": 9, "colinear": [], "collabor": 22, "collaps": 8, "collect": [2, 6, 10, 11, 17, 20, 24, 26, 27], "collinear": [5, 28, 29], "color": [0, 3, 4, 6, 8, 9, 10, 24, 30], "color_channel": 3, "color_cod": 6, "colorbar": [1, 6], "colsample_bytre": 10, "colsaobject": 10, "column": [0, 1, 2, 5, 6, 7, 8, 9, 11, 12, 16, 17, 18, 21, 27, 28, 29, 30], "columntransform": 9, "com": [4, 6, 15, 16, 20, 22, 26, 27, 29, 30], "combin": [1, 2, 5, 6, 7, 10, 15, 18, 24, 30], "come": [0, 1, 3, 4, 5, 12, 13, 14, 15, 27, 28, 29, 30], "command": [0, 1, 15], "comment": [0, 4, 5, 6, 22], "commerci": [0, 20, 22, 27], "commit": 15, "commod": [0, 27], "common": [0, 1, 3, 5, 6, 7, 9, 11, 13, 14, 16, 22, 24, 27, 28, 29, 30], "commonli": [0, 1, 4, 6, 7, 9, 13, 14, 28, 30], "commun": [0, 12, 15, 22], "commut": 3, "commutatitav": 3, "compact": [0, 1, 3, 5, 6, 7, 9, 11, 12, 13, 14, 27, 28], "compair": 0, "compar": [0, 3, 4, 5, 6, 11, 13, 18, 21, 22, 27, 28, 29, 30], "comparison": [2, 4, 13], "compat": 7, "compet": 0, "competit": 10, "compil": [0, 1, 3, 4, 13, 20, 21, 27], "complet": [0, 2, 3, 4, 9, 12, 15, 16, 17, 18, 19, 27], "completenn": 12, "complex": [1, 5, 8, 9, 11, 12, 13, 16, 19, 27, 29, 30], "complic": [0, 1, 9, 13, 22, 27, 29, 30], "compoment": 28, "compon": [0, 1, 3, 4, 5, 6, 7, 9, 14, 16, 20, 27, 28, 29], "components_": 11, "compos": [9, 12, 13, 14, 20, 27], "compphys": [0, 6, 16, 20, 22, 23, 25, 26, 27, 28, 29], "compress": [0, 27, 28], "compris": 6, "compromis": [5, 28, 29], "compulsori": [20, 27], "comput": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 16, 17, 18, 20, 21, 22, 23, 24, 26, 27, 28, 29], "computation": [0, 3, 6, 9, 13, 24, 27, 29, 30], "computationalscienceuio": 27, "computerlab": 22, "concaten": [2, 4, 6, 14], "concav": [1, 13, 28, 29], "concentr": 10, "concept": [0, 2, 20, 27, 28], "conceptu": [12, 13, 29], "concern": [0, 1, 4, 7, 27, 29], "concic": 27, "conclud": [0, 5, 13], "conclus": 1, "cond": 2, "conda": [0, 1, 20, 22, 27], "condis": 28, "condit": [0, 2, 4, 5, 6, 8, 9, 11, 13, 24, 27, 28, 30], "conduct": 20, "condwav": 2, "confid": [0, 5, 6, 7, 8, 19, 27, 28], "configur": 3, "confirm": [5, 12], "confus": [5, 6, 7, 10, 21, 28], "confusion_matrix": 9, "congruenti": 24, "conjug": [4, 8], "conjugaci": 13, "conjunct": 3, "connect": [0, 1, 3, 4, 9, 11, 12, 13, 21, 27, 28, 29, 30], "consequ": [5, 6, 8, 10, 12, 13, 28, 29, 30], "conserv": [5, 14, 28, 29], "consid": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 16, 21, 22, 24, 27, 28, 29, 30], "consider": [0, 1, 5, 13, 27, 28, 29], "consist": [1, 2, 3, 4, 6, 12, 13, 22, 24, 28, 29, 30], "constant": [0, 2, 4, 5, 6, 8, 12, 13, 16, 18, 24, 27, 28, 29, 30], "constitu": [0, 27], "constitut": [2, 6], "constrain": [1, 3, 5, 7, 11, 29], "constraint": [5, 6, 8, 13, 28, 29, 30], "construct": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 21, 24, 27, 28], "contact": [0, 27], "contain": [0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 18, 19, 21, 22, 24, 26, 27, 28, 29, 30], "contemporari": 27, "content": [1, 15, 20, 21, 27, 29, 30], "context": [6, 10, 13, 22, 29, 30], "contigu": 21, "contin": 19, "continu": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 19, 21, 22, 24, 27, 28, 29, 30], "contour": [9, 10, 13], "contourf": [8, 9, 10], "contrast": [1, 4, 9, 10, 12, 27, 30], "contribut": [0, 3, 5, 13, 18, 24, 27, 28, 29, 30], "contributor": [0, 22], "control": [0, 1, 3, 9, 13, 15, 20, 27, 30], "conv": [3, 4], "conv2d": [3, 4], "conv2dtranspos": 4, "convei": 27, "conveni": [5, 6, 12, 13, 21, 22, 27, 29, 30], "convent": [12, 28], "converg": [1, 2, 4, 5, 8, 13, 14, 18, 28, 29], "convergencewarn": [], "convers": 30, "convert": [0, 1, 4, 5, 9, 11, 13, 21, 27, 28, 29], "converttomatrix": 4, "convex": [4, 5, 7, 28, 30], "convinc": [13, 29], "convolut": [1, 4, 20, 27], "cool": [4, 9], "coolwarm": 6, "coordin": [5, 12, 14, 28, 29, 30], "coorel": [], "copi": [0, 1, 14, 15, 28], "core": 10, "corel": 27, "coronari": 7, "corr": [5, 7, 11, 28], "correalt": [11, 20], "correct": [0, 1, 2, 3, 4, 5, 7, 13, 15, 21, 24, 27, 28, 29, 30], "correctli": [1, 2, 6, 7, 10, 18, 19, 22], "correl": [0, 1, 3, 5, 6, 7, 10, 12, 13, 20, 24, 27, 29, 30], "correlation_matrix": [5, 7, 11, 28], "correspond": [0, 3, 5, 6, 8, 9, 11, 12, 20, 21, 22, 24, 27, 28, 29], "cortex": 12, "cosin": [3, 6], "cost": [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 16, 17, 18, 19, 22, 27], "cost_deep_grad": 2, "cost_funct": 2, "cost_function_deep": 2, "cost_function_deep_grad": 2, "cost_function_grad": 2, "cost_grad": 2, "cost_histori": [], "cost_ol": [], "cost_ridg": [], "cost_sum": 2, "costol": [13, 30], "could": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 21, 22, 24, 27, 28, 29, 30], "coulomb": [0, 27], "count": [0, 9, 15, 23, 24, 25, 27], "counterpart": 27, "countor": 13, "coupl": [4, 5, 6], "cours": [0, 1, 3, 5, 11, 15, 16, 17, 22, 25, 28], "coursework": 15, "courvil": [26, 27, 28, 30], "cov": [5, 6, 11, 21, 24, 27, 28], "cov_xi": [5, 11, 28], "cov_xx": [5, 11, 28], "cov_yi": [5, 11, 28], "covari": [0, 7, 20, 21, 27, 29], "covariance_matrix": [5, 11, 14], "cover": [0, 5, 20, 25, 26, 28, 29], "covert": [0, 27], "covxi": 24, "covxx": 24, "covxz": 24, "covyi": 24, "covyz": 24, "covzz": 24, "cpu": 1, "craft": 3, "creat": [1, 3, 4, 5, 9, 10, 11, 12, 15, 18, 19, 20, 27, 30], "create_biases_and_weight": 1, "create_convolutional_neural_network_kera": 3, "create_neural_network_kera": 1, "create_x": [5, 11], "credit": [0, 7, 25, 27], "crim": [], "crime": [], "criteria": [0, 4, 9, 10, 14, 24, 27], "criterion": [9, 10, 13, 18, 29, 30], "critic": [6, 22, 28], "critiqu": 22, "cross": [0, 1, 3, 7, 9, 10, 13, 15, 20, 24, 27, 28, 29, 30], "cross_entropi": 4, "cross_val_scor": 6, "cross_valid": [7, 10], "crossvalid": 6, "crucial": [1, 24, 30], "cs231": 3, "csr_matrix": [21, 27], "csv": [0, 4, 6, 7, 9], "ctnk": 1, "cubic": 0, "cumbersom": 5, "cumsum": [10, 11, 27], "cumul": [7, 10, 24, 30], "cumulative_heads_ratio": 10, "cup": 5, "current": [1, 2, 3, 4, 13, 14, 15, 16, 26, 29, 30], "curs": [0, 28], "curv": [6, 7, 10, 12, 22], "curvatur": [13, 29, 30], "custom": [6, 14], "custom_cmap": [9, 10], "custom_cmap2": [9, 10], "cutpoint": 9, "cv": [6, 7, 10], "cvxbook": [13, 29], "cvxopt": [5, 8, 28], "cycl": [1, 12], "d": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 19, 21, 24, 25, 27, 28, 29, 30], "d2_g_t": 2, "d_f": [13, 29], "d_g_t": 2, "d_net_out": 2, "da": 3, "dagger": [5, 21, 28, 29], "dai": [1, 9, 20], "damp": 3, "darget": 9, "darkr": 24, "dat": [0, 27], "dat_id": [0, 6, 7, 9, 27], "data": [2, 4, 5, 8, 10, 12, 13, 14, 16, 19, 21, 22, 26, 29, 30], "data1": 14, "data2": 14, "data3": 14, "data4": 14, "data_id": [0, 6, 7, 9, 27], "data_indic": 1, "data_panda": 27, "data_path": [0, 6, 7, 9, 27], "databas": 1, "datafil": [0, 6, 7, 9, 27], "datafram": [0, 4, 5, 7, 9, 11, 27, 28], "datapoint": [1, 5, 6, 7, 11, 13, 16, 29, 30], "datasci": [15, 16], "dataset": [0, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 22, 27, 29, 30], "datatyp": 4, "date": [15, 18, 22, 27, 28, 29, 30], "daughter": 10, "david": 26, "dbh": 1, "dbo": 1, "dcomposit": 21, "ddot": 2, "dead": 1, "deadlin": 15, "deal": [0, 1, 3, 5, 6, 8, 11, 13, 14, 19, 21, 24, 27, 28, 29, 30], "dealt": 0, "debt": 7, "debug": [0, 5, 6, 28, 29, 30], "decad": [0, 3, 30], "decai": [0, 13, 24, 27], "decemb": [25, 27], "decent": 10, "decid": [0, 2, 3, 5, 6, 9, 18, 28, 29, 30], "decim": [0, 27], "decis": [0, 1, 8, 11, 20, 26, 27], "decision_funct": 8, "decision_tre": 9, "decisiontreeclassifi": [9, 10], "decisiontreeregressor": [0, 9, 10], "declar": [0, 4, 21, 27], "decompos": [5, 6, 21, 28, 29], "decomposit": [0, 6, 12, 27], "decompost": [5, 28, 29], "deconvolut": 3, "decorrel": [10, 13, 30], "decreas": [1, 2, 4, 5, 6, 10, 11, 13, 19, 29, 30], "deduc": [0, 27], "deep": [3, 7, 12, 13, 20, 26, 28, 29, 30], "deep_neural_network": 2, "deep_param": 2, "deep_tree_clf": [9, 10], "deep_tree_clf1": 9, "deep_tree_clf2": 9, "deepen": [5, 20, 27], "deeper": [0, 3, 4, 27], "deeplearningbook": [26, 27, 29, 30], "deer": 3, "def": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 24, 27, 28, 29, 30], "def_covari": 24, "default": [0, 1, 2, 4, 6, 7, 21, 27, 28], "default_tim": 4, "defect": [5, 28, 29], "defici": [5, 28, 29], "defin": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 28, 29, 30], "definit": [1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 21, 24, 28, 29, 30], "defint": 24, "degre": [3, 5, 6, 8, 9, 10, 11, 15, 16, 19, 22, 24, 27, 29, 30], "deisenroth": 28, "del": 1, "delet": [6, 15], "delimit": 4, "deliv": [15, 23, 27], "delta": [0, 2, 3, 6, 8, 12, 13, 14, 27, 30], "delta_": [1, 21], "delta_0": 3, "delta_1": 3, "delta_2": 3, "delta_3": 3, "delta_4": 3, "delta_5": 3, "delta_h": [0, 1, 27], "delta_j": [3, 12], "delta_k": 12, "delta_l": [1, 3], "delta_momentum": [13, 30], "delta_n": [0, 3, 27], "delug": 20, "delv": 0, "demand": [13, 29], "demonstr": [0, 3, 5, 6, 7, 11, 12, 20, 27, 28, 29], "den": 4, "denomin": [1, 5, 30], "denot": [1, 2, 6, 7, 13, 24, 29, 30], "dens": [1, 3, 4], "densiti": [0, 2, 6, 24], "depart": [25, 27, 28, 29, 30], "depend": [0, 1, 2, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 20, 21, 22, 24, 27, 28, 29, 30], "depict": 24, "deploy": [0, 20, 22, 27], "depth": [0, 3, 9, 10, 21], "deriv": [0, 1, 2, 6, 7, 8, 10, 11, 13, 18, 20, 22, 27], "derivati": 13, "derivative_fn": 13, "descend": [5, 9, 11, 28, 29], "descent": [0, 1, 3, 7, 8, 12, 27, 28], "describ": [0, 2, 4, 5, 6, 8, 10, 11, 12, 13, 19, 21, 22, 27, 30], "descript": [0, 8, 9, 22, 27], "design": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 17, 18, 22, 27, 29, 30], "designmatrix": [0, 27], "desir": [0, 2, 4, 5, 13, 14, 27, 28, 29, 30], "desktop": 15, "despit": [1, 12], "destroi": 21, "det": [5, 21, 28, 29], "detail": [0, 6, 11, 13, 14, 18, 21, 22, 28, 29], "detect": [3, 8, 12], "determin": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 18, 21, 24, 27, 28, 29, 30], "determinist": [7, 13, 24, 29, 30], "dev": [1, 22], "develop": [0, 3, 5, 8, 10, 11, 12, 20, 21, 22, 27, 28], "deviat": [0, 1, 2, 4, 5, 6, 17, 18, 19, 22, 24, 27, 28, 30], "devis": 12, "df": [4, 8, 11, 13, 27], "df1": 27, "di": [], "diag": [5, 8, 28, 29, 30], "diagnost": [1, 10], "diagon": [0, 5, 7, 13, 18, 19, 21, 24, 27, 28, 29, 30], "diagonaliz": [5, 28, 29], "diagram": 10, "diagsvd": 6, "dice": [6, 24], "dict": [6, 8], "dictionari": [], "did": [0, 1, 5, 6, 7, 10, 11, 14, 16, 22, 27], "die": 1, "diff": 2, "diff1": 2, "diff2": 2, "diff_ag": 2, "diffeent": 8, "differ": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29], "differenti": [0, 3, 16, 20, 21, 27, 28, 29], "difficult": [0, 1, 6, 10, 13, 24, 27, 30], "difficulti": [0, 1, 13, 27, 29], "diffonedim": 2, "digit": [0, 1, 3, 4, 6, 25, 27], "dilemma": [13, 30], "dilut": 1, "dim": [4, 11, 14, 21], "dimens": [0, 1, 2, 3, 4, 5, 8, 11, 14, 16, 21, 27, 28, 29, 30], "dimension": [0, 4, 5, 6, 9, 11, 13, 14, 19, 20, 21, 22, 27, 28, 29, 30], "dimensionless": [0, 3, 27], "diment": 21, "diminish": 30, "dimnsion": 4, "diod": 3, "direct": [0, 1, 2, 4, 11, 12, 13, 14, 27, 28, 29, 30], "directli": [1, 4, 5, 6, 18, 24, 28, 29], "disadvantag": [0, 27, 30], "disappear": [3, 6], "disc_loss": 4, "disc_tap": 4, "discard": [6, 11], "disciplin": [0, 3, 12], "disclaim": 24, "discord": 27, "discourag": [13, 15, 29], "discov": [0, 27], "discover": 5, "discret": [1, 3, 5, 7, 13, 30], "discrimin": [4, 7, 10, 11], "discriminator_loss": 4, "discriminator_loss_list": 4, "discriminator_model": 4, "discriminator_optim": 4, "discuss": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29, 30], "diseas": 7, "disguis": [6, 28, 30], "disord": [1, 7], "displai": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 22, 24, 27, 28], "displaystyl": [0, 5, 17, 27, 28, 29, 30], "disregard": [0, 27], "dissimilar": [11, 14], "dist": 14, "distanc": [8, 9, 11, 14, 24], "distance_list": 9, "distinct": [3, 7, 8, 9, 10, 14], "distinctli": 8, "distinguish": [0, 4, 7, 8, 24, 27], "distplot": [], "distribut": [0, 1, 4, 6, 7, 10, 11, 13, 14, 18, 19, 20, 21, 22, 27, 28, 29, 30], "distrubut": [0, 20, 22, 27], "dive": [0, 8, 21, 27], "diverg": [1, 13, 29, 30], "divid": [0, 1, 3, 5, 6, 7, 8, 9, 11, 12, 18, 24, 27, 28, 30], "divis": [6, 8, 9, 13, 18, 21, 24, 30], "dna": 7, "dnn": [0, 1, 2, 4, 12, 27], "dnn1": 4, "dnn2_gru2": 4, "dnn_kera": 1, "dnn_model": 1, "dnn_numpi": 1, "dnn_scikit": [0, 1, 27], "do": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 22, 27, 28, 29], "doc": [0, 15, 16, 20, 22, 23, 25, 26, 27], "document": [4, 13, 15], "doe": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 17, 18, 21, 22, 24, 27, 30], "doesn": [3, 9, 12, 27], "dog": [1, 3, 4], "domain": [5, 8, 13, 22, 29], "domin": [0, 27], "don": [0, 1, 3, 5, 6, 8, 11, 13, 15, 16, 20, 22, 27, 28, 30], "done": [0, 2, 3, 4, 5, 6, 9, 10, 11, 13, 16, 21, 22, 27, 28, 29, 30], "dot": [0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 18, 21, 22, 24, 27, 28, 29, 30], "doubl": [3, 4, 16, 21, 27], "doubli": 1, "doubt": 22, "down": [0, 3, 6, 9, 11, 12, 13, 29, 30], "download": [0, 1, 3, 5, 6, 15, 21, 26, 27], "downsampl": 3, "dozen": 1, "dq": 6, "drag": [13, 30], "dramat": 11, "drastic": 4, "draw": [4, 6, 10, 13, 29], "drawback": [0, 1, 3, 13, 28, 29, 30], "drawn": [1, 4, 6, 7, 11, 24, 27], "drive": [3, 4], "driven": 3, "drop": [0, 1, 5, 6, 11, 13, 24, 27, 28, 29, 30], "dropna": [0, 6, 27], "dropout": 4, "dt": [2, 3, 13, 24, 30], "dtype": [0, 1, 3, 4, 14, 21, 27], "dub": [0, 27], "due": [1, 2, 5, 6, 8, 10, 12, 13, 18, 25, 27, 28, 29, 30], "dummi": [], "dure": [0, 1, 3, 4, 8, 9, 11, 20, 22, 27, 30], "dwell": [], "dwh": 1, "dwo": 1, "dx": [2, 3, 8, 24], "dx_1": 24, "dx_1p": 6, "dx_2p": 6, "dx_mp": 6, "dx_n": 24, "dxp": 6, "dy": [1, 8, 24], "dynam": 4, "dz": 8, "e": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 24, 25, 27, 28, 29, 30], "e_": [0, 2, 27], "each": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 23, 24, 25, 27, 28, 29, 30], "eapprox": [0, 27], "earli": [1, 13, 30], "earlier": [0, 5, 7, 8, 9, 11, 12, 13, 27, 28, 30], "earthexplor": 6, "eas": [6, 9, 14], "easi": [0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 20, 21, 27, 28, 29, 30], "easier": [5, 6, 8, 9, 13, 15, 22, 24, 27, 28, 29], "easiest": [13, 18], "easili": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 21, 22, 27, 28, 29, 30], "eastern": [25, 27], "ebind": [0, 27], "eblock": 9, "econometr": 27, "economi": 5, "ecosystem": [20, 27], "ect": 23, "edg": 3, "edgecolor": 6, "editor": 15, "edu": [13, 22, 29], "educ": [0, 22, 27], "eff": 24, "effect": [1, 4, 10, 13, 16, 17, 18, 24, 30], "effic": 1, "effici": [0, 3, 10, 13, 20, 21, 24, 27, 30], "efron": 6, "egrad": 13, "eig": [5, 11, 13, 21, 24, 27, 28, 29, 30], "eigen": 24, "eigenpair": [5, 11, 28, 29], "eigenvalu": [0, 5, 8, 11, 13, 21, 27, 28, 29, 30], "eigenvector": [5, 11, 13, 28, 29], "eight": [21, 27], "eigval": [21, 24, 27], "eigvalu": [11, 13, 29, 30], "eigvec": [21, 24, 27], "eigvector": [11, 13, 29, 30], "eir": [25, 27], "eispack": [21, 27], "either": [1, 5, 6, 7, 8, 9, 10, 11, 13, 18, 22, 24, 27, 28, 29, 30], "eivind": 25, "eivinsto": 25, "ekstr\u00f8m": 4, "elabor": 24, "elarn": 3, "electr": [0, 3, 12, 27], "electron": 27, "eleg": 11, "element": [1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 19, 20, 21, 22, 26, 28, 30], "elementari": [10, 13, 21], "elementwis": [3, 13], "elementwise_grad": [2, 13], "elessar": 27, "elif": 14, "elim": 21, "elimin": [3, 8], "elin": [25, 27], "ell_": 30, "ellipsi": 16, "els": [1, 3, 4, 7, 9, 12, 13, 16, 21], "elu": 1, "elus": [0, 27], "email": [23, 25, 27], "embed": [0, 11, 28], "embodi": [6, 22], "emit": 24, "emner": 26, "emphas": [0, 10, 20, 27], "emphasi": [0, 20, 26, 27], "empir": [1, 11, 24], "emploi": [0, 1, 5, 6, 11, 13, 24, 27, 28, 29], "employ": 0, "empti": [6, 10, 15], "emul": 12, "en": [20, 22, 26], "enabl": 11, "enbodi": 6, "encod": [0, 3, 5, 9, 11, 14, 27, 28, 29], "encompass": [0, 22, 24], "encount": [0, 1, 5, 7, 13, 15, 22, 24, 27, 28, 29, 30], "encourag": [15, 22], "end": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 24, 25, 27, 28, 29, 30], "endpoint": [3, 6], "energi": [0, 4, 6], "enforc": 12, "eng": 26, "engin": [0, 1, 3, 4, 20, 27], "english": 22, "enocurag": 22, "enorm": 3, "enough": [0, 6, 13, 27, 29, 30], "ensembl": [1, 9, 27], "ensur": [0, 1, 2, 3, 5, 6, 11, 13, 18, 24, 28, 29, 30], "entail": 27, "enter": [5, 6, 28, 29, 30], "enthought": [0, 20, 22, 27], "entir": [1, 3, 7, 9, 20, 24, 27, 30], "entiti": [9, 12, 21, 27], "entri": [0, 5, 8, 11, 12, 21, 27, 28, 30], "entropi": [1, 3, 7, 10, 13, 27, 29, 30], "enumer": [0, 1, 2, 3, 4, 6, 8, 27, 28, 30], "env": 24, "environ": [2, 20, 22, 27], "environemnt": 15, "eo": [0, 6], "eol": 0, "eosfit": 0, "epoch": [0, 1, 3, 4, 12, 13, 27, 30], "epsilon": [0, 5, 6, 7, 13, 22, 27, 28, 29, 30], "epsilon_": [0, 27], "epsilon_0": [0, 27], "epsilon_1": [0, 27], "epsilon_2": [0, 27], "epsilon_i": [0, 27, 28], "eq": [3, 13, 14, 21, 24, 29], "eqnarrai": [3, 5, 6], "equal": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 16, 18, 21, 22, 24, 27, 28, 29, 30], "equat": [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 17, 19, 21, 24, 27, 30], "equilibrium": [2, 12], "equiv": [3, 13, 21, 24, 29, 30], "equival": [0, 1, 5, 7, 8, 11, 13, 20, 21, 27, 28, 29, 30], "erf": 24, "eriador": 27, "err": [0, 10], "err_": 6, "err_sqr": 2, "errat": [13, 29, 30], "erron": 2, "error": [1, 2, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 30], "error_estimate_corr_tim": 24, "error_hidden": 1, "error_output": 1, "escap": [13, 29, 30], "especi": [1, 3, 9, 12, 13, 15, 18, 22, 30], "essenti": [0, 5, 6, 9, 10, 12, 14, 15, 22, 24, 28, 29], "establish": [0, 6, 10, 11, 16, 22], "estim": [0, 1, 5, 6, 7, 10, 11, 13, 19, 20, 24, 27, 28, 29, 30], "estimated_mse_fold": 6, "estimated_mse_kfold": 6, "estimated_mse_sklearn": 6, "et": [0, 2, 4, 16, 17, 26, 27, 28, 29], "eta": [0, 1, 3, 8, 12, 13, 18, 27, 29, 30], "eta0": [8, 13], "eta_": [13, 30], "eta_t": [13, 30], "eta_v": [0, 1, 3, 27], "etc": [0, 1, 3, 5, 7, 8, 9, 11, 12, 13, 14, 20, 21, 22, 24, 28, 29, 30], "ethic": 20, "euclidean": [0, 14, 28, 30], "evalu": [0, 2, 3, 4, 5, 6, 9, 13, 15, 16, 17, 19, 22, 24, 27, 28, 29, 30], "evalut": [13, 22], "even": [0, 1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 20, 21, 24, 27, 28, 29, 30], "evenli": 4, "event": [5, 7, 10, 24], "eventu": [0, 5, 6, 11, 12, 13, 22, 25, 28, 29, 30], "everi": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15, 20, 24, 25, 27, 28, 29, 30], "everyth": [4, 12, 16, 18], "everywher": [4, 13, 29], "evolv": 0, "exact": [0, 5, 11, 12, 13, 21, 24, 27, 28], "exactli": [0, 3, 4, 6, 12, 18, 20, 28, 30], "exam": 27, "examin": 6, "exampl": [0, 5, 11, 12, 13, 15, 16, 18, 20, 21, 22, 24, 26], "exce": [1, 12, 13, 30], "excel": [0, 1, 4, 5, 10, 22, 27, 28], "except": [3, 4, 6, 8, 9, 21], "excess": [0, 27], "excit": 0, "exclud": [1, 6, 12, 28, 30], "exclus": [0, 1, 3, 6, 24, 27], "execut": [2, 5, 13, 15, 28, 29], "exemplifi": [13, 30], "exercic": [25, 27], "exercis": [5, 20, 22, 23, 25, 27, 29, 30], "exhaust": 6, "exhibit": [0, 5, 6, 8, 27, 28], "exist": [0, 1, 2, 3, 5, 6, 7, 8, 9, 13, 19, 21, 22, 27, 29], "exit": [5, 21, 28, 29], "exp": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 16, 17, 19, 24, 28, 29, 30], "exp_term": 1, "expand": [5, 7, 11, 13, 29], "expans": [0, 3, 5, 8, 10, 12, 13, 27, 28, 29], "expect": [0, 1, 5, 6, 7, 11, 12, 13, 15, 18, 20, 22, 27, 28, 30], "expectation_value_of_h_wrt_p": 24, "expens": [6, 10, 13, 16, 29, 30], "experi": [0, 1, 6, 8, 13, 15, 20, 22, 27, 28, 29], "experiment": [0, 4, 6, 9, 24, 27], "expert": [1, 9], "explain": [0, 6, 9, 10, 11, 13, 16, 19, 22, 27, 29, 30], "explained_variance_ratio_": 11, "explanatori": [0, 27], "explicit": [0, 3, 6, 13, 21, 22, 27, 28, 29, 30], "explicitli": [0, 4], "explod": 1, "exploit": [0, 3, 12, 13, 27], "explor": [1, 4, 6, 8, 13, 18, 20, 22, 27, 29, 30], "expon": 1, "exponenti": [0, 1, 5, 6, 10, 13, 24, 27, 29, 30], "export": [9, 15, 16], "export_graphviz": 9, "export_text": 9, "exporttext": 9, "expos": 20, "express": [0, 2, 3, 5, 6, 7, 10, 12, 13, 18, 21, 22, 24, 27, 29, 30], "exptmean": 24, "exptvari": 24, "extend": [0, 2, 7, 11, 13, 20, 27, 30], "extens": [0, 12, 15, 20, 27], "extent": [0, 1, 6, 26], "extern": [3, 6, 9], "extra": [1, 3, 5, 15, 25, 27, 28, 29], "extract": [0, 3, 5, 6, 7, 8, 11, 13, 16, 17, 21, 27, 28], "extrapol": [0, 27], "extrem": [0, 1, 4, 5, 6, 7, 8, 9, 13, 15, 16, 21, 28, 29, 30], "extremum": [13, 29], "extrins": 11, "ey": [0, 5, 6, 13, 14, 18, 21, 27, 28, 29, 30], "f": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 27, 28, 29, 30], "f1": 13, "f11": [0, 27], "f12": [0, 27], "f13": [0, 27], "f1_grad": 13, "f1d": 13, "f2": 13, "f2_grad_x1": 13, "f2_grad_x1_analyt": 13, "f2_grad_x2": 13, "f2_grad_x2_analyt": 13, "f3": 13, "f3_grad": 13, "f3_grad_analyt": 13, "f4": 13, "f4_grad": 13, "f4_grad_analyt": 13, "f5": 13, "f5_grad": 13, "f6": 13, "f6_for": 13, "f6_for_grad": 13, "f6_grad_analyt": 13, "f6_while": 13, "f6_while_grad": 13, "f7": 13, "f7_grad": 13, "f7_grad_analyt": 13, "f8": 13, "f8_grad": 13, "f9": [0, 13, 27], "f9_altern": 13, "f9_alternative_grad": 13, "f9_grad": 13, "f_": 10, "f_0": [3, 10], "f_1": [10, 13, 29], "f_2": [12, 13, 29], "f_3": 12, "f_d": 24, "f_grad": 13, "f_grad_analyt": 13, "f_i": [0, 6, 12, 16], "f_m": [3, 10], "f_n": 3, "f_vec": 2, "face": [13, 27, 29], "facecolor": [6, 8, 24], "facil": [0, 20], "facilit": 12, "fact": [0, 1, 3, 5, 9, 11, 12, 13, 27, 28, 29, 30], "factor": [0, 1, 3, 5, 6, 9, 10, 11, 13, 21, 24, 27, 28, 29], "factori": 13, "fade": 6, "fafab0": [9, 10], "fail": [0, 6, 13, 25, 27, 29], "failur": 7, "fairli": [1, 2, 18, 24, 30], "faisal": [16, 28], "fake": 4, "fake_loss": 4, "fake_output": 4, "fall": [8, 9, 23], "fals": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 14, 16, 17, 21, 27, 28, 29, 30], "famili": [0, 7, 8, 24, 28, 30], "familiar": [0, 3, 5, 6, 8, 15, 20, 21, 22, 24, 27], "famou": [6, 12], "far": [0, 3, 4, 5, 6, 8, 11, 12, 13, 14, 16, 27, 28, 29], "fashion": [0, 9, 10, 27], "fast": [1, 3, 6, 10, 12, 13, 20, 24, 27, 29, 30], "faster": [1, 11, 13, 30], "fastest": [13, 21, 29], "favor": 7, "favorit": 24, "fc": 3, "featur": [0, 1, 3, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18, 19, 20, 24, 27, 29, 30], "feature_nam": [1, 7, 9], "feautur": 9, "fed": 1, "feed": [0, 2, 3, 11, 20, 27], "feed_forward": 1, "feed_forward_out": 1, "feed_forward_train": 1, "feedback": [4, 27], "feeddorward": 4, "feedforward": [1, 4, 12], "feel": [0, 5, 6, 11, 13, 15, 16, 18, 20, 22, 25, 27], "feet": [], "felt": 22, "fetch": [6, 15], "few": [1, 3, 4, 5, 9, 17, 18, 24, 27], "fewer": [0, 9, 11, 27], "ffnn": [1, 12], "field": [0, 3, 6, 12, 20], "fifth": [0, 6, 27], "fig": [0, 1, 2, 3, 4, 6, 7, 12, 13, 14, 22, 27], "fig_id": [0, 6, 7, 9, 27], "figaxi": 24, "figsiz": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 27], "figur": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 20, 22, 27, 28, 29, 30], "figure_id": [0, 6, 7, 9, 27], "figurefil": [0, 6, 7, 9, 27], "file": [0, 4, 5, 6, 7, 9, 15, 22, 27], "file_prefix": 4, "filenam": 27, "fill": [5, 9, 18, 28, 29], "filter": [3, 4], "final": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 19, 22, 23, 24, 25, 27, 29], "financ": 0, "find": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 22, 24, 27, 28, 29, 30], "fine": [0, 14], "finish": 2, "finit": [3, 5, 6, 12, 13, 17, 24, 28, 29], "finnicki": 15, "first": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 21, 22, 24, 25, 26, 28, 30], "first_moment": 30, "first_term": 30, "firsteigvector": 11, "fit": [1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 17, 18, 19, 22, 24, 28, 30], "fit_beta": 28, "fit_intercept": [0, 5, 6, 16, 28, 29, 30], "fit_mod": 9, "fit_theta": [6, 30], "fit_transform": [0, 6, 8, 9, 11, 15, 19], "fiti": [0, 27], "five": [0, 9, 27, 28], "fix": [0, 3, 4, 6, 10, 11, 12, 13, 22, 27], "flag": 4, "flat": [12, 13, 29, 30], "flatten": [1, 3, 4, 5, 21], "flexibl": [1, 6, 8, 10, 12, 27, 30], "flip": [25, 27], "float": [0, 3, 4, 5, 9, 11, 13, 14, 21, 27, 28, 29], "float32": [4, 9], "float64": [4, 21, 27], "flop": [5, 21, 28, 29], "flow": [1, 4, 12], "fluctuat": 5, "fly": 11, "fm": 0, "fmax": 3, "fmesh": 13, "fn": 7, "focu": [0, 3, 4, 5, 6, 15, 20, 22, 26, 27, 28, 29, 30], "focus": [1, 6, 7, 21, 28, 30], "fold": [6, 9, 22], "folder": [0, 4, 6, 15, 22, 27], "follow": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30], "font": [7, 24, 27], "fontdict": 24, "fontsiz": [1, 6, 8, 9, 10, 24], "fontweight": 1, "footprint": 3, "foral": [8, 28], "forc": [0, 5, 6, 10, 11, 28, 29, 30], "forcast": 4, "forecast": [4, 12], "forest": [0, 1, 9, 20, 27], "forget": 11, "form": [0, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 20, 21, 22, 24, 27, 28, 29, 30], "formal": [3, 4, 14, 18, 24], "format": [0, 1, 3, 4, 6, 7, 8, 9, 10, 11, 20, 24, 26], "format_data": 4, "formatstrformatt": [6, 13, 29, 30], "formul": [4, 6, 11, 14], "formula": [3, 13, 24, 29, 30], "forth": [4, 12], "fortran": [0, 20, 21, 27], "fortran2003": [20, 27], "fortran2008": 22, "fortran90": 24, "fortun": [0, 11, 28], "forward": [0, 3, 6, 20, 21, 27], "found": [1, 2, 4, 5, 6, 12, 13, 19, 22, 27, 28, 30], "foundat": [20, 27], "four": [4, 5, 6, 8, 12, 21, 23, 25, 27, 29], "fourier": [0, 27], "fourierdef1": 3, "fourierdef2": 3, "fourierseriessign": 3, "fourth": [12, 27, 28], "fp": 7, "frac": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 22, 24, 27, 28, 29, 30], "fraction": 9, "frame": 7, "framework": [1, 8, 10, 24], "frank": [5, 11], "frankefunct": [5, 6, 11], "fredli": [25, 27], "free": [0, 6, 11, 13, 15, 16, 18, 20, 21, 22, 24, 25, 26, 27], "freecodecamp": 20, "freedom": [5, 29], "freeli": [0, 22], "freez": 15, "frequenc": [3, 6, 7, 24], "frequent": [0, 8, 9, 13, 29], "frequentist": 20, "fresh": 10, "fridai": [15, 25, 27], "friedman": [6, 19, 22, 26, 27], "friendli": 4, "fro": 22, "frodo": 27, "frog": 3, "from": [0, 1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26], "from_cod": 9, "from_logit": [3, 4], "from_tensor_slic": 4, "front": [0, 4, 5, 27, 28, 29], "frustrat": 15, "fulfil": [2, 5, 12, 28, 29], "full": [1, 3, 5, 7, 9, 10, 13, 24, 27, 28, 29, 30], "full_matric": [5, 28, 29], "fulli": [3, 6, 12, 24], "fun": [20, 27], "func": 2, "function": [2, 3, 4, 5, 9, 14, 15, 16, 17, 18, 19, 20, 21], "functionali": 11, "fundament": [0, 6, 20, 27], "funtion": 2, "further": [2, 7, 9, 19, 27], "furthermor": [0, 3, 5, 6, 7, 11, 12, 13, 20, 22, 27, 28, 29, 30], "futur": [0, 4, 8, 9, 27], "fy": [15, 22, 23, 25, 26, 27], "fys4155": 22, "fys5419": [26, 27], "fys5429": [26, 27], "f\u00f8470": [25, 27], "g": [0, 1, 2, 3, 4, 6, 8, 9, 10, 11, 13, 15, 18, 24, 27, 28, 29, 30], "g0": 2, "g_": [2, 9, 10, 30], "g_0": 2, "g_1": [2, 10], "g_2": [2, 10], "g_analyt": 2, "g_dnn_ag": 2, "g_euler": 2, "g_i": 2, "g_m": [3, 10], "g_n": 3, "g_re": 2, "g_t": [2, 30], "g_t_d2t": 2, "g_t_d2x": 2, "g_t_dt": 2, "g_t_hessian": 2, "g_t_hessian_func": 2, "g_t_jacobian": 2, "g_t_jacobian_func": 2, "g_trial": 2, "g_trial_deep": 2, "g_vec": 2, "gain": [1, 5, 7, 9, 10, 13, 28, 29, 30], "galleri": [0, 27], "game": 4, "gamge": 27, "gamma": [0, 2, 8, 9, 10, 11, 13, 27, 29, 30], "gamma1": 8, "gamma2": 8, "gamma_": [0, 27], "gamma_0": 10, "gamma_1": 10, "gamma_1x": 10, "gamma_i": [0, 8, 24, 27], "gamma_j": [13, 30], "gamma_k": [13, 29], "gamma_m": 10, "gamma_x": [0, 27], "gap": 8, "gate": [4, 12], "gather": [0, 1, 12, 28], "gaug": 12, "gaussbacksub": 21, "gaussian": [4, 5, 6, 8, 14, 18, 24, 27], "gaussian_point": 14, "gaussian_rbf": 8, "gave": [13, 30], "gavra": 27, "gbc": 27, "gca": [2, 6, 8, 13], "gd": [1, 29], "gd_clf": 10, "gdclassiffiercgain": 10, "gdclassiffierconfus": 10, "gdclassiffierroc": 10, "gdm": [13, 30], "gdregress": 10, "ge": [1, 5, 7, 24, 28, 29], "gen_loss": 4, "gen_tap": 4, "gender": [0, 27], "genener": 4, "gener": [0, 1, 2, 3, 5, 6, 8, 10, 11, 12, 13, 14, 15, 16, 18, 21, 22, 24, 26, 28, 29, 30], "generaliz": 16, "generallay": 12, "generate_and_save_imag": 4, "generate_imag": 4, "generate_latent_point": 4, "generate_simple_clustering_dataset": 14, "generated_imag": 4, "generator_loss": 4, "generator_loss_list": 4, "generator_model": 4, "generator_optim": 4, "genom": 20, "geodes": 11, "geoff": 30, "geometr": [0, 13, 27], "geometri": 5, "georg": 26, "geotif": 6, "geq": [2, 5, 8, 9, 13, 28, 29, 30], "geron": [0, 26, 27], "get": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 15, 19, 20, 21, 22, 24, 25, 27, 28, 29, 30], "get_dummi": 9, "get_paramet": 2, "get_split": 9, "get_yaxi": 8, "get_yticklabel": 6, "gh": 15, "gibb": [20, 27], "gif": 4, "gini": 10, "gini_index": 9, "ginvers": 13, "git": [0, 15, 20, 27], "giter": [13, 30], "github": [0, 20, 22, 23, 25, 26, 27, 28], "gitignor": 15, "gitlab": [0, 15, 20, 22, 27], "give": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 18, 19, 20, 22, 24, 27, 28, 29, 30], "given": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 19, 21, 24, 27, 28, 29, 30], "global": [6, 7, 13, 29, 30], "glorot": 1, "gnew": 13, "go": [0, 1, 3, 5, 6, 8, 9, 11, 12, 13, 15, 16, 18, 27, 28, 29], "goal": [0, 7, 9, 27], "goe": [0, 1, 2, 5, 6, 13, 14, 15, 19, 21, 27, 28, 29, 30], "golden": 13, "gone": [5, 28, 29], "gong": 1, "good": [1, 3, 4, 5, 6, 9, 10, 11, 13, 15, 18, 20, 24, 26, 28, 29, 30], "goodfellow": [4, 26, 27, 28, 29], "googl": [1, 4, 20, 27], "got": [1, 6, 22], "gotten": 27, "gov": 6, "govern": 27, "gp": 26, "gpu": [1, 13, 20, 27], "grad": [2, 13, 30], "grad_analyt": 13, "grad_ol": 18, "grad_ridg": 18, "grade": [22, 23], "gradient": [0, 3, 4, 7, 8, 9, 12, 20, 27, 28], "gradient_desc": 30, "gradientboostingclassifi": 10, "gradientboostingregressor": 10, "gradients_of_discrimin": 4, "gradients_of_gener": 4, "gradienttap": 4, "gradual": [1, 14], "grai": [4, 6], "graph": [1, 9, 11, 12, 13, 16, 29], "graph_from_dot_data": 9, "graphic": [0, 1, 9, 15, 27], "grasp": 0, "gray_r": [1, 3], "grayscal": 3, "great": [5, 13, 15, 29, 30], "greater": [1, 7, 24, 28], "greatli": [13, 30], "greedi": 9, "green": [0, 3, 9, 24], "grei": 4, "grid": [1, 3, 6, 7, 8, 12, 24, 28, 30], "grossli": [13, 29], "ground": [0, 27], "group": [0, 6, 7, 9, 14, 15, 20, 22, 23, 25, 27], "groupbi": [0, 27], "grow": [1, 3, 9, 10, 30], "growth": [0, 27], "gru": 4, "guarante": [0, 4, 13, 24, 27, 28, 29, 30], "guess": [1, 4, 10, 13, 14, 29, 30], "guestrin": 10, "gui": 15, "guid": 1, "h": [0, 1, 5, 6, 8, 13, 15, 19, 24, 25, 26, 27, 28, 29, 30], "h1": 2, "h_": [0, 13, 27, 29, 30], "h_0": 30, "h_1": [2, 13, 29], "h_2": [2, 13, 29], "h_m": 10, "h_t": 30, "ha": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 24, 27, 28, 29, 30], "haanen": [25, 27], "habit": [0, 28], "had": [0, 1, 6, 7, 13, 27, 29, 30], "hadamard": [1, 12, 13, 30], "half": [1, 8, 9], "halv": 10, "hand": [0, 1, 2, 3, 5, 11, 12, 13, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30], "handi": [3, 22], "handl": [0, 1, 2, 5, 9, 11, 15, 18, 20, 28, 29, 30], "handle_unknown": 9, "handsid": 12, "handwrit": 12, "handwritten": [1, 5], "happen": [1, 2, 3, 4, 5, 6, 10, 13, 24, 28, 29, 30], "hard": [1, 7, 8, 10, 13, 29, 30], "hardcopi": [20, 27], "harder": [0, 1, 28], "harmon": 3, "hasn": [], "hassl": [0, 20, 27], "hast": [20, 27], "hasti": [0, 6, 16, 17, 19, 22, 26, 27, 28], "hat": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13, 16, 17, 18, 19, 21, 28, 29, 30], "have": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29, 30], "haven": 1, "he": 7, "head": [4, 10, 24], "header": [0, 27], "heads_proba": 10, "health": [0, 28], "hear": [0, 13, 27, 30], "heart": [0, 7, 27], "heatmap": [0, 1, 3, 7, 17, 27], "heavili": 0, "heavisid": 1, "height": [1, 3, 6, 28], "held": [13, 30], "help": [0, 1, 4, 12, 13, 15, 16, 22, 27, 30], "helper": [4, 14], "henc": [0, 5, 6, 8, 9, 10, 12, 13, 27, 28, 29, 30], "henrik": [25, 27], "her": 7, "here": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "hereaft": [0, 8, 12, 27], "hermitian": 21, "hessenberg": 21, "hessian": [0, 2, 5, 13], "heterogen": [9, 10], "hi": 7, "hidden": [1, 3, 4, 12], "hidden_bia": 1, "hidden_bias_gradi": 1, "hidden_layer_s": [0, 1, 27], "hidden_neuron": 4, "hidden_weight": 1, "hidden_weights_gradi": 1, "hierarch": [5, 28, 29], "high": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 13, 14, 20, 21, 22, 27, 28, 29, 30], "higher": [0, 1, 3, 5, 6, 8, 13, 18, 22, 27, 28, 29, 30], "highest": [1, 2], "highli": [0, 3, 4, 10, 19, 20, 21, 26, 27, 28, 29, 30], "highwai": [], "hing": 8, "hint": [13, 15, 16, 28, 29], "hinton": 30, "hip": 20, "hire": 0, "hist": [4, 6, 7, 24], "histogram": [6, 7, 24], "histor": [7, 11], "histori": [3, 4, 12, 15, 30], "hitherto": 5, "hjorth": [25, 27, 28, 29, 30], "hobbi": 24, "hoc": [5, 28, 29], "hoff": 26, "hold": [1, 3, 6, 13, 14, 29, 30], "holder": [0, 27], "home": [], "homepag": [22, 27], "homework": [6, 13, 29, 30], "homogen": [1, 3, 9, 10, 13, 30], "honchar": 2, "hopefulli": [0, 11, 15, 24, 27, 30], "horizont": 11, "horlyk": [25, 27], "hors": [3, 7, 27], "hot": [1, 9], "hour": [1, 20, 23, 24, 25, 27], "how": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "howev": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 22, 24, 27, 28, 29, 30], "hspace": [0, 4, 8, 10, 24, 27], "hstack": 1, "htf": 27, "html": [0, 16, 20, 22, 23, 25, 26, 27, 28, 29, 30], "http": [0, 3, 4, 6, 13, 15, 16, 20, 21, 22, 23, 25, 26, 27, 28, 29, 30], "huang": [0, 27], "huber": [0, 27], "huge": [1, 3, 4, 20], "human": [0, 1, 3, 6, 9, 12, 28], "humid": 9, "hundr": 1, "hungri": 1, "hybrid": 23, "hydrogen": [0, 27], "hyperbol": [1, 4, 12], "hyperparam": 8, "hyperparamet": [3, 4, 5, 6, 9, 13, 18, 22, 28, 29, 30], "hyperplan": 11, "h\u00f8rlyk": [25, 27], "i": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30], "i0": [0, 27], "i1": [0, 6, 8, 12, 27, 28, 30], "i2": [0, 8, 12, 27], "i3": [0, 12, 27], "i5": [0, 27], "i_": [13, 29, 30], "i_1": [5, 6], "i_2": [5, 6], "i_t": 30, "ian": 26, "ic": [1, 22], "id": [7, 13, 29, 30], "ida": [25, 27], "idea": [0, 1, 2, 3, 4, 6, 9, 10, 12, 13, 21, 22, 28, 29, 30], "ideal": [0, 2, 6, 8, 13, 24, 27, 30], "idem": 6, "ident": [5, 6, 12, 13, 17, 18, 21, 28, 29, 30], "identifi": [0, 1, 7, 9, 11, 12, 13, 14, 27, 28, 30], "ieor": 24, "ifi": 26, "ifs": [20, 27], "ignor": [0, 1, 3, 9, 15, 28, 30], "ii": [21, 24], "iii": [21, 27], "ij": [0, 1, 3, 6, 8, 12, 14, 16, 21, 24, 27, 28, 30], "ik": [0, 21, 27, 28], "illustr": [5, 7, 10, 12, 13, 14, 20, 27], "ilsvrc": 30, "im": 6, "imag": [1, 3, 4, 6, 9, 11, 12, 14, 26, 27], "image_at_epoch_": 4, "image_batch": 4, "image_height": 3, "image_path": [0, 6, 7, 9, 27], "image_width": 3, "imageio": 6, "images_from_seed_imag": 4, "imagin": 1, "immedi": [0, 3, 4, 6, 20, 27], "implement": [0, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 22, 24, 27, 28, 29, 30], "impli": [3, 5, 6, 7, 13, 21, 28, 29, 30], "implicit": 3, "implicitli": [11, 24], "import": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 30], "importantli": 3, "impos": [0, 6, 11, 12, 27], "imposs": [0, 5, 27, 28, 29], "impress": [0, 12, 27], "improv": [0, 4, 5, 9, 10, 11, 13, 15, 22, 28, 29], "impur": 9, "imread": 6, "imshow": [1, 3, 4, 6], "in3050": [26, 27], "in3310": 27, "in4080": [26, 27], "in4300": [26, 27], "in4310": 26, "in5400": 3, "in5550": 26, "in_out_neuron": 4, "inaccur": [13, 29], "inact": 12, "inadequ": [0, 27], "inappropri": 30, "inch": [6, 28], "includ": [0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 15, 16, 17, 18, 20, 24, 25, 26, 27, 28, 29], "include_bia": [6, 9], "incom": [12, 16], "incorrect": 1, "incoveni": 8, "increas": [0, 1, 3, 4, 5, 6, 9, 12, 13, 19, 22, 24, 27, 28, 30], "increasingli": 24, "increment": 30, "ind": 6, "inde": [0, 2, 4, 5, 6, 13, 27, 28, 29], "indefinit": 4, "independ": [0, 5, 6, 7, 8, 12, 13, 24, 27, 28, 29], "index": [0, 1, 3, 4, 10, 14, 20, 21, 22, 24, 26, 27], "index_col": [0, 27], "indic": [0, 1, 3, 4, 5, 6, 9, 10, 11, 13, 16, 22, 27, 28, 30], "indispens": 6, "individu": [1, 6, 7, 10, 12, 24, 27, 28, 30], "indu": [], "indx": 21, "indx1": 2, "indx2": 2, "indx3": 2, "ineffici": [3, 13], "inequ": [8, 13], "inequaltii": 29, "inertia": [13, 30], "inf1000": [20, 27], "inf1100": [20, 27], "inf1100l": [20, 27], "inf1110": [20, 27], "inf3000": 27, "infeas": 9, "infer": [0, 1, 4, 6, 26, 27], "inferenc": 1, "infil": [0, 6, 7, 9, 27], "infin": [5, 6, 7, 11, 19, 28, 29], "infinit": [3, 30], "infinitesim": 24, "influenc": [6, 10, 18], "influenti": 1, "info": 27, "inform": [0, 1, 3, 4, 6, 9, 11, 12, 13, 14, 21, 22, 26, 27, 29, 30], "inforom": 15, "infrequ": 30, "infti": [3, 6, 13, 24, 29], "ingeni": [13, 29, 30], "ingredi": [0, 9, 27], "inher": 6, "inherit": [21, 27], "initi": [0, 1, 2, 6, 10, 13, 14, 18, 21, 24, 27, 29, 30], "inject": 14, "inlin": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 24, 27, 28, 29, 30], "inner": [0, 13, 28], "inp": 4, "inplac": 13, "input": [0, 1, 3, 4, 5, 6, 7, 8, 12, 13, 14, 16, 22, 24, 27, 28, 29, 30], "input_dim": 1, "input_shap": [3, 4], "inputs": 1, "inputs_shuffl": [0, 1, 28], "insert": [3, 5, 6, 8, 10, 24, 28, 29], "insid": [4, 7], "insight": [0, 1, 5, 20, 27, 28, 29], "insist": [6, 13, 28, 30], "inspir": [0, 1, 12, 22, 27], "instabl": 2, "instal": [0, 1, 5, 6, 9, 15], "instanc": [0, 1, 2, 4, 6, 9, 11, 13, 16, 27, 28, 29], "instanti": 10, "instead": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 17, 21, 24, 27, 28, 30], "institut": 1, "instruct": [0, 1, 15], "int": [0, 1, 2, 3, 4, 5, 6, 11, 13, 14, 21, 24, 28, 30], "int32": 10, "int_": [3, 6, 24], "int_0": 24, "int_a": 24, "intak": [0, 28], "integ": [1, 2, 13, 14, 21, 24, 27], "integer_vector": 1, "integr": [3, 6, 24, 27], "intellig": [0, 14, 26, 27], "intend": 10, "intens": [1, 18], "intention": 14, "interact": [0, 6, 9, 12, 20, 22, 27], "intercept": [0, 6, 8, 11, 13, 16, 17, 18, 27, 28, 29, 30], "intercept_": [0, 6, 8, 9, 13, 27, 28, 30], "interchang": [5, 12, 21], "interconnect": 1, "interesit": 19, "interest": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 20, 22, 24, 27, 28, 29], "interfac": [0, 1, 15, 21, 28], "interior": [0, 9, 27], "intermedi": [21, 28], "intern": [1, 10, 12], "interpol": [1, 3, 4, 6, 12], "interpr": [5, 28, 29], "interpret": [0, 1, 6, 9, 10, 12, 13, 15, 16, 19, 21, 22, 24], "interv": [0, 3, 5, 6, 7, 13, 19, 24, 27, 28, 29], "intial": [13, 29], "intract": [0, 4, 28], "intrins": [3, 11, 21, 24, 27], "intro": [20, 26, 27], "introduc": [0, 1, 5, 6, 8, 10, 12, 21, 22, 24, 27, 29, 30], "introduct": [1, 2, 4, 13, 26, 28, 29, 30], "introductori": [0, 4, 21, 26, 27, 28], "intuit": [0, 5, 6, 8, 12, 13, 22, 27, 30], "inv": [0, 5, 13, 17, 27, 28, 29, 30], "invalu": [0, 13, 20, 27, 29], "invari": 1, "invd": 5, "inver": 8, "invers": [0, 3, 6, 13, 27, 28, 29, 30], "inverse_transform": 8, "invert": [0, 5, 7, 10, 13, 16, 18, 27, 30], "invh": [13, 30], "invok": 8, "involv": [0, 2, 6, 7, 11, 12, 27, 28, 30], "io": [0, 20, 22, 23, 25, 26, 27, 28], "ip": [0, 8, 24, 27], "ipca": 11, "ipynb": [20, 27], "ipython": [0, 5, 7, 9, 11, 14, 20, 22, 27, 28], "iq": 6, "iri": [8, 9], "irreduc": 6, "irrelev": [5, 28, 29], "irrespect": [0, 27], "irvin": 22, "iseffici": 30, "isn": 5, "isnul": [], "isomap": 11, "issu": [1, 9, 15, 21, 30], "it_arrai": 13, "item": [0, 13, 27], "items": [21, 27], "iter": [1, 2, 4, 6, 8, 13, 14, 18, 22, 24, 29, 30], "its": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 21, 22, 24, 27, 29, 30], "itself": [5, 6, 12, 19, 22, 24, 27, 28], "j": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14, 15, 16, 21, 22, 24, 26, 27, 28, 29, 30], "j1": 21, "j_": 6, "j_lasso_sk": 6, "j_ridge_sk": 6, "j_sk": 6, "jackknif": [6, 20, 27], "jacobian": [2, 13, 29], "jason": 4, "jax": [20, 27, 30], "jensen": [25, 27, 28, 29, 30], "jerom": [19, 22, 26], "ji": [12, 21], "jit": 13, "jj": [0, 5, 6, 27], "jk": [0, 1, 6, 12, 21, 27], "jl": [0, 27], "jm": 21, "jnp": 13, "job": [2, 8, 10, 15], "join": [0, 4, 6, 7, 9, 27], "joint": [4, 5], "judg": [13, 29], "judgement": 6, "julia": [20, 21, 22], "jump": 24, "junk": 4, "jupit": 27, "jupyt": [0, 15, 16, 20, 22, 26, 27], "just": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 24, 27, 28, 29, 30], "justif": 0, "justifi": [3, 10], "k": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 20, 21, 22, 24, 25, 27, 28, 29, 30], "k0": 7, "k1": 7, "kaggl": [6, 22], "kappa_d": 24, "karl": [25, 27], "karush": 8, "katrin": [25, 27], "keep": [0, 1, 4, 5, 6, 11, 13, 14, 15, 18, 21, 22, 27, 28, 29, 30], "keepdim": [1, 6, 10, 21], "kei": [1, 3, 6, 12], "kept": [4, 6, 14], "kera": [0, 4, 20, 22, 27], "kernel": [0, 1, 3, 20, 27, 28], "kernel_regular": [1, 3], "kernel_s": 4, "kernelpca": 11, "kev": [0, 27], "kevin": [26, 27], "keyword": [18, 21, 27], "kfold": 6, "kg": 1, "ki": 21, "kick": [1, 13, 30], "kiener": 2, "kilomet": [6, 28], "kind": [0, 2, 3, 4, 8, 12, 13, 14, 27, 28], "kingma": 30, "kj": [6, 12, 21, 28, 30], "kjm": [20, 27], "kkt": 8, "kl": 24, "km": [12, 27], "kmean": 14, "kmeanspoint": 14, "kn_k": 14, "know": [0, 1, 2, 5, 6, 8, 13, 15, 16, 17, 19, 20, 27, 28, 29], "knowledg": [0, 20, 27], "known": [1, 3, 4, 5, 6, 7, 8, 9, 12, 18, 21, 22, 24, 26, 28, 30], "kondev": [0, 27], "kp": 24, "kpca": 11, "kroneck": 14, "kuhn": 8, "kvalsund": [25, 27], "kwown": [0, 27], "l": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 21, 22, 24, 27, 29, 30], "l0": 7, "l1": [0, 1, 3, 7, 27], "l1_l2": [1, 3], "l1regl": 5, "l2": [1, 3], "l_": 21, "l_1": 7, "l_2": [7, 13, 29, 30], "l_j": 12, "la": 13, "la_i": 12, "la_k": 12, "lab": [20, 22, 27], "label": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 20, 21, 22, 24, 27, 28, 29, 30], "labelencod": [7, 10], "labels": [6, 8, 9], "labels_shuffl": [0, 1, 28], "laboratori": 23, "lack": [0, 27], "lagari": 2, "lagrang": [8, 11], "lam": 18, "lambda": [0, 1, 2, 3, 5, 6, 7, 8, 10, 12, 13, 17, 18, 19, 22, 24, 27, 28, 29, 30], "lambda_": 11, "lambda_0": 11, "lambda_1": [5, 8, 11, 28, 29], "lambda_2": [8, 11], "lambda_i": [8, 11], "lambda_iy_i": 8, "lambda_jy_iy_j": 8, "lambda_k": 8, "lambda_n": [5, 8, 28, 29], "lamda": 1, "land": 8, "landmark": 8, "landscap": [13, 18, 29, 30], "langl": [0, 6, 11, 24, 27, 28], "languag": [0, 1, 4, 8, 20, 21, 22, 26, 27], "lapack": [21, 27], "laplac": 5, "laptop": [15, 20], "larg": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 13, 18, 20, 21, 22, 24, 26, 27, 28, 29, 30], "larger": [0, 3, 5, 6, 8, 10, 11, 13, 17, 24, 27, 28, 29, 30], "largest": [4, 8, 11], "lasso": [0, 7, 20, 27, 30], "lasso_sk": 6, "last": [0, 1, 3, 4, 5, 6, 7, 8, 12, 16, 17, 19, 21, 22, 24, 25, 27, 29], "latent": 4, "latent_dim": 4, "latent_point": 4, "latent_space_value_rang": 4, "later": [0, 1, 4, 7, 8, 12, 13, 14, 15, 20, 22, 27, 30], "latest": [4, 15, 20], "latest_checkpoint": 4, "latex": 27, "latter": [0, 3, 6, 7, 8, 11, 13, 21, 24, 27, 28, 29, 30], "lattic": 12, "law": 0, "layer": [0, 4, 13, 27, 30], "lbfg": [7, 9, 10], "lcc": [5, 6], "lda": 11, "ldot": [0, 6, 11, 22, 27], "le": [5, 7, 10, 13, 17, 24, 28, 29, 30], "lead": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 21, 24, 27, 28, 29, 30], "leaf": 9, "leaki": 1, "leakyrelu": 4, "lear": [13, 29], "learn": [3, 4, 5, 6, 7, 8, 9, 10, 12, 21, 25, 26], "learnabl": 3, "learner": 10, "learnig": 27, "learning_r": [8, 10], "learning_rate_init": [0, 1, 27], "learning_schedul": [13, 30], "learnt": 22, "least": [0, 7, 8, 10, 11, 17, 18, 20, 21, 24], "leat": [13, 30], "leav": [0, 1, 3, 5, 6, 9, 11, 27, 29], "lectur": [0, 1, 5, 10, 11, 12, 13, 20, 21, 22, 23, 25, 26, 28], "lecturenot": [0, 20, 22, 26, 27], "left": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 22, 24, 27, 28, 29, 30], "leftarrow": [8, 12], "legend": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 15, 27, 28, 29, 30], "leinonen": 27, "len": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 16, 17, 21, 27, 28, 29, 30], "length": [0, 1, 3, 4, 8, 9, 13, 16, 20, 27, 28, 29, 30], "length_of_sequ": 4, "leq": [0, 5, 7, 8, 13, 14, 24, 27, 28, 29, 30], "less": [0, 1, 3, 4, 5, 6, 8, 9, 13, 20, 24, 27, 28, 29, 30], "lessen": 1, "let": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 21, 24, 27, 28, 29, 30], "letter": [0, 16, 21, 24, 27, 28], "level": [0, 1, 5, 6, 9, 20, 21, 22, 23, 25, 27], "li": [8, 11], "lib": [], "liberti": 30, "liblinear": 10, "librari": [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 21, 22, 24, 26, 28, 29, 30], "licens": [0, 1, 20, 22, 27], "lie": [0, 6, 11, 24, 27, 28], "life": [0, 1, 8, 12, 27], "lifetim": [13, 30], "like": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 20, 21, 22, 24, 27, 28, 29, 30], "likelihood": [0, 1, 5, 9, 27, 28], "lim_": 24, "limit": [0, 5, 6, 8, 12, 21, 22, 27, 28], "lin_clf": 8, "lin_model": [], "lin_reg": 9, "linalg": [0, 2, 5, 6, 8, 11, 13, 17, 21, 24, 27, 28, 29, 30], "line": [0, 3, 6, 8, 11, 13, 15, 16, 27, 29, 30], "line1": 8, "line2": 8, "line3": 8, "line_model": 15, "line_ms": 15, "line_predict": 15, "linear": [1, 3, 5, 6, 7, 9, 10, 11, 12, 16, 17, 18, 19, 20, 22, 24, 30], "linear_model": [0, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 19, 27, 28, 29, 30], "linear_regress": 6, "linearli": [5, 28, 29], "linearloc": [6, 13, 29, 30], "linearregress": [0, 6, 7, 9, 15, 16, 19, 27, 28, 30], "linearsvc": 8, "lineat": 29, "liner": [1, 3], "linerar": 10, "linewidth": [0, 2, 4, 6, 8, 9, 10], "link": [0, 4, 9, 12, 15, 20, 22, 23, 25, 27], "linlag": 5, "linpack": [21, 27], "linreg": [0, 27], "linspac": [0, 2, 3, 4, 6, 8, 9, 10, 13, 16, 17, 19, 21, 24, 27, 28, 30], "linu": 4, "linux": [0, 1, 20, 22, 27], "liquid": [0, 27], "list": [1, 2, 3, 4, 9, 15, 20, 22, 27, 30], "listedcolormap": [9, 10], "literatur": [1, 7, 14, 26], "littl": [1, 3, 9, 12, 30], "live": [8, 16], "ll": [0, 18, 24, 27, 28], "lle": [0, 28], "lloyd": [4, 14], "lmb": [0, 2, 5, 6, 28, 29, 30], "lmbd": [0, 1, 3, 27], "lmbd_val": [0, 1, 3, 27], "lmbda": [13, 29, 30], "ln": [1, 13, 29], "load": [1, 4, 6, 7, 9, 10], "load_boston": [], "load_breast_canc": [1, 7, 9, 10, 11], "load_data": [3, 4], "load_digit": [1, 3], "load_iri": [8, 9], "loc": [3, 6, 7, 8, 9, 10, 27], "local": [0, 1, 3, 7, 12, 13, 15, 28, 29, 30], "locat": [2, 3, 8, 15], "log": [0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 13, 15, 21, 22, 27], "log10": [0, 5, 6, 28, 29, 30], "log_": [0, 27], "log_clf": 10, "logarithm": [0, 5, 7, 17, 21, 27], "logbook": 22, "logic": [0, 1, 9, 27], "login": 15, "logist": [0, 1, 2, 8, 9, 10, 11, 12, 13, 20, 28, 29, 30], "logisticregress": [7, 9, 10, 11], "logit": 7, "logreg": [7, 9, 10, 11], "logspac": [0, 1, 3, 5, 6, 27, 28, 29, 30], "long": [0, 1, 3, 4, 12, 13, 27, 29, 30], "longer": [2, 3, 8, 10, 14, 21, 24, 27, 30], "loocv": 6, "look": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 19, 21, 22, 24, 27, 28, 29, 30], "loop": [1, 4, 6, 10, 12, 14, 16, 17, 18, 20, 21, 27], "lose": 1, "loss": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13, 18, 21, 22, 27], "loss_fil": 4, "lossfil": 4, "lost": 4, "lot": [1, 4, 6, 16, 30], "low": [0, 6, 9, 10, 11, 22, 27, 28], "lower": [0, 1, 3, 6, 9, 10, 16, 21, 28], "lowercas": [21, 27], "lowest": [9, 13, 24, 30], "lr": [1, 3, 4, 10], "lstat": [], "lstm": 4, "lstm_2layer": 4, "lstsq": [0, 27, 28], "lt": 6, "lu": [0, 5, 27, 28, 29], "lubksb": 21, "luckili": 2, "ludcmp": 21, "lux": 21, "lvert": 1, "lw": [0, 27], "m": [0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 15, 21, 24, 25, 26, 27, 28, 29, 30], "m_": [9, 12], "m_1": 14, "m_h": [0, 27], "m_k": 14, "m_l": 12, "m_n": [0, 27], "m_p": [0, 27], "m_t": [13, 30], "ma": 11, "machin": [1, 3, 4, 5, 6, 7, 9, 10, 11, 12, 15, 16, 21, 26, 28], "machinelearn": [0, 6, 16, 20, 22, 23, 25, 26, 27, 28, 29], "mackai": 26, "made": [0, 1, 3, 4, 5, 6, 7, 9, 11, 12, 22, 27, 28], "mae": [0, 27], "magic": 4, "magnitud": [1, 6, 7, 13, 28, 30], "mai": [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 19, 20, 21, 22, 24, 27, 28, 29, 30], "mail": [23, 25], "main": [0, 1, 3, 4, 5, 6, 7, 9, 21, 22, 26, 28, 29, 30], "mainli": [0, 5, 6, 7, 9, 27, 28], "maintain": [6, 30], "major": [1, 6, 9, 10, 13, 21, 27, 29, 30], "make": [1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 15, 16, 18, 20, 21, 22, 24, 26, 27, 29, 30], "make_axes_locat": 6, "make_moon": [8, 9, 10], "make_pipelin": [0, 6, 10, 28], "makedir": [0, 6, 7, 9, 27], "malcondit": 21, "malign": [1, 7, 9], "mammographi": 5, "manag": [0, 2, 3, 15, 20, 22, 27], "mandatori": [25, 27], "mani": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 26, 27, 28, 29, 30], "manifold": 11, "manner": 3, "manual": [6, 28, 30], "map": [0, 1, 2, 6, 7, 8, 11, 12, 14, 24, 27], "marc": 28, "margin": [0, 5, 8], "marit": [0, 27], "mark": 27, "marker": [7, 21, 27], "markov": [20, 27], "marsaglia": 24, "mass": [0, 1, 5, 13, 28, 29, 30], "massag": [0, 27], "masses2016": [0, 27], "masses2016ol": [0, 27], "masses2016tre": 0, "masseval2016": [0, 27], "master": [23, 25], "mat": [20, 27], "mat1100": [20, 27], "mat1110": [20, 27], "mat1120": [20, 27], "match": [1, 4, 5, 13, 14, 15, 28, 29, 30], "materi": [4, 5, 7, 13, 15, 21, 23, 25], "math": [3, 7, 12, 13, 21, 24, 26, 27, 30], "mathbb": [0, 4, 5, 6, 7, 8, 11, 12, 13, 14, 17, 19, 21, 22, 24, 27, 28, 29, 30], "mathbf": [0, 5, 6, 7, 8, 13, 19, 21, 22, 27, 28, 29, 30], "mathcal": [1, 5, 6, 7, 13, 22], "matheemat": 3, "mathemat": [0, 6, 11, 12, 13, 20, 21, 24, 26, 27, 30], "mathemati": 27, "mathrm": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 22, 24, 27, 28, 29, 30], "matmul": [1, 2, 5], "matnat": 26, "matplotlib": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 20, 21, 22, 24, 27, 28, 29, 30], "matric": [0, 1, 3, 4, 6, 7, 8, 11, 13, 16, 17, 20, 28, 29], "matrix": [0, 2, 3, 4, 6, 7, 8, 10, 13, 17, 18, 19, 22, 24], "matshow": 1, "matter": [2, 3, 13, 28, 29, 30], "max": [0, 1, 2, 3, 4, 9, 10, 12, 13, 25, 27, 29, 30], "max_depth": [0, 9, 10], "max_diff": 2, "max_diff1": 2, "max_diff2": 2, "max_it": [0, 1, 8, 13, 27], "max_iter": 14, "max_leaf_nod": 10, "max_sampl": 10, "maxdegre": [0, 6, 10, 28], "maxdepth": 10, "maxim": [1, 4, 5, 7, 8, 11], "maximum": [0, 2, 3, 5, 7, 8, 9, 10, 13, 14, 27, 28, 29, 30], "maxpolydegre": [5, 6, 28, 29, 30], "maxpooling2d": 3, "mbox": [5, 6, 28, 29], "mcculloch": 12, "md": 11, "mdoel": 4, "mean": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 27, 30], "mean_absolute_error": [0, 27], "mean_divisor": 14, "mean_i": 24, "mean_matrix": 14, "mean_squared_error": [0, 4, 6, 7, 10, 15, 19, 27, 28], "mean_squared_log_error": [0, 27], "mean_vector": 14, "mean_x": 24, "meaning": [0, 4, 7, 27], "meansquarederror": [0, 27], "meant": [3, 7, 10, 13], "measur": [0, 1, 2, 5, 6, 9, 11, 12, 14, 16, 18, 19, 22, 24, 27, 28, 30], "mechan": [0, 4, 24, 27, 30], "median": [0, 27, 28, 30], "medicin": 12, "medium": [4, 8, 13, 30], "medv": [], "meet": [0, 25], "mehta": [0, 27, 28, 29], "memori": [3, 4, 11, 12, 13, 18, 21, 30], "mention": [0, 12, 13, 22, 24, 27, 29, 30], "mere": [0, 22], "meshgrid": [2, 5, 6, 8, 9, 10, 11], "mess": 15, "messag": [5, 13], "messi": 2, "met": [0, 3, 8, 28], "meteorolog": 9, "meter": [6, 28], "method": [0, 1, 2, 3, 4, 5, 7, 8, 11, 12, 14, 15, 16, 17, 18, 19, 20, 21, 24, 26, 28], "metion": 6, "metric": [0, 1, 3, 6, 7, 9, 10, 14, 15, 19, 27, 28], "metropoli": [20, 27], "mev": [0, 24, 27], "mgd": [13, 30], "mglearn": [20, 27], "mgrid": 13, "mhjensen": [], "mi": 10, "mia": [25, 27], "microsoft": 26, "mid": 1, "midel": 4, "midnight": 15, "midpoint": 9, "might": [0, 1, 2, 4, 6, 9, 13, 15, 17, 18, 28, 29, 30], "migth": 17, "mild": 9, "millimet": [6, 28], "million": [0, 27, 28, 30], "mimic": 12, "min": [0, 2, 5, 8, 9, 29], "min_": [0, 2, 5, 14, 17, 27, 28, 29], "min_samples_leaf": 9, "mind": [0, 6, 13, 15, 18, 27, 28, 29, 30], "mindboard": 4, "mine": [20, 27], "mini": [1, 11, 12, 13, 29], "minibatch": [1, 11, 13], "minibathc": [13, 30], "miniforge3": [], "minim": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 28, 29, 30], "minima": [0, 1, 7, 13, 27, 29, 30], "minimum": [0, 1, 2, 6, 8, 9, 11, 13, 28, 29, 30], "minmaxscal": [0, 28, 30], "minor": 24, "minst": 1, "minu": 7, "mirjalili": 27, "mirror": 9, "misc": 6, "misclassif": [8, 9, 10], "misclassifi": [8, 10], "miser": 0, "mismatch": 1, "miss": [7, 10], "mistak": 4, "mit": 26, "mix": [1, 2, 27], "mixtur": [13, 30], "mk": [9, 21], "mkdir": [0, 6, 7, 9, 27], "ml": [0, 1, 10, 13, 21, 22, 28, 29, 30], "mlab": 24, "mle": [5, 7], "mlp": 1, "mlpclassifi": 1, "mlpregressor": [0, 27], "mm": 21, "mml": 28, "mn": [12, 24], "mnist": [1, 11], "mod": 24, "mode": [23, 25, 27], "model": [2, 3, 5, 7, 8, 9, 10, 11, 13, 14, 16, 18, 19, 20, 22, 24, 26, 28, 29, 30], "model_select": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 27, 28, 29, 30], "moder": 10, "modern": [0, 6, 7, 20, 27], "modif": [2, 12, 13, 30], "modifi": [0, 1, 3, 5, 7, 8, 10, 12, 13, 27, 28, 29, 30], "modul": [0, 16, 21, 27], "modular": 24, "modulo": 24, "moe": [11, 28], "moment": [5, 6, 13, 24], "mondai": [25, 27], "monitor": [13, 30], "monoton": [5, 12, 24], "mont": [0, 6, 20, 24, 26, 27], "montli": 16, "moor": [5, 6], "more": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 20, 24], "moreov": [0, 3], "morten": [25, 27, 28, 29, 30], "mortenhj": 27, "most": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 22, 24, 27, 28, 29, 30], "mostli": [1, 11, 18], "motion": [0, 13, 30], "motiv": [1, 4], "move": [0, 4, 5, 6, 7, 9, 12, 13, 14, 15, 16, 22, 24, 28, 29, 30], "mpl": [7, 27], "mpl_toolkit": [2, 6, 13, 29, 30], "mplot3d": [2, 6, 13, 29, 30], "mplregressor": 1, "mse": [0, 4, 5, 6, 9, 10, 15, 16, 17, 19, 22, 27, 28, 29, 30], "mse_simpletre": 10, "mselassopredict": [5, 29], "mselassotrain": [5, 29], "mseownridgepredict": [6, 28, 29, 30], "msepredict": [5, 29], "mseridgepredict": [0, 5, 6, 28, 29, 30], "msetrain": [5, 29], "msle": [0, 27], "mt": [7, 12], "mu": [0, 6, 11, 13, 24, 27, 30], "mu0": 24, "mu1": 24, "mu2": 24, "mu_": [6, 24, 28, 30], "mu_i": [6, 28, 30], "mu_n": 11, "mu_x": 24, "much": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 21, 22, 24, 27, 28, 29, 30], "multi": [0, 1, 3, 7, 20, 27], "multiclass": [1, 7], "multidimension": [11, 12, 27], "multilay": 1, "multinomi": 7, "multipl": [2, 4, 5, 6, 7, 12, 13, 15, 24, 28, 29, 30], "multipli": [3, 5, 6, 11, 13, 18, 21, 24, 28, 29, 30], "multiplum": 8, "multivari": [0, 2, 10, 11, 20, 24, 27], "multivariate_norm": [11, 14], "multpli": 16, "murphi": [11, 26, 27], "must": [1, 2, 5, 6, 8, 10, 12, 13, 14, 15, 24, 28, 29, 30], "mutat": 7, "mutual": [1, 3, 6, 13], "mx_": 24, "my": 27, "myenv": [], "myriad": [0, 20, 27], "mz1": 24, "mz2": 24, "m\u00f8svatn": 6, "n": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 27, 28, 29, 30], "n1": 21, "n2": 21, "n_": [1, 2, 3, 8, 12, 24], "n_0": [12, 24], "n_boostrap": [6, 10], "n_bootstrap": 6, "n_categori": [1, 3], "n_cluster": 14, "n_compon": 11, "n_epoch": [13, 30], "n_estim": 10, "n_examples_to_gener": 4, "n_featur": [1, 18], "n_filter": 3, "n_hidden": 2, "n_hidden_neuron": [0, 1, 27], "n_i": 24, "n_input": [0, 1, 3, 28], "n_instanc": 9, "n_iter": 30, "n_job": 10, "n_k": 14, "n_l": [12, 24], "n_layer": 1, "n_m": 9, "n_neuron": 1, "n_neurons_connect": 3, "n_neurons_layer1": 1, "n_neurons_layer2": 1, "n_point": 14, "n_sampl": [6, 8, 9, 10, 14, 18], "n_split": 6, "n_step": 4, "n_t": 2, "n_x": 2, "nabla": [1, 13, 29, 30], "nabla_": [2, 13, 29, 30], "nabla_w": [13, 30], "nag": [13, 30], "naimi": [0, 27], "naiv": 7, "naive_kmean": 14, "name": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 18, 20, 21, 22, 24, 25, 27, 28, 29, 30], "narrow": [13, 30], "nation": [1, 5], "nativ": [20, 27], "natur": [0, 1, 4, 8, 9, 12, 13, 22, 24, 26, 27, 29, 30], "navier": 12, "navig": 15, "nb": 24, "nb_": 21, "nbconvert": 27, "nd": 14, "ndarrai": 6, "ne": [9, 10, 21, 24, 28, 29], "nearest": [1, 3, 6, 11], "nearli": [13, 29], "neat": 27, "neccesari": 6, "necess": 2, "necessari": [0, 1, 3, 4, 8, 14, 18, 27], "necessarili": [0, 4, 11, 24, 27], "necesserali": 5, "neck": 7, "need": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 24, 28, 29, 30], "neg": [0, 1, 3, 5, 6, 7, 10, 13, 21, 24, 27, 29], "neg_mean_squared_error": 6, "neglect": [24, 30], "neglig": 24, "neighbor": [3, 6, 11], "neither": [4, 13], "neq": [13, 14, 24, 29], "nervou": 12, "nest": [9, 12], "nesterov": [13, 30], "net": [2, 4, 12], "netlib": [21, 27], "network": [0, 9, 13, 20, 26, 28, 30], "neural": [0, 13, 20, 26, 28, 30], "neural_network": [0, 1, 2, 27], "neuralnetwork": 1, "neuron": [1, 2, 3, 4, 12], "neutral": [0, 27], "neutron": [0, 27], "never": [1, 4, 6, 9, 24], "new": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 21, 27, 28, 29, 30], "new_chang": [13, 30], "new_hobbit": 27, "newaxi": [0, 3, 6, 9], "newli": [0, 27], "newton": [1, 7, 8, 13, 24], "next": [0, 1, 2, 3, 4, 5, 6, 8, 9, 13, 14, 15, 16, 27, 28, 29], "next_guess": 13, "next_input": 4, "ng": 1, "ni": 14, "nice": [0, 1, 5, 11, 27, 28, 29], "nicer": [18, 30], "niter": [13, 29, 30], "nitric": [], "nlambda": [0, 5, 6, 28, 29, 30], "nlp": 26, "nm": 24, "nm_n": [0, 27], "nmse": 6, "nn": [2, 5, 6, 12, 21, 27], "nn_model": 1, "nnmin": 2, "node": [1, 3, 9, 10, 12], "nois": [0, 4, 5, 6, 8, 9, 10, 13, 18, 19, 22, 27, 28, 29, 30], "noise_dimens": 4, "noisi": [1, 6, 22], "non": [0, 1, 3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 18, 21, 24, 27, 28, 29, 30], "nondifferenti": 30, "none": [0, 1, 2, 4, 5, 9, 10, 13, 24, 27, 28], "nonlinear": [3, 6, 8, 9, 11, 12], "nonneg": [6, 9, 13, 29], "nonparametr": 6, "nonsens": 24, "nonsingular": 21, "nonumb": [3, 7, 8, 13, 21, 30], "nor": [1, 4, 13], "norm": [0, 1, 5, 6, 8, 11, 13, 18, 27, 28, 29, 30], "normal": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "normali": [21, 27], "norwai": [6, 22, 27, 29, 30], "notat": [0, 2, 5, 6, 13, 14, 24, 27, 28, 29, 30], "note": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 26, 27, 30], "notebook": [0, 1, 3, 9, 15, 16, 20, 22, 27], "noth": [1, 2, 5, 8, 12, 14, 24, 28, 29], "notic": [4, 5, 12, 13, 21, 24, 27, 30], "notion": 3, "novel": [3, 6, 10, 27], "novemb": [1, 25, 27], "now": [0, 2, 4, 5, 6, 7, 8, 10, 11, 12, 14, 15, 16, 19, 20, 21, 22, 24, 27, 28], "nowadai": [0, 1, 3, 9, 20, 27], "nox": [], "np": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 24, 27, 28, 29, 30], "npr": 2, "nsampl": 6, "nt": 2, "nu": 24, "nuclear": [5, 28, 29], "nuclei": [0, 24, 27], "nucleon": [0, 27], "nucleu": [0, 27], "num": 4, "num_coordin": 2, "num_hidden_neuron": 2, "num_it": [2, 18], "num_neuron": 2, "num_neurons_hidden": 2, "num_point": 2, "num_tre": 10, "num_valu": 2, "number": [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 21, 22, 23, 25, 27, 29], "numberid": 7, "numberparamet": 3, "numer": [0, 5, 6, 9, 10, 11, 12, 13, 20, 21, 26, 27, 28, 29, 30], "numpi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 28, 29, 30], "nunmpi": [5, 28], "nve_frngahw": 29, "nx": 2, "ny": 24, "o": [0, 1, 4, 5, 6, 7, 8, 9, 11, 21, 25, 26, 27, 28, 29, 30], "obei": [6, 11, 13, 28, 30], "object": [0, 1, 4, 8, 10, 15, 19, 21, 27, 30], "obliqu": [5, 28, 29], "observ": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 24, 27, 29, 30], "obtain": [0, 1, 5, 6, 7, 8, 9, 10, 12, 13, 14, 17, 21, 22, 24, 27, 28, 29, 30], "obviou": [5, 6, 11, 24, 28, 29], "obviouli": 27, "obvious": [0, 4, 5, 6, 21, 27], "oc": [28, 29], "occupi": [], "occur": [0, 6, 8, 9, 21, 24, 27], "octob": [25, 27], "od": 0, "odd": [0, 3, 7, 27, 28, 30], "odenum": 2, "odesi": 2, "oen": 0, "off": [1, 3, 4, 5, 9, 13, 24], "offer": [6, 11, 20, 21, 23, 25, 27], "offic": [25, 27], "offici": [23, 27], "often": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 22, 24, 27, 28, 29, 30], "ofter": [21, 27], "ol": [0, 13, 17, 19, 28, 30], "old": [1, 5, 10, 13, 15, 18], "ols_paramet": 16, "ols_sk": 6, "ols_svd": 6, "olsbeta": 29, "olstheta": [0, 5], "omega": [2, 3, 6], "omega_0": 3, "omit": [0, 5, 27, 28, 29], "onc": [1, 6, 9, 11, 13, 30], "one": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 19, 20, 21, 22, 24, 25, 27, 28, 30], "onehot": 1, "onehot_vector": 1, "onehotencod": 9, "ones": [0, 2, 5, 6, 8, 9, 10, 11, 13, 16, 18, 21, 22, 27, 28, 29, 30], "ones_lik": 4, "ong": 28, "onl": 3, "onli": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 21, 22, 24, 27, 28, 29, 30], "onlin": [11, 15, 23], "onto": [5, 11, 28, 29], "open": [0, 1, 4, 6, 7, 9, 15, 20, 22, 23, 25, 27], "oper": [0, 1, 3, 5, 6, 10, 11, 12, 13, 15, 16, 20, 24, 27, 28, 29, 30], "operation": 24, "oplu": 24, "opmiz": [13, 30], "opportun": 0, "oppos": [6, 13], "opposit": [1, 5, 8, 28, 29], "opt": [1, 5, 22, 27, 29], "optim": [0, 2, 3, 4, 5, 6, 7, 9, 10, 11, 14, 16, 17, 19, 22], "optimis": [1, 3], "option": [0, 1, 3, 5, 6, 8, 11, 15, 18, 21, 28, 30], "optmiz": [1, 8, 13, 28], "oral": 27, "orang": 0, "order": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 15, 19, 21, 22, 24, 27, 28, 29], "ordinari": [0, 2, 3, 7, 11, 13, 17, 18, 20], "oreilli": [26, 27], "org": [0, 3, 4, 16, 20, 21, 22, 26, 27, 28, 29, 30], "organ": [6, 7, 10, 21], "orient": [1, 5, 24, 28, 29], "origin": [0, 3, 5, 6, 8, 11, 12, 13, 15, 21, 27, 28, 29], "orthogn": [5, 28, 29], "orthogon": [0, 5, 6, 8, 11, 13, 21, 27, 28, 29], "orthonorm": [5, 28, 29], "os": [25, 27], "oscar": 1, "oscil": [3, 13, 30], "oskar": 27, "oskarlei": 27, "osl": 18, "oslo": [0, 20, 22, 23, 25, 27, 28, 29, 30], "osx": [0, 20, 22, 27], "other": [0, 1, 2, 3, 5, 6, 7, 8, 10, 13, 14, 16, 20, 23, 24, 25, 26, 28, 29, 30], "otherwis": [0, 1, 4, 7, 13, 21, 27, 30], "ouput": [5, 7, 12], "our": [1, 2, 3, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 24, 30], "ourmodel": 0, "ourselv": [0, 5, 6, 8, 11, 13, 27, 28, 29], "out": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 20, 21, 22, 24, 27, 28, 30], "out_fil": 9, "outcom": [0, 7, 9, 10, 12, 24, 28], "outdoor": 9, "outer": [6, 12, 13], "outfil": 4, "outlier": [0, 8, 27, 28, 30], "outlin": [6, 10, 11], "outlook": 9, "outperform": [10, 30], "output": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 21, 22, 24, 27, 28, 29, 30], "output_bia": 1, "output_bias_gradi": 1, "output_shap": 4, "output_weight": 1, "output_weights_gradi": 1, "outputlayer1": 12, "outputlayer2": 12, "outsid": 4, "over": [0, 1, 3, 4, 5, 6, 9, 10, 12, 13, 15, 16, 21, 22, 27, 28, 29, 30], "over1": [13, 30], "overal": [1, 10, 30], "overcast": 9, "overcom": [12, 13], "overdetermin": [0, 27], "overfit": [0, 1, 3, 6, 9, 10, 13, 30], "overflow": 5, "overhead": 12, "overlap": [3, 7, 8, 9], "overlin": [0, 5, 6, 9, 10, 11, 14, 21, 27, 28, 30], "overshoot": 30, "overst": 0, "overtrain": 4, "overview": 3, "own": [4, 5, 6, 8, 12, 13, 16, 18, 20, 21, 29, 30], "owner": [], "ownmsepredict": 0, "ownmsetrain": 0, "ownridgebeta": 28, "ownridgetheta": [0, 6, 28, 29, 30], "ownypredictridg": 0, "ownytilderidg": 0, "oxid": [], "p": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 24, 27, 28, 29, 30], "p0": 2, "p1": 2, "p_": [2, 4, 8, 9], "p_hidden": 2, "p_i": [5, 24], "p_j": 24, "p_n": 24, "p_output": 2, "p_x": 24, "pack": [0, 27], "packag": [0, 1, 3, 4, 5, 8, 11, 13, 15, 20, 22, 24, 28, 29, 30], "packtpub": 27, "packtpublish": 27, "pad": [3, 4], "page": [0, 20, 27, 29, 30], "pai": [0, 1, 9, 13, 15, 30], "pair": [0, 2, 3, 9, 20, 24, 27], "paltform": 15, "panda": [0, 4, 5, 6, 7, 9, 11, 20, 22, 29, 30], "panel": 27, "paper": 1, "paradigm": [0, 27], "parallel": [10, 13, 20, 21, 27], "param": 2, "paramat": 2, "paramet": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 16, 17, 18, 19, 22, 24, 29], "parameter": [0, 6, 10, 27, 28], "parametr": [0, 6, 27, 28], "paramt": [3, 5], "part": [0, 1, 3, 5, 6, 10, 17, 19, 21, 23, 24, 25, 27, 28], "partial": [0, 1, 5, 6, 7, 8, 10, 11, 12, 13, 16, 24, 27, 28, 29, 30], "particip": [15, 20, 23, 25, 27], "particl": [0, 4, 13, 24, 27, 30], "particular": [0, 1, 2, 3, 5, 6, 9, 10, 11, 12, 13, 16, 22, 24, 26, 27, 28, 29, 30], "particularli": [5, 6, 8, 11, 13, 24, 28, 29, 30], "partit": [1, 4, 9], "partli": [6, 27], "partner": 15, "pass": [2, 3, 12, 14], "password": 22, "past": [10, 24, 30], "patch": [6, 24], "path": [0, 4, 6, 7, 9, 20, 27], "pathcollect": 17, "patient": 7, "patter": 4, "pattern": [0, 3, 4, 12, 26, 27], "pauli": [0, 27], "pc": [11, 15, 20], "pca": [0, 7, 20, 27, 28], "pd": [0, 4, 5, 6, 7, 9, 11, 27, 28, 29, 30], "pde": 2, "pdf": [0, 3, 4, 5, 6, 9, 15, 16, 22, 26, 27], "pedagog": [0, 27, 28], "penal": [6, 18, 28, 30], "penalti": [6, 13, 18, 22, 28, 30], "penros": [5, 6], "pentagon": [13, 29], "peopl": [1, 9, 13, 20, 30], "per": [0, 1, 6, 23, 25, 27, 30], "percentag": [10, 11, 25], "perceptron": [0, 1, 7, 27], "peregrin": 27, "perfect": [0, 1, 13, 27, 30], "perfectli": [4, 6], "perform": [0, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "performac": 4, "perhap": [0, 5, 13, 27, 28, 29], "perimet": 1, "period": [1, 4, 24], "permiss": 15, "permut": 11, "persist": [13, 30], "person": [5, 6, 7, 16, 23, 25, 27, 28], "perspect": 26, "pertin": [12, 27], "petal": [8, 9], "peter": [26, 28], "phantom": 24, "phase": [6, 12], "phenomena": 24, "phi": 8, "phi_k": 8, "philosophi": 13, "phone": [25, 27], "photo": [4, 27], "php": 22, "phrase": [0, 27], "physic": [0, 1, 4, 7, 12, 13, 24, 25, 26, 27, 28, 29, 30], "pi": [2, 3, 5, 6, 7, 9, 12, 13, 24], "pick": [1, 9, 10, 11, 13, 14, 30], "pickl": 1, "pictur": [0, 27], "pie": [20, 27], "piec": [11, 14], "pillow": [0, 20, 22, 27], "pinv": [5, 6, 13, 22, 28, 29, 30], "pip": [0, 1, 15, 20, 22, 27], "pip3": [0, 1, 22, 27], "pipelin": [0, 6, 8, 10, 28], "pippin": 27, "pit": 4, "pitfal": [6, 28], "pitt": 12, "pixel": [1, 3, 4, 27], "pixel_height": [1, 3], "pixel_width": [1, 3], "place": [0, 4, 6, 8, 13, 15, 21, 22, 27, 29], "plai": [0, 3, 4, 5, 6, 8, 11, 18, 20, 22, 27, 28, 29], "plain": [8, 10, 12, 13, 14, 29, 30], "plan": [6, 9, 25, 26, 27], "plane": [8, 9], "plateau": [5, 29], "platform": [20, 27], "plausibl": 12, "pleas": [13, 22, 25, 27], "plenti": 1, "plethora": [3, 12], "plot": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 20, 21, 22, 24, 27, 28, 29, 30], "plot_all_sc": [22, 28], "plot_confusion_matrix": [7, 10], "plot_count": 6, "plot_cumulative_gain": [7, 10], "plot_data": 1, "plot_dataset": 8, "plot_decision_boundari": [9, 10], "plot_import": 10, "plot_max": 4, "plot_min": 4, "plot_model": 4, "plot_numb": 4, "plot_predict": 8, "plot_regression_predict": 9, "plot_result": 4, "plot_roc": [7, 10], "plot_surfac": [2, 6, 13], "plot_train": 9, "plot_tre": [9, 10], "plt": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 24, 27, 28, 29, 30], "plu": [0, 3, 5, 7, 18, 27, 28], "pm": 8, "pmatrix": 2, "pml": 26, "pn": 3, "png": [0, 4, 6, 7, 9, 27], "point": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 19, 21, 22, 24, 25, 27, 28, 29, 30], "point_1": 4, "point_2": 4, "poisson": [20, 24, 27], "poli": [6, 8], "poly100_kernel_svm_clf": 8, "poly3": 0, "poly3_plot": 0, "poly_featur": [8, 9, 15], "poly_features10": 9, "poly_fit": 9, "poly_fit10": 9, "poly_kernel_svm_clf": 8, "poly_model": 15, "poly_ms": 15, "poly_predict": 15, "polydegre": [0, 5, 6, 10, 28], "polygon": [13, 29], "polym": 12, "polymi": 22, "polynomi": [0, 5, 6, 7, 8, 9, 10, 11, 15, 17, 19, 22, 27, 28], "polynomial_featur": [6, 15, 16, 17], "polynomial_svm_clf": 8, "polynomialfeatur": [0, 6, 8, 9, 15, 16, 19, 28], "polytrop": [0, 6], "pool": 3, "pool_siz": 3, "poor": [1, 13, 29, 30], "poorli": [0, 28], "popul": [0, 5, 27, 28], "popular": [0, 1, 3, 6, 7, 8, 9, 11, 12, 15, 20, 21, 22, 24, 28], "popularli": [0, 27], "portabl": 10, "portion": [11, 13, 30], "pose": [0, 4, 5, 6, 11, 24, 27], "posit": [0, 1, 2, 3, 5, 7, 8, 10, 11, 13, 14, 21, 24, 27, 28, 29, 30], "possibl": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 22, 24, 25, 27, 28, 29, 30], "possibli": [6, 8, 13, 22], "posterior": 5, "postpon": [0, 28], "postscript": 22, "postul": 5, "potenti": [0, 3, 5, 6, 12, 13, 28, 30], "pott": 12, "power": [0, 1, 5, 6, 8, 9, 12, 13, 27, 28, 29, 30], "pp": [5, 6, 19], "practic": [0, 5, 6, 7, 8, 16, 18, 22, 24, 28], "practition": [0, 1, 3, 27], "pre": 27, "preced": [1, 11, 12, 24], "preceed": 4, "preceq": 8, "precis": [0, 2, 5, 11, 13, 21, 22, 24, 27, 28], "pred": 6, "predicit": 0, "predict": [0, 1, 5, 6, 7, 8, 9, 10, 15, 16, 17, 19, 20, 22, 26, 27, 28, 29, 30], "predict_prob": 1, "predict_proba": [7, 10], "predictor": [0, 5, 6, 7, 9, 10, 11, 27, 28, 30], "prefer": [0, 1, 6, 8, 9, 11, 13, 15, 20, 22, 27], "prepar": [0, 6, 21, 22, 27, 28], "preprocess": [0, 4, 6, 7, 8, 9, 10, 11, 15, 16, 17, 18, 19, 22], "prerequisit": 0, "prescript": 22, "presenc": [13, 30], "present": [0, 5, 6, 7, 9, 12, 13, 21, 22, 24, 27, 28, 29, 30], "preserv": [3, 11, 21], "press": [13, 15, 26, 29], "pretrain": [1, 4], "pretti": [0, 4, 8, 9, 20, 22, 27], "prev_centroid": 14, "prevent": [13, 24, 30], "previou": [0, 1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 21, 22, 24, 28, 29, 30], "previous": [2, 3, 9, 10, 24], "price": [0, 4, 9, 13, 30], "primal": 8, "primari": [0, 7, 27], "prime": 24, "princip": [0, 5, 7, 20, 27, 28, 29], "principl": [0, 6, 7, 8, 14, 27], "print": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 18, 21, 24, 27, 28, 29, 30], "print_funct": [8, 9], "printout": [0, 27], "prior": [0, 5, 6, 27], "privat": 0, "prob": [1, 24], "probabilist": [0, 26, 27, 28], "probabl": [0, 1, 3, 4, 6, 7, 10, 13, 20, 27, 28, 30], "problem": [0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 17, 19, 20, 21, 22, 24], "probml": 26, "proce": [0, 5, 6, 7, 8, 9, 10, 11, 13, 21, 27, 28], "procedur": [2, 4, 5, 6, 8, 10, 11, 13, 28, 29, 30], "proceed": 21, "process": [0, 2, 4, 6, 9, 10, 12, 13, 20, 21, 22, 24, 26, 27, 29, 30], "prod": 26, "prod_": [1, 5, 7], "produc": [0, 3, 4, 5, 6, 9, 10, 11, 12, 13, 18, 20, 21, 24, 27, 28], "product": [0, 1, 3, 5, 6, 7, 8, 12, 13, 16, 17, 20, 21, 27, 28, 30], "profess": [0, 27], "program": [0, 1, 4, 5, 6, 8, 12, 14, 15, 20, 21, 23, 24, 25, 27, 28], "programm": 21, "progress": [1, 4, 14, 30], "prohibit": 6, "project": [0, 1, 2, 3, 5, 11, 13, 15, 19, 20, 23, 28, 29, 30], "project_root_dir": [0, 6, 7, 9, 27], "promin": 12, "promis": 8, "promot": [25, 27], "prone": [9, 15], "pronounc": [13, 20, 27, 30], "proof": [0, 11, 12, 13, 27, 29], "propag": [2, 3, 13, 30], "proper": [0, 2, 6, 7], "properli": [1, 6, 8, 10, 13, 18, 22, 30], "properti": [0, 1, 3, 12, 13, 16, 21, 27], "proport": [0, 1, 5, 9, 11, 13, 24, 27, 28, 30], "propos": [1, 4, 6, 10, 22, 27, 30], "propto": [5, 13, 29, 30], "proton": [0, 27], "prove": [3, 13, 29, 30], "provid": [0, 1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 20, 21, 22, 24, 27, 28, 29, 30], "proxi": [1, 13, 30], "prune": 9, "pseudo": [21, 24, 30], "pseudocod": 22, "pseudoinv": 5, "pseudoinvers": [5, 6, 22], "pseudorandom": [6, 24], "psychologi": [0, 27], "pt": 13, "public": [0, 15, 20, 27], "pull": 15, "punish": [0, 1, 27], "pure": [3, 9, 24], "purest": 9, "puriti": 9, "purpos": [0, 3, 10, 12, 14, 27], "push": 15, "put": 1, "py": 5, "pycod": 27, "pydata": 20, "pydot": 9, "pyhton2": 27, "pylab": [7, 27], "pypi": 20, "pyplot": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 21, 24, 27, 28, 29, 30], "pythagora": 5, "python": [1, 2, 3, 5, 6, 8, 11, 12, 13, 14, 18, 22, 24, 28, 30], "python2": [0, 22], "python3": [0, 20, 22, 27], "pytorch": [0, 20, 22, 27], "q": [5, 6, 8, 11, 24], "qp": 8, "qquad": [2, 11, 13, 21, 30], "qr": [5, 6, 21, 28, 29], "quad": [1, 13, 21], "quadrat": [0, 8, 9, 13, 27], "qualit": [4, 9, 22, 24], "qualiti": [0, 9, 20, 27, 28], "quantifi": 1, "quantil": 10, "quantit": [0, 6, 9, 22, 27], "quantiti": [0, 2, 5, 6, 7, 9, 10, 11, 12, 14, 16, 21, 24, 27, 28, 29, 30], "quantum": [4, 12, 26, 27], "quartil": [0, 28, 30], "quench": 5, "queri": 9, "question": [0, 5, 6, 9, 11, 12, 13, 22, 25, 27, 28, 30], "qugan": 4, "quick": [4, 24], "quickli": [1, 3, 9, 11, 13, 29], "quit": [1, 5, 6, 9, 10, 12, 15, 28, 29], "quot": 4, "r": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 20, 21, 22, 24, 28, 29, 30], "r2": [0, 5, 6, 27, 28, 29], "r2_score": [0, 27], "r2score": [0, 27], "r_1": 9, "r_2": 9, "r_j": 9, "r_m": 9, "rad": [], "rade": [], "radial": [8, 12], "radioact": 24, "radiu": [0, 1, 28, 30], "rain": 9, "ramp": 1, "ran0": 24, "ran1": 24, "ran2": 24, "ran3": 24, "rand": [0, 4, 5, 6, 9, 10, 13, 15, 19, 21, 27, 28, 29, 30], "randint": [6, 9, 13, 30], "randn": [0, 1, 2, 5, 6, 9, 11, 13, 15, 18, 27, 28, 29, 30], "random": [0, 1, 2, 3, 4, 5, 6, 8, 9, 13, 14, 15, 16, 17, 18, 19, 20, 21, 27, 28, 29, 30], "random_forest_model": 10, "random_index": [13, 30], "random_indic": [1, 3], "random_st": [7, 8, 9, 10, 11], "randomforestclassifi": 10, "randomli": [1, 6, 9, 13, 14, 18, 29, 30], "rang": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 18, 19, 21, 24, 27, 28, 29, 30], "rangl": [0, 6, 11, 24, 27, 28], "rangle_x": 24, "rank": [5, 28, 29], "rankdir": 4, "raphson": [1, 8, 13], "rapidli": 0, "rare": [1, 13, 30], "raschka": [27, 28], "rasckha": 27, "rashcka": [29, 30], "rate": [1, 2, 3, 4, 8, 9, 10, 12, 13, 18, 29], "rather": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 21, 24, 27, 28, 29, 30], "ratio": [4, 7, 9, 10, 11], "rational": [0, 27], "ravel": [5, 6, 7, 8, 9, 10, 11, 13, 21], "raw": 3, "rbf": [8, 11, 12], "rbf_kernel_svm_clf": 8, "rbf_pca": 11, "rc": 24, "rcond": [0, 27, 28], "rcparam": [1, 3, 7, 8, 9, 10, 24, 27], "re": [2, 4, 13, 15, 29], "reach": [1, 4, 5, 6, 9, 10, 12, 13, 14, 29, 30], "read": [0, 2, 3, 4, 5, 6, 7, 8, 11, 12, 16, 17, 21, 22, 24, 26, 29], "read_csv": [0, 6, 7, 9], "read_fwf": [0, 27], "reader": [0, 6, 21, 24, 27, 28, 30], "readi": [0, 1, 5, 6, 8, 10, 11, 12, 21, 27], "readili": 1, "readm": 15, "readthedoc": 20, "real": [0, 1, 4, 7, 10, 11, 12, 16, 18, 19, 21, 28], "real_loss": 4, "real_output": 4, "realist": [8, 27], "realiti": 24, "realiz": [1, 12], "realli": [0, 1, 27], "rearrang": [13, 30], "reason": [0, 1, 3, 4, 10, 13, 26, 27, 29, 30], "reassign": 1, "recal": [5, 6, 9, 10, 11, 12, 21, 24, 27, 28, 29, 30], "recast": 3, "receiv": [1, 3, 10, 12, 24], "recent": [0, 6, 13, 26, 30], "recept": [3, 12], "receptive_field": 3, "recip": [0, 6, 7, 21, 22, 27, 28], "reciproc": 5, "recogn": [0, 4, 5, 10, 27], "recognit": [0, 1, 3, 12, 26, 27], "recommen": 27, "recommend": [0, 2, 3, 4, 5, 6, 8, 13, 15, 19, 20, 21, 22, 26, 29, 30], "reconsid": 9, "reconstruct": 11, "record": [10, 22, 23, 25, 27], "recreat": 15, "rectangl": [9, 13, 29], "rectangular": [5, 28, 29], "rectifi": [1, 3, 12], "recur": [0, 20, 27], "recurr": [0, 1, 20, 27], "recurs": [9, 20, 21, 27], "red": [0, 3, 4, 6, 8, 9, 30], "redefin": [0, 10, 27, 28, 29], "redefinit": 29, "reduc": [1, 3, 5, 6, 9, 10, 11, 13, 27, 29, 30], "reduct": [0, 10, 11, 20, 24, 27, 28], "reegress": 22, "refer": [0, 1, 2, 3, 5, 6, 11, 12, 13, 14, 21, 26, 27, 28, 29, 30], "referenc": 2, "refin": 12, "refit": 6, "reflect": [0, 1, 4, 5, 22, 24, 27], "refresh": [20, 27], "refreshprogrammingskil": 27, "reg": [10, 11], "regard": [1, 9, 13], "regardless": [12, 16], "region": [3, 4, 6, 9, 12, 22], "regist": [6, 24], "reglasso": [5, 29], "regr_1": [0, 9], "regr_2": [0, 9], "regr_3": [0, 9], "regress": [1, 8, 11, 12, 16, 20, 21], "regressor": [0, 7, 10], "regret": 30, "regridg": [0, 5, 6, 28, 29, 30], "regular": [0, 3, 4, 5, 6, 7, 9, 13, 17, 18, 25, 27, 28, 29, 30], "regularli": 15, "reilli": [0, 26, 27], "reinforc": [0, 8, 20, 27], "reiter": 1, "reject": 7, "rel": [0, 4, 6, 7, 9, 12, 13, 24, 27, 28, 30], "relat": [0, 1, 3, 4, 5, 11, 13, 14, 19, 21, 24, 27, 28, 29, 30], "relationship": [0, 4, 9, 18, 27], "relativeerror": [0, 27, 28], "releas": [1, 20, 27], "relev": [0, 1, 5, 7, 11, 20, 22, 24, 27, 29], "reli": [0, 6, 8], "reliabilti": 22, "reliabl": [7, 24], "relu": [3, 4, 27], "remain": [1, 2, 4, 6, 12, 21, 24, 28, 30], "remaind": 24, "reman": 2, "remark": 1, "rememb": [0, 8, 13, 21, 22, 27], "remind": [0, 5, 11, 13, 21, 24], "remot": 15, "remov": [4, 5, 6, 18, 28, 29, 30], "renam": 15, "render": [0, 27, 28], "reorder": [5, 7, 28, 29], "reorgan": [0, 27], "repeat": [0, 1, 3, 4, 5, 6, 9, 10, 11, 13, 14, 21, 22, 24, 27, 28, 29, 30], "repeated": 27, "repeatedli": [0, 6, 10, 13], "repet": 3, "repetit": [6, 27, 28], "rephras": [13, 29], "replac": [0, 1, 3, 4, 5, 6, 10, 12, 14, 20, 22, 27, 28, 29], "replica": 6, "repo": [15, 22], "report": [27, 30], "repositori": [4, 22, 27], "repres": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 22, 24, 27, 28, 29, 30], "represent": [0, 1, 3, 6, 24, 27], "representd": 3, "reproduc": [0, 5, 6, 9, 12, 15, 16, 18, 20, 24, 27, 28], "repuls": [0, 27], "request": [0, 13, 30], "requir": [0, 1, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 17, 18, 21, 27, 28, 29, 30], "res1": 2, "res2": 2, "res3": 2, "res_analyt": 2, "res_analytical1": 2, "res_analytical2": 2, "res_analytical3": 2, "resaml": 6, "resampl": [0, 7, 10, 20, 27, 28], "rescal": [0, 11, 12, 30], "rescu": 5, "reseach": 6, "research": [0, 4, 13, 20, 26, 27], "resembl": [6, 24], "reserv": [1, 5, 6, 24], "reshap": [0, 1, 2, 3, 4, 6, 8, 9, 10, 21, 27, 28], "residenti": [], "residu": [0, 5, 13, 27], "resiz": [5, 28, 29], "resourc": 27, "respect": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 22, 24, 27, 28, 29, 30], "respond": 12, "respons": [0, 7, 9, 12, 27, 28], "rest": [0, 5, 18, 28, 29, 30], "restat": [0, 12, 27], "restor": 4, "restored_discrimin": 4, "restored_gener": 4, "restrict": [0, 3, 9, 12, 27], "result": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 27, 30], "retail": [], "retain": [5, 6, 28, 29, 30], "return": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 21, 24, 27, 28, 29, 30], "return_data": 14, "return_sequ": 4, "return_x_i": 9, "reus": [1, 3, 6, 19, 22], "reveal": [0, 12, 27], "revers": [1, 21], "review": [20, 21], "revisit": 14, "revolut": 27, "reward": [0, 4, 27], "rewrit": [0, 3, 5, 6, 7, 8, 10, 11, 12, 13, 16, 19, 21, 22, 24, 29, 30], "rewritten": [2, 6, 8, 10, 24], "rewrot": 13, "rf": 10, "rgb": 3, "rgoj5yh7evk": 20, "rh": 6, "rho": [0, 10, 13, 30], "rho_1": 10, "rho_2": 10, "rho_m": 10, "rich": [0, 27], "ride": 9, "rideclass": 9, "ridedata": 9, "ridg": [7, 11, 13, 20, 27], "ridge_paramet": 17, "ridge_sk": 6, "ridgebeta": 29, "ridgetheta": 5, "right": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 19, 21, 22, 24, 27, 28, 29, 30], "right_sid": 2, "rightarrow": [0, 1, 5, 6, 8, 11, 12, 13, 24, 27, 28, 29, 30], "rigor": [0, 27, 28, 29], "ring": 6, "rise": [0, 27], "risk": [0, 13, 27, 29, 30], "rival": 4, "river": [], "rlm": 27, "rm": 24, "rmse": [], "rmsporp": [13, 30], "rmsprop": [1, 3, 4, 13, 22], "rnd_clf": 10, "rng": 24, "rnn": [4, 12], "rnn1": 4, "rnn2": 4, "rnn_2layer": 4, "rnn_input": 4, "rnn_output": 4, "rnn_train": 4, "rntrick1": 24, "rntrick2": 24, "rntrick3": 24, "rntrick4": 24, "ro": [0, 13, 27, 29, 30], "robert": [19, 22, 26], "robust": [0, 27, 30], "robustscal": [0, 28, 30], "roc": [7, 10], "role": [0, 2, 5, 6, 8, 18, 20, 22, 27, 28, 29, 30], "roll": 6, "room": [0, 25, 27], "root": [0, 5, 9, 13, 15, 24, 28, 29, 30], "rot": 27, "rotat": [1, 8, 9, 10], "rotation_matrix": 9, "roughli": [1, 3, 18], "round": [7, 9, 13], "routin": [13, 21, 27, 29], "row": [0, 1, 2, 5, 6, 9, 11, 16, 21, 27, 28, 29], "rr": [5, 28, 29], "rrr": [5, 28, 29], "rudg": [], "rug": [13, 29, 30], "rule": [0, 1, 5, 6, 13, 22, 27, 28, 29], "run": [0, 1, 2, 4, 5, 6, 8, 9, 11, 13, 15, 20, 22, 27, 28, 29, 30], "runtim": [1, 6, 14, 15], "rust": [0, 20, 21, 27], "rvert": 1, "rvert_2": 1, "s_": [3, 6], "s_1": 6, "s_i": [6, 7], "s_j": 6, "s_k": 6, "s_phenomenon": 22, "saddl": [13, 29, 30], "safeguard": [18, 30], "sai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 19, 21, 22, 24, 27, 28, 29], "said": [6, 9, 13, 29], "sake": [0, 5, 7, 11, 27, 28, 29], "sale": [0, 27], "sam": 27, "same": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 14, 15, 16, 18, 21, 22, 24, 27, 28, 29], "samm": 10, "sampl": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 18, 19, 20, 21, 22, 24, 27, 28, 30], "sample_vari": 14, "sampleexptvari": 24, "samwis": 27, "sastri": 11, "satisfactori": [0, 27], "satisfi": [1, 2, 3, 6, 8, 13, 21, 24, 29], "satur": [1, 6], "save": [0, 4, 6, 7, 9, 13, 27, 30], "save_fig": [0, 6, 7, 9, 10, 27], "savefig": [0, 4, 6, 7, 9, 24, 27], "savetxt": 4, "saw": [5, 28], "scalabl": 10, "scalar": [2, 5, 6, 10, 28], "scale": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 20, 21, 22, 25, 27, 29], "scale_mean": 4, "scale_std": 4, "scaler": [0, 7, 8, 9, 10, 11, 17, 22, 28], "scan": [5, 7], "scari": 5, "scatter": [0, 1, 6, 7, 8, 9, 14, 15, 17, 27, 28, 30], "scenario": [6, 13, 29, 30], "schedul": [13, 30], "scheme": [1, 13, 29, 30], "schrage": 24, "sch\u00f8yen": [6, 28, 30], "scienc": [0, 1, 10, 12, 13, 20, 23, 24, 25, 26, 29], "scientif": [0, 20, 22, 27], "scientist": [0, 27], "scikit": [3, 5, 6, 8, 9, 10, 13, 15, 16, 20, 21, 22, 26], "scikit_learn": 0, "scikitlearn": 27, "scikitplot": [7, 10], "scipi": [0, 3, 5, 6, 13, 20, 21, 22, 27, 28, 29], "scl": 6, "scm": 15, "score": [0, 1, 3, 6, 7, 9, 10, 11, 15, 16, 22, 25, 27, 28, 30], "scores_kfold": 6, "scratch": [1, 13, 16], "sdg": [13, 30], "sdv4f4s2sb8": [29, 30], "seaborn": [0, 1, 3, 6, 7, 27], "seamless": [0, 20, 22, 27], "search": [0, 1, 3, 5, 9, 13, 15, 27, 29, 30], "sebastian": 27, "sebastianraschka": 27, "sec": 6, "second": [0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 15, 16, 20, 21, 24, 25, 27, 28, 29], "second_mo": 30, "second_term": 30, "secondeigvector": 11, "secondli": 12, "section": [4, 11, 16, 21, 22, 24, 28, 30], "sector": 0, "see": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 22, 24, 27, 28, 29, 30], "seed": [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 18, 24, 27, 28, 29, 30], "seed_imag": 4, "seek": [1, 2, 8], "seem": [1, 3, 4, 30], "seemingli": [0, 27], "seen": [0, 1, 3, 5, 10, 12, 24], "segment": [13, 29], "seismic": 6, "seldomli": [0, 27], "select": [1, 5, 6, 8, 9, 10, 11, 15, 22, 23, 24, 25, 26, 27, 28, 29], "selevet": 15, "self": [1, 5, 28], "sell": 4, "semest": [7, 23], "semi": [8, 13, 29, 30], "semilogx": 6, "send": [5, 12, 13, 25, 27], "senior": [23, 25], "sens": [0, 4, 6, 8, 27], "sensibl": 3, "sensit": [0, 5, 6, 9, 13, 27, 28, 30], "sent": 2, "sentenc": [4, 12], "separ": [0, 1, 2, 4, 6, 8, 9, 12, 14, 18, 20, 22, 24, 27, 30], "septemb": [18, 22, 27], "sequenc": [3, 4, 7, 9, 10, 12, 13, 20, 21, 24, 27, 29], "sequenti": [1, 3, 4, 10, 12, 24], "seri": [0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 13, 21, 27, 28, 29], "serif": [7, 24, 27], "serv": [0, 1, 2, 3, 5, 7, 13, 26, 27, 28, 29, 30], "servic": 22, "session": [1, 15, 22, 23, 25, 27], "set": [1, 4, 5, 6, 7, 8, 10, 11, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 30], "set_major_formatt": 6, "set_major_loc": 6, "set_tick": [1, 8], "set_ticklabel": 1, "set_titl": [0, 1, 2, 3, 7, 12, 14, 27], "set_xlabel": [0, 1, 2, 3, 7, 12, 27], "set_xlim": [7, 12], "set_xticklabel": 1, "set_ylabel": [0, 1, 2, 3, 7, 27], "set_ylim": [7, 12], "set_ytick": 7, "set_yticklabel": [1, 6], "set_zlim": 6, "seth": 4, "setminu": 6, "setosa": [8, 9], "setosa_or_versicolor": 8, "setp": 6, "setup": [1, 4, 6, 8, 20, 27, 28, 29], "sever": [0, 3, 5, 6, 7, 8, 9, 11, 12, 13, 16, 20, 21, 22, 24, 27, 28, 29, 30], "sgd": [1, 3, 29], "sgd_clf": 8, "sgdclassifi": 8, "sgdreg": 13, "sgdregressor": 13, "sgn": [5, 28, 29], "shallow": [13, 30], "shape": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 21, 27, 28, 29, 30], "share": [1, 3, 15, 27], "shareabl": 15, "she": 7, "shift": [1, 6, 12, 15, 18, 24, 28, 30], "ship": 3, "shire": 27, "short": [4, 5, 22], "shortcom": [13, 29, 30], "shorten": 4, "shorter": 24, "shorthand": 27, "shortli": [21, 27], "should": [0, 2, 3, 5, 6, 8, 9, 11, 12, 15, 18, 21, 22, 24, 27, 28, 30], "show": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 21, 22, 24, 27, 28, 29, 30], "show_shap": 4, "shown": [0, 4, 5, 8, 12, 13, 21, 28, 29, 30], "shrink": [3, 5, 6, 8, 11, 28, 29], "shrinkag": [5, 6, 28, 29], "shrunk": 11, "shuffl": [0, 1, 4, 6, 13, 28, 30], "side": [0, 2, 5, 8, 12, 13, 21, 22, 27, 29], "sigh": [20, 27], "sigma": [0, 1, 5, 6, 7, 10, 11, 12, 13, 19, 21, 22, 24, 27, 28, 29, 30], "sigma0": 24, "sigma1": 24, "sigma2": 24, "sigma_": [5, 21, 27, 28, 29], "sigma_0": [5, 28, 29], "sigma_1": [5, 28, 29], "sigma_2": [5, 28, 29], "sigma_fn": [7, 12], "sigma_i": [0, 5, 27, 28, 29], "sigma_j": [5, 28, 29], "sigma_m": [6, 24], "sigma_n": [11, 24], "sigma_t": [13, 30], "sigma_x": 24, "sigmoid": [1, 2, 4, 7, 8, 10, 12], "sigmundson": [6, 28, 30], "sign": [1, 2, 7, 8, 10, 24, 25], "signal": [1, 3, 10, 12, 30], "signifi": 4, "signific": [1, 30], "significantli": [1, 13, 18, 24, 29, 30], "sim": [4, 5, 6, 13, 19, 24, 30], "similar": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 18, 20, 21, 22, 27, 29], "similarli": [0, 1, 3, 5, 8, 10, 13, 24, 27, 28, 29, 30], "simpl": [1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 14, 16, 17, 20, 21, 24], "simplepredict": 10, "simpler": [0, 1, 5, 6, 7, 13, 16, 20, 22, 27, 29, 30], "simplernn": 4, "simplest": [0, 1, 3, 4, 9, 10, 12, 14, 22, 27], "simpletre": 10, "simpli": [0, 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, 20, 21, 22, 24, 27, 28, 29, 30], "simplic": [2, 5, 6, 7, 8, 9, 10, 11, 12, 14, 28, 29, 30], "simplicti": [5, 28, 29], "simplifi": [0, 6, 9, 18, 20, 22, 27, 28, 30], "simplist": [3, 6, 24], "simul": [6, 18], "simultan": 6, "sin": [0, 1, 2, 3, 4, 9, 12, 13, 21, 27], "sinc": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13, 16, 18, 21, 22, 24, 26, 27, 28, 29, 30], "sine": [3, 12], "singl": [0, 1, 2, 3, 5, 6, 7, 8, 9, 12, 13, 18, 19, 21, 24, 27, 28, 29, 30], "singular": [0, 6, 13, 21, 27], "sinusoid": 3, "site": [0, 22, 23, 28], "situat": [0, 4, 5, 7, 13, 24, 27, 28, 29, 30], "six": [3, 24], "size": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 13, 18, 21, 22, 24, 27], "sizesp": 30, "sketch": 10, "ski": 9, "skill": 0, "skip": 11, "skl": [0, 6, 27, 28, 30], "sklearn": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 19, 27, 28, 29, 30], "skplt": [7, 10], "sl": [6, 28, 30], "slack": 8, "slice": [2, 21, 27], "slide": [0, 3, 16, 22, 24, 27, 28, 29], "slight": [6, 13, 30], "slightli": [1, 2, 3, 5, 6, 7, 10, 24, 28, 29], "slope": [8, 11, 12], "slow": [0, 2, 8, 13, 18, 28, 29, 30], "slower": [5, 21, 27, 28, 29, 30], "slowest": 21, "slowli": [12, 30], "slp": 1, "small": [0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 18, 20, 21, 24, 27, 28, 29, 30], "smaller": [0, 1, 2, 5, 6, 8, 9, 11, 13, 24, 27, 28, 29, 30], "smallest": [0, 4, 14, 27], "smallest_row_index": 14, "smooth": [0, 3, 6, 13, 22, 27, 29, 30], "sn": [0, 1, 3, 6, 7, 27], "sne": 11, "so": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29, 30], "soar": 6, "social": 0, "soft": [1, 7, 10, 12], "soften": 8, "softmax": [3, 7], "softwar": [0, 8, 20, 21], "sol": 8, "sole": [0, 6, 27], "solid": [0, 7], "solut": [0, 1, 2, 3, 5, 6, 8, 10, 11, 13, 18, 21, 22, 24, 27, 28, 29, 30], "solution_ev": 30, "soluton": 2, "solv": [0, 1, 3, 5, 6, 8, 10, 11, 12, 13, 16, 21, 22, 27, 28], "solve_expdec": 2, "solve_ode_deep_neural_network": 2, "solve_ode_neural_network": 2, "solve_pde_deep_neural_network": 2, "solveod": 2, "solveode_popul": 2, "solver": [2, 7, 8, 9, 10, 21, 27], "some": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 22, 24, 27, 30], "some_model": [6, 28, 30], "somehow": 4, "someon": 16, "someth": [0, 1, 3, 4, 7, 9, 11, 15, 19, 22, 24, 27, 28], "sometim": [0, 1, 11, 12, 13, 14, 28, 30], "soon": [21, 25, 28], "sophist": [0, 27], "sopt": 13, "sort": [5, 6, 9, 11, 24], "sound": [3, 5], "sourc": [0, 1, 3, 6, 20, 21, 22, 24, 27], "space": [0, 1, 4, 5, 8, 9, 11, 12, 13, 14, 24, 28, 29, 30], "span": [0, 3, 5, 9, 11, 21, 27, 28, 29], "spare": 1, "spars": [3, 6, 18, 21, 27, 30], "sparse_mtx": [21, 27], "sparsecategoricalcrossentropi": 3, "sparsiti": [10, 18], "spatial": [1, 2, 3, 12], "speak": 24, "special": [6, 7, 10, 12, 13, 21, 24, 27, 28, 29, 30], "specif": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 15, 16, 20, 21, 22, 24, 26, 27, 28, 29], "specifi": [0, 3, 5, 6, 7, 9, 11, 13, 14, 24, 27, 29, 30], "specifici": [0, 10, 27], "spectacular": 3, "spectral": 1, "speech": [0, 1, 3, 4, 12], "speed": [1, 2, 4, 13, 30], "spend": [16, 24], "spent": 22, "sphere": [0, 28, 30], "spin": 6, "spite": 0, "spline": 8, "split": [1, 3, 4, 5, 6, 8, 9, 10, 11, 14, 16, 17, 22, 24, 27, 29, 30], "splite": 0, "splitter": [1, 10], "spontan": 24, "spot": 3, "spread": [0, 11, 24, 27, 28], "springer": [19, 22, 26, 27], "spuriou": [13, 30], "sqquar": 29, "sqrsignal": 3, "sqrt": [3, 4, 5, 6, 8, 10, 11, 13, 24, 28, 29, 30], "squar": [1, 2, 3, 4, 7, 8, 9, 11, 13, 14, 15, 17, 18, 20, 21, 24], "squarederror": 10, "squaredeuclidean": 14, "squash": 12, "srtm": 6, "srtm_data_norway_1": 6, "stabil": [5, 22, 30], "stabl": [0, 4, 5, 6, 9, 16, 20, 22, 27, 28, 29], "stack": [3, 4], "stage": [5, 13, 15, 22, 30], "stai": [0, 2, 4, 5, 11, 27, 28, 30], "stand": [0, 5, 9, 12, 27, 28, 29], "standard": [0, 1, 4, 5, 6, 7, 8, 10, 12, 17, 18, 21, 22, 24, 27, 29, 30], "standardscal": [0, 6, 7, 8, 9, 10, 11, 17, 28, 30], "stanford": [13, 29], "start": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 21, 24, 25, 27, 28, 29, 30], "start_tim": 14, "stat": 6, "state": [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 20, 24, 27, 28, 29], "statement": [0, 7, 21, 27], "stationari": [29, 30], "statist": [0, 1, 3, 4, 7, 9, 10, 11, 12, 13, 14, 19, 21, 22, 26, 28, 29], "statu": [0, 7, 15, 27], "stavang": 6, "std": [0, 4, 6, 18, 27, 28, 30], "steep": [13, 29, 30], "steepest": 30, "step": [0, 1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 15, 18, 21, 22, 27, 29], "step_fn": [7, 12], "step_length": [13, 30], "step_siz": 30, "steps_list": 9, "stereo": 3, "still": [0, 2, 3, 5, 6, 11, 13, 24, 28, 29, 30], "stimuli": 12, "stk": [26, 27], "stk2100": [26, 27], "stk3155": [15, 22, 23, 25], "stk4021": [26, 27], "stk4051": [26, 27], "stk4155": [23, 25], "stk5000": 26, "stochast": [0, 1, 5, 6, 8, 11, 12, 29], "stock": 4, "stoke": 12, "stone": [0, 7], "stop": [1, 4, 9, 13, 14, 18, 29], "storag": [5, 28, 29], "store": [0, 1, 2, 3, 6, 11, 13, 24, 27, 30], "storehaug": [25, 27], "str": [1, 3, 4], "straight": [0, 6, 8, 13, 27, 29], "straightforward": [0, 2, 3, 5, 6, 8, 9, 10, 13, 21, 27, 28, 29], "strategi": [0, 1, 9, 27], "stratifi": 6, "strength": [0, 5, 14, 28, 29], "stretch": 11, "strict": [8, 13, 29], "strictli": [8, 13, 29], "stride": [4, 21], "strike": 6, "string": 1, "stroke": 7, "strong": [3, 6, 9, 10, 12, 21, 24], "strongli": [0, 8, 15, 20, 21], "stronli": [], "structur": [0, 1, 2, 3, 6, 9, 10, 12, 20, 27], "stuck": [1, 13, 29, 30], "student": [0, 15, 22, 23, 25, 26, 27], "studi": [0, 3, 4, 5, 6, 7, 8, 11, 12, 13, 20, 22, 26, 27, 28, 29, 30], "studier": 26, "style": [7, 9, 21, 27], "st\u00f8land": 25, "sub": [9, 12, 30], "subdivid": [0, 21, 27], "subfield": 0, "subgradi": 30, "subject": [6, 8, 24], "submit": 27, "subplot": [0, 1, 3, 4, 6, 7, 8, 9, 10, 14, 27], "subplots_adjust": [8, 24], "subprogram": [21, 27], "subract": [0, 28], "subroutin": [0, 27], "subscript": 1, "subsequ": [1, 4, 5, 6, 12, 21, 24, 28, 29], "subset": [1, 6, 9, 12, 13, 20, 27, 29, 30], "subspac": [0, 8, 11, 28], "substanti": [9, 10], "substep": 11, "substitut": [3, 6, 12, 16, 21], "subsubset": 9, "subtask": 6, "subtl": 1, "subtract": [0, 4, 5, 6, 11, 13, 18, 21, 22, 24, 28, 30], "subtre": 9, "succeed": [0, 4, 27], "success": [3, 7, 9, 13, 24], "successfulli": [4, 9], "sudo": [0, 20, 22, 27], "suffer": [0, 1, 2, 5, 10, 27, 28, 29], "suffici": [1, 6, 8, 11, 13, 29], "suggest": [1, 13, 22, 26, 29, 30], "suit": [8, 12], "suitabl": [0, 15, 19, 24, 28], "sum": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19, 21, 24, 27, 28, 29, 30], "sum_": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 19, 21, 22, 24, 27, 28, 29, 30], "sum_i": [0, 2, 5, 6, 8, 13, 19, 22, 28, 29, 30], "sum_j": [6, 18, 30], "sum_ja_": 0, "sum_k": [6, 8, 12, 21], "sum_logist": 13, "sum_m": 3, "sum_n": 3, "sum_nx_": 3, "summar": [5, 6, 9], "summari": [1, 3, 4, 10, 23, 29], "summat": [0, 3, 16, 28, 29], "sunni": 9, "super": [5, 28, 29], "superfici": 3, "superscript": [1, 12], "supervis": [0, 5, 6, 7, 9, 12, 20, 27, 28, 29], "supplement": [7, 22], "support": [0, 1, 9, 10, 11, 13, 20, 27, 28], "suppos": [0, 5, 6, 7, 8, 10, 11, 12, 13, 21, 27, 28, 29, 30], "suppress": [5, 13, 29, 30], "sure": [0, 1, 4, 6, 16, 22], "surf": 6, "surfac": [0, 6, 27], "surpass": 6, "surpris": [0, 27], "surround": [3, 20], "survei": [0, 5, 6, 27, 28], "svc": [8, 9, 10], "svd": [0, 6, 11, 27], "svdinv": 5, "svm": [8, 9, 10, 11], "svm_clf": [8, 10], "swath": [5, 28, 29], "switch": 0, "sy": [13, 29, 30], "symbol": [1, 5, 11, 13, 20, 24, 27, 28, 29], "symmeteri": 1, "symmetr": [0, 5, 8, 11, 12, 13, 21, 27, 28], "symmetri": 6, "sympi": [0, 20, 22, 27], "synonim": 24, "syntax": 13, "system": [0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 20, 21, 22, 27, 29], "systemat": [4, 6], "t": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 29, 30], "t0": [3, 6, 13, 30], "t1": [2, 13, 30], "t2": 2, "t3": 2, "t_": 2, "t_0": [2, 9, 13, 30], "t_1": [13, 30], "t_b": 10, "t_i": [1, 2, 5, 12, 28, 29], "t_j": 12, "t_k": 9, "tabl": [9, 22, 24, 25, 27], "tabul": [0, 27], "tabular": 27, "tackl": 4, "tag": [2, 3, 4, 5, 6, 7, 12, 13, 14, 21, 24, 28, 29, 30], "taht": [0, 27], "tail": 24, "tailor": [2, 8, 11, 27], "taiwan": [0, 27], "take": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 20, 21, 24, 27, 28, 29, 30], "taken": [0, 1, 3, 6, 10, 13, 21], "tan": 3, "tangent": [1, 4, 12, 13, 29], "tanh": [1, 4, 7, 8, 12], "target": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 18, 19, 27, 28, 29, 30], "target_nam": 9, "task": [0, 1, 3, 6, 9, 11, 12, 14, 22, 27], "tau": [3, 5, 24], "taught": 27, "tax": [], "taylor": [2, 13, 29], "taylornr": [13, 29], "tc": 8, "teach": [15, 23, 27], "team": 1, "teaser": 0, "technic": [0, 5, 6, 13, 22, 29, 30], "techniqu": [0, 1, 8, 10, 13, 20, 24, 26, 27, 28, 30], "technologi": [0, 1], "tell": [0, 4, 6, 10, 11, 13, 16, 24, 30], "temp": 1, "temp1": 1, "temp2": 1, "temperatur": [0, 9, 27], "templat": 18, "temporarili": 1, "ten": [3, 27], "tend": [3, 5, 6, 8, 9, 10, 12, 13, 14, 28, 30], "tendenc": [0, 27], "tension": 6, "tensor": 3, "tensorflow": [0, 2, 4, 8, 14, 20, 21, 22, 26, 27, 28], "term": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 22, 24, 27, 28, 29, 30], "term1": [5, 6, 11], "term2": [5, 6, 11], "term3": [5, 6, 11], "term4": [5, 6, 11], "termin": [0, 4, 5, 9, 10, 13, 15, 28, 29, 30], "terminarl": 15, "terrain": 6, "terrain1": 6, "test": [3, 4, 5, 6, 7, 8, 9, 10, 13, 16, 21, 22, 24, 27, 29, 30], "test_acc": 3, "test_accuraci": [1, 3], "test_error": 6, "test_imag": [3, 4], "test_ind": 6, "test_input": 4, "test_label": [3, 4], "test_loss": 3, "test_pr": 1, "test_predict": 1, "test_rnn": 4, "test_scor": [7, 10], "test_siz": [0, 1, 3, 5, 6, 10, 15, 17, 28, 29, 30], "test_split": 9, "testerror": [0, 6, 28], "testi": 4, "testpredict": 4, "testx": 4, "text": [0, 1, 2, 4, 5, 8, 9, 11, 13, 15, 18, 21, 24, 26, 28, 29, 30], "textbf": 30, "textbook": [16, 22, 28, 29], "textual": 9, "textur": 1, "tf": [1, 3, 4, 13, 14, 29], "th": [0, 1, 2, 5, 6, 7, 9, 12, 13, 14, 21, 22, 24, 27, 28, 30], "than": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 17, 20, 24, 27, 28, 30], "thank": [4, 6, 28, 30], "theano": [1, 20, 27], "thei": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 18, 21, 22, 24, 27, 28, 29, 30], "them": [0, 1, 3, 4, 6, 8, 9, 10, 11, 12, 13, 18, 19, 21, 22, 27, 28], "theme": [0, 15, 27], "themselv": [0, 22, 24, 27], "thenc": 6, "theorem": [2, 6, 7, 28, 29], "theoret": [0, 4, 10], "theori": [0, 1, 3, 8, 9, 12, 13, 19, 20, 22, 26, 27], "thereaft": [0, 5, 6, 11, 12, 21, 22, 27], "therebi": [0, 5, 7, 11, 22, 27, 28, 29], "therefor": [0, 1, 2, 3, 4, 6, 7, 8, 11, 13, 19, 24, 27, 28, 29, 30], "therein": 11, "thereof": [0, 6, 13, 27, 30], "theta": [0, 1, 4, 5, 6, 7, 13, 16, 22, 24, 27, 28, 29, 30], "theta_": [0, 1, 6, 7, 13, 27, 28, 29, 30], "theta_0": [0, 5, 6, 7, 16, 27, 28, 29, 30], "theta_0x_": [0, 27, 28], "theta_1": [0, 5, 6, 7, 27, 28, 29, 30], "theta_1x_": [0, 27, 28], "theta_1x_0": [0, 27], "theta_1x_1": [0, 7, 27], "theta_1x_2": [0, 27], "theta_1x_i": [7, 28, 29, 30], "theta_2": [0, 27, 28], "theta_2x_": [0, 27, 28], "theta_2x_0": [0, 27], "theta_2x_1": [0, 27], "theta_2x_2": [0, 7, 27], "theta_2x_i": 28, "theta_3x_i": 28, "theta_4x_i": 28, "theta_closed_form": 18, "theta_closed_formol": 18, "theta_closed_formridg": 18, "theta_gdol": 18, "theta_gdridg": 18, "theta_i": [0, 1, 5, 27, 28, 29], "theta_j": [0, 5, 6, 18, 27, 28, 30], "theta_k": [29, 30], "theta_linreg": [13, 29, 30], "theta_ol": 18, "theta_p": 7, "theta_px_p": 7, "theta_ridg": 18, "theta_t": [13, 30], "theta_tru": 18, "thetavalu": 5, "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 28, 29, 30], "thing": [0, 1, 2, 4, 5, 7, 9, 15, 16, 18, 24, 27], "think": [0, 1, 3, 4, 6, 9, 12, 13, 14, 24, 27, 28, 29, 30], "third": [0, 3, 6, 13, 25, 27, 29, 30], "thirti": 7, "thorughout": 27, "those": [0, 3, 5, 6, 8, 9, 10, 11, 21, 22, 27, 28, 29, 30], "though": [1, 2, 3, 4, 13, 16, 17, 21, 24, 30], "thought": [6, 14, 22, 24], "thousand": [0, 1, 22, 28, 30], "three": [0, 1, 3, 5, 6, 8, 9, 12, 21, 22, 23, 24, 25, 27, 28, 29], "threshold": [1, 3, 9, 10, 11, 12, 13, 30], "through": [0, 1, 2, 3, 4, 5, 6, 8, 11, 12, 13, 14, 15, 20, 21, 22, 24, 27, 28, 29, 30], "throughout": [0, 4, 5, 14, 15, 20, 21, 24, 27], "throw": [3, 6, 24], "thu": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 25, 27, 28, 29, 30], "thumb": [0, 6, 22, 28], "thursdai": [], "tibshirani": [6, 19, 22, 26, 27], "tick_param": 6, "ticker": [6, 13, 24, 29, 30], "tif": 6, "tight_layout": [1, 7], "tightli": 11, "tild": [0, 5, 6, 7, 11, 19, 22, 24, 27, 28, 29, 30], "till": [0, 4, 7, 8, 9, 10, 12, 21, 27, 28], "time": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 27, 28, 29], "timeit": 4, "timer": 4, "tini": 1, "tip": 3, "titl": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 13, 15, 24, 27, 29, 30], "tmp": 13, "tn": [2, 3, 7], "to_categor": [1, 3, 4], "to_categorical_numpi": 1, "to_numer": [0, 6, 27], "todai": 3, "togeth": [0, 3, 6, 8, 11, 13, 20, 27], "toi": 14, "told": 13, "toler": [2, 14], "tolist": 4, "tomographi": 12, "too": [0, 2, 4, 5, 6, 9, 11, 13, 17, 18, 24, 26, 28, 29, 30], "took": [8, 27], "tool": [0, 1, 3, 6, 13, 15, 20, 28], "toolbox": 8, "top": [0, 3, 5, 6, 9, 10, 19, 20, 27], "topic": [0, 5, 6, 7, 8, 20, 22, 28, 29], "topolog": [3, 12], "topologi": [1, 12], "torkjellsdatt": [25, 27], "toss": [10, 24], "total": [0, 1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 13, 14, 21, 24, 25, 27, 28, 29, 30], "total_loss": 4, "totalclustervari": 14, "totalscatt": 14, "toward": [1, 2, 7, 12, 13, 15, 29], "towardsdatasci": 30, "town": [], "tp": [4, 7], "tpng": 9, "tpu": [13, 20, 27], "tqdm": 6, "tr": [], "track": [3, 13, 14, 15, 21, 28, 29, 30], "tract": [], "tractabl": [0, 27, 28], "trade": [5, 9], "tradeoff": [0, 5, 19, 22, 27, 28, 29], "tradit": [0, 1, 4, 6, 27], "train": [2, 3, 5, 6, 8, 9, 10, 11, 12, 13, 16, 17, 22, 29, 30], "train_accuraci": [0, 1, 3, 27], "train_dataset": 4, "train_end": [0, 1, 28], "train_error": 6, "train_imag": [3, 4], "train_ind": 6, "train_label": [3, 4], "train_pr": 1, "train_siz": [0, 1, 3, 28], "train_step": 4, "train_test_split": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 27, 28, 29, 30], "train_test_split_numpi": [0, 1, 28], "trainable_vari": 4, "trained_model": [6, 28, 30], "trainerror": [0, 28], "traini": 4, "training_checkpoint": 4, "training_dataset": 4, "training_gradi": [13, 30], "trainingerror": 6, "trainpredict": 4, "trainscor": 4, "trainx": 4, "trait": [0, 27], "trajectori": 4, "transfer": [9, 27], "transform": [0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 20, 21, 27, 28, 29, 30], "transit": [6, 12], "translat": [1, 4, 6, 10, 27, 28, 30], "transpos": [1, 5, 11, 21, 28, 29], "travers": [0, 5], "treat": [0, 1, 3, 6, 12, 13, 18, 24, 27, 28, 29, 30], "tree": [0, 1, 20, 27], "tree_clf": [9, 10], "tree_clf_": 9, "tree_clf_sr": 9, "tree_reg": 9, "tree_reg1": 9, "tree_reg2": 9, "trend": 24, "treue": 7, "trevor": [19, 22, 26], "tri": [2, 3, 4, 9, 13, 16], "triain": 0, "trial": [0, 2, 4, 6, 13, 24, 27, 29, 30], "triangl": [13, 29], "triangular": 21, "trick": [3, 4, 8, 11, 13, 24, 30], "trickier": 24, "tridiagon": 21, "trillion": 20, "trivial": [0, 1, 5, 11, 24, 27, 29], "troubl": [0, 8, 12, 15, 28, 30], "truck": 3, "true": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 27, 28, 29, 30], "true_beta": 28, "true_fun": 6, "true_theta": [6, 30], "truli": 27, "try": [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 18, 20, 21, 22, 24, 27, 28, 29, 30], "tucker": 8, "tuesdai": [25, 27], "tumor": [7, 9], "tumour": 7, "tunabl": 1, "tune": [4, 9, 13, 21, 27, 30], "turn": [0, 1, 5, 6, 7, 8, 9, 10, 11, 12, 13, 21, 22, 24, 27, 28, 29, 30], "tutori": [1, 4], "tv": 2, "tveito": 2, "tweak": [1, 4, 10, 24], "twice": [13, 29], "twist": 11, "two": [0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 17, 21, 22, 23, 24, 26, 27, 28, 29, 30], "tx": [13, 29, 30], "tx_1": [13, 29], "txt": [4, 15], "ty": [13, 29], "type": [0, 1, 3, 6, 8, 10, 13, 21, 24, 28, 29, 30], "typic": [0, 1, 2, 3, 4, 5, 7, 9, 10, 12, 13, 15, 16, 24, 27, 28, 29, 30], "typo": 22, "u": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 21, 22, 24, 26, 27, 28, 29, 30], "u_": 21, "u_i": 12, "u_m": 10, "ua": [0, 27], "ubuntu": [0, 20, 22, 27], "uci": 22, "uio": [15, 22, 25, 26], "un": 14, "unabl": 15, "unari": [21, 27], "unbalanc": [6, 9], "unbias": [0, 5, 6, 27], "uncent": [6, 28, 30], "uncertainti": [0, 5, 27], "uncertitud": 24, "unchang": [1, 3], "uncorrel": [10, 24], "undefin": [5, 28, 29], "under": [0, 1, 5, 6, 10, 13, 20, 22, 27, 28, 29], "underdetermin": [0, 27], "underfit": [1, 6], "underflowproblem": 5, "undergo": 5, "undergradu": [23, 25], "underli": [0, 1, 9, 13, 18, 24, 27, 30], "underset": [4, 14], "understand": [0, 1, 3, 5, 6, 10, 13, 14, 15, 20, 27, 28, 29, 30], "understood": [8, 13, 30], "undesir": 8, "undetermin": [5, 8], "undo": 4, "unexpect": 6, "unexpected": 24, "unexplain": 18, "unfair": [6, 28], "unfortun": [1, 8, 9, 10], "unicode_liter": [8, 9], "uniform": [0, 1, 5, 6, 11, 13, 22, 24, 27, 29, 30], "uniformli": [13, 24, 29, 30], "unifrompdf": 24, "unimport": [13, 29], "union": [5, 6], "uniqu": [0, 2, 6, 13, 14, 21, 27], "unique_cluster_label": 14, "unit": [0, 1, 3, 4, 5, 10, 12, 18, 24, 27, 28, 29, 30], "unitari": [5, 6, 21, 28, 29], "unitarili": [21, 27], "uniti": 24, "univari": 24, "univers": [0, 1, 2, 13, 20, 22, 23, 25, 27, 28, 29, 30], "unix": 1, "unknow": [0, 21, 27], "unknown": [0, 1, 3, 4, 5, 6, 8, 10, 13, 21, 22, 27, 28, 29, 30], "unknowwn": 12, "unlabel": 1, "unless": [0, 3, 6, 11, 13, 22, 27, 29], "unlik": [1, 3, 8, 13, 29, 30], "unnecessarili": 9, "unord": 3, "unpublish": 30, "unravel": 1, "unrol": [3, 11], "unseen": [0, 7, 9, 15], "unstabl": 1, "unsupervis": [0, 1, 4, 12, 20, 27], "unsymmetr": [21, 27], "until": [1, 2, 4, 9, 12, 13, 14, 29], "untouch": 0, "unusu": 12, "up": [1, 3, 4, 5, 6, 8, 10, 11, 13, 14, 16, 18, 19, 20, 21, 22, 24, 25, 30], "updat": [1, 2, 10, 12, 13, 14, 15, 18], "uploa": 27, "upload": [15, 20, 22, 26], "upon": [0, 1, 6, 7, 11, 21], "upper": [0, 8, 9, 16, 21, 28], "uppercas": [21, 27], "upsampl": 4, "upscal": 4, "url": [27, 28], "us": [4, 5, 6, 8, 9, 10, 11, 12, 14, 15, 17, 19, 21, 24, 26], "usag": [0, 8, 20, 27, 28, 30], "usd": [], "usd10000": [], "use_bia": 4, "usecol": [0, 27], "useless": 1, "user": [0, 1, 2, 4, 6, 7, 15, 20, 21, 22, 27, 28], "usernam": [15, 22], "usetex": 24, "usg": 6, "usr": 24, "usual": [0, 3, 4, 7, 12, 13, 14, 27, 30], "ut": 5, "util": [1, 3, 4, 6, 7, 10, 14, 19, 27], "ux": 21, "v": [2, 4, 5, 6, 11, 13, 15, 20, 28, 29, 30], "v0": 24, "v1": 24, "v2": 24, "v_": 30, "v_0": 11, "v_t": 30, "va": 1, "vahid": 27, "val": 13, "val_accuraci": 3, "val_loss": 4, "vale": 2, "valid": [0, 1, 4, 7, 9, 10, 13, 20, 24, 27, 28, 30], "validation_data": 3, "validation_split": 4, "valu": [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 27, 30], "valuat": 9, "valy": 4, "van": [0, 19, 22, 27, 28, 29, 30], "vandenbergh": [8, 13, 29], "vandermond": [0, 27], "vanilla": [0, 6, 11, 14, 28, 30], "vanish": [1, 4, 13, 24, 29], "var": [5, 6, 10, 11, 19, 22, 24, 28], "var_x": 24, "varabl": 8, "varepsilon": [5, 6, 19], "varepsilon_": [5, 6], "varepsilon_i": [5, 6], "vari": [0, 1, 3, 5, 6, 10, 27], "variabl": [0, 1, 2, 5, 6, 7, 8, 10, 11, 12, 13, 14, 21, 27, 28, 30], "varianc": [0, 1, 5, 7, 9, 10, 11, 13, 14, 18, 20, 21, 24, 27, 28, 29, 30], "variance_i": [5, 11, 28], "variance_x": [5, 11, 28], "variant": [0, 1, 6, 8, 12, 13, 27, 28, 29, 30], "variat": [3, 4, 11, 27], "varieti": [0, 3, 12, 20, 22, 27], "variou": [1, 3, 5, 6, 7, 8, 9, 11, 12, 13, 16, 19, 20, 21, 22, 24, 27, 28, 29, 30], "varydimens": 4, "vastli": 3, "vaue": 1, "vault": 0, "vdot": [2, 13, 29, 30], "ve": [22, 30], "vec": 6, "vector": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 13, 14, 17, 18, 20, 29, 30], "vector_mean": 14, "ventur": [0, 8, 20, 27], "venv": 15, "verbos": [1, 3, 4], "veri": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 22, 24, 26, 27, 28, 29, 30], "verifi": [3, 11, 21, 27], "versatil": [8, 27], "versicolor": [8, 9], "version": [0, 3, 10, 13, 14, 15, 20, 21, 22, 24, 27], "versu": 1, "vert": [0, 1, 5, 6, 7, 8, 9, 11, 13, 16, 17, 27, 28, 29, 30], "vert_1": [5, 6, 28, 29, 30], "vert_2": [5, 6, 11, 17, 28, 29, 30], "via": [0, 5, 6, 7, 8, 9, 10, 11, 12, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30], "vidal": 11, "video": [0, 1, 12, 20, 23, 25, 27, 28, 29], "view": [1, 3, 5, 6, 12, 13, 24, 26, 27, 29, 30], "violat": 8, "virginica": 9, "viridi": [0, 1, 2, 3, 27], "virtual": 1, "viscos": [13, 30], "viscou": [13, 30], "visibl": 15, "vision": [0, 3], "visual": [0, 3, 11, 12, 18, 20, 27, 28], "visualis": 1, "visualstudio": [15, 16], "viz": [6, 8, 24], "vmap": 13, "vmax": [1, 6], "vmh0zpt0tli": 30, "vmin": [1, 6], "voic": 3, "volatil": 30, "volum": [0, 3, 27], "vote": [10, 27], "voting_clf": 10, "votingclassifi": 10, "votingsimpl": 10, "vstack": [5, 11, 21, 24, 27, 28], "vt": [5, 28, 29], "w": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 21, 24, 27, 28, 29, 30], "w1": 8, "w2": [8, 11], "w3": 8, "w_": [1, 12, 30], "w_1": [8, 21], "w_1x_": 8, "w_1x_1": 8, "w_2": [8, 21], "w_2x_": 8, "w_2x_2": 8, "w_3": 21, "w_4": 21, "w_hidden": 2, "w_i": [1, 2, 10], "w_ix_i": 12, "w_j": 21, "w_m": 21, "w_output": 2, "w_px_": 8, "w_px_p": 8, "w_t": 30, "wa": [1, 3, 4, 5, 6, 7, 10, 11, 12, 14, 17, 21, 27, 28, 30], "wai": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 18, 19, 21, 24, 27, 28, 29, 30], "walk": 9, "walker": 24, "wang": [0, 27], "want": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 20, 22, 24, 27, 28, 29, 30], "warn": 4, "warrant": 6, "wast": [3, 30], "watch": [20, 29, 30], "wave": 3, "wavelet": 8, "we": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29], "weak": [9, 10, 14], "weather": [1, 12], "web": [20, 23, 25, 27], "webpag": 27, "websit": [6, 21, 22, 23, 27], "wedg": [8, 24], "wednesdai": [25, 27], "wee": 11, "week": [0, 5, 6, 7, 22, 23, 25], "weekli": [15, 16, 20, 22, 23, 25, 26, 27], "weight": [1, 2, 3, 6, 7, 9, 10, 12, 13, 18, 24, 30], "weigth": 2, "welcom": [8, 15, 20], "well": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 20, 21, 22, 24, 26, 27, 28, 29, 30], "went": 8, "were": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 14, 24, 27], "wessel": [0, 19, 22, 27, 28, 29, 30], "what": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 30], "whatev": 3, "when": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 22, 24, 27, 28, 29], "whenev": [13, 15, 24, 30], "where": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 27, 28, 29, 30], "wherea": [6, 24, 30], "wherefrom": 22, "wherein": [1, 12], "whether": [0, 3, 5, 7, 9, 22, 24, 27], "which": [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29], "whichev": [1, 3], "while": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 16, 19, 24, 27, 28, 29, 30], "white": 9, "whiteboard": [28, 29], "who": [0, 15], "whole": [1, 3, 4, 5, 9, 11, 13, 30], "whose": [0, 6, 10, 24, 28], "whow": [11, 28], "why": [0, 1, 3, 6, 13, 15, 16, 17, 19, 22, 28, 29, 30], "wide": [0, 1, 3, 6, 7, 12, 20, 21, 22, 27], "widehat": 6, "width": [0, 3, 8, 9, 27], "wieringen": [0, 19, 22, 27, 28, 29, 30], "wiki": 22, "wikipedia": 22, "win": 10, "wind": 9, "wing": [25, 27], "winther": 2, "wiothout": 6, "wiscons": 7, "wisconsin": 10, "wisdom": [6, 28, 30], "wise": [1, 5, 12, 13, 28, 29, 30], "wish": [0, 2, 5, 7, 8, 11, 13, 14, 18, 21, 22, 27, 28, 29, 30], "with_std": [0, 28], "wither": 6, "within": [0, 2, 3, 4, 7, 9, 12, 13, 14, 24, 26, 27, 29], "withinclust": 14, "without": [0, 1, 5, 6, 8, 9, 11, 12, 13, 15, 18, 22, 27, 28, 29, 30], "won": [0, 15, 27], "wonder": 8, "word": [0, 1, 3, 4, 5, 6, 7, 14, 24, 27, 28, 29, 30], "work": [0, 1, 4, 6, 7, 8, 9, 13, 15, 16, 18, 20, 22, 23, 24, 25, 27, 28, 30], "workabl": 30, "workshop": 27, "world": [0, 8, 16, 28], "worldwid": [0, 27], "worri": 15, "wors": [0, 1, 3, 4, 6, 27], "worth": 9, "worthi": 22, "would": [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 18, 21, 22, 24, 27, 28, 29, 30], "wrap": [6, 21, 27], "write": [0, 1, 2, 3, 5, 6, 7, 8, 12, 13, 15, 16, 18, 21, 27, 28, 30], "written": [0, 2, 3, 5, 11, 12, 13, 16, 20, 21, 22, 24, 27, 28, 29, 30], "wrong": [1, 8, 15], "wrongli": 10, "wrote": [5, 11, 28], "wrt": [10, 13, 30], "wth": [10, 13, 30], "www": [20, 21, 22, 26, 27, 29, 30], "wx_1": 8, "x": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 27, 29, 30], "x0": 8, "x1": [4, 8, 9, 10, 13], "x1_exampl": 8, "x1d": 8, "x2": [8, 9, 10, 13], "x2d": [8, 11], "x2d_train": 11, "x2dsl": 11, "x3": 8, "x_": [0, 2, 3, 5, 6, 8, 10, 11, 13, 14, 21, 24, 27, 28, 29, 30], "x_0": [0, 5, 11, 18, 21, 27, 28], "x_1": [0, 2, 5, 6, 7, 8, 9, 10, 11, 13, 18, 21, 24, 27, 28, 29, 30], "x_2": [0, 2, 5, 6, 7, 8, 9, 10, 11, 13, 21, 24, 27, 28, 29], "x_3": [8, 21, 24], "x_4": 21, "x_6": 18, "x_center": 11, "x_data": 1, "x_data_ful": 1, "x_hidden": 2, "x_i": [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 21, 24, 27, 28, 29, 30], "x_input": 2, "x_ix_": [0, 27], "x_iy_i": 8, "x_j": [0, 2, 8, 9, 12, 16, 24, 28, 30], "x_jy_j": 8, "x_k": [12, 14, 21, 24, 28], "x_l": 24, "x_m": [6, 12, 21, 24], "x_mean": [18, 30], "x_n": [0, 2, 3, 6, 8, 11, 12, 13, 21, 24, 27, 29], "x_new": [9, 10], "x_norm": [18, 30], "x_offset": [6, 28, 30], "x_output": 2, "x_p": [3, 7, 9], "x_poli": 9, "x_poly10": 9, "x_pred": 4, "x_prev": 2, "x_reduc": 11, "x_sampl": 19, "x_scale": 8, "x_small": 13, "x_std": [18, 30], "x_t": 30, "x_test": [0, 1, 3, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 28, 29, 30], "x_test_": 17, "x_test_own": 6, "x_test_scal": [0, 6, 7, 9, 10, 11, 28, 30], "x_tot": 4, "x_train": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 27, 28, 29, 30], "x_train_": 17, "x_train_mean": [6, 28, 30], "x_train_own": 6, "x_train_scal": [0, 6, 7, 9, 10, 11, 28, 30], "x_val": 1, "xarrai": [20, 27], "xavier": 1, "xbnew": [13, 29, 30], "xcode": [0, 20, 22, 27], "xdclassiffierconfus": 10, "xdclassiffierroc": 10, "xg_clf": 10, "xgb": 10, "xgbclassifi": 10, "xgboost": 9, "xgboot": 10, "xgbregressor": 10, "xgparam": 10, "xgtree": 10, "xi": [8, 13, 30], "xi_": 8, "xi_1": 8, "xi_i": 8, "xk": 8, "xla": [13, 20, 27], "xlabel": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 24, 27, 28, 29, 30], "xlim": [6, 10], "xm": 9, "xmesh": 13, "xnew": [0, 13, 27, 29, 30], "xp": 24, "xpanda": [0, 28], "xpd": [5, 11, 28], "xplot": 0, "xscale": [0, 28], "xsr": 9, "xt_x": [13, 29, 30], "xtest": 6, "xtick": [3, 6, 8, 9], "xtrain": 6, "xu": [0, 27], "xx": [0, 21, 27], "xy": [0, 6, 8, 21, 27], "xytext": 8, "xz": [21, 27], "y": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 27, 28, 29, 30], "y1": 4, "y2": 4, "y3": 4, "y_": [0, 1, 5, 6, 10, 11, 21, 27, 28], "y_0": [0, 5, 11, 21, 27, 28], "y_1": [0, 5, 8, 9, 11, 13, 21, 27, 28, 29, 30], "y_1y_1": 8, "y_1y_1k": 8, "y_1y_2": 8, "y_1y_2k": 8, "y_1y_n": 8, "y_1y_nk": 8, "y_2": [0, 5, 8, 9, 11, 21, 27, 28], "y_2y_1": 8, "y_2y_1k": 8, "y_2y_2": 8, "y_2y_2k": 8, "y_3": [0, 9, 21], "y_4": 21, "y_center": [18, 30], "y_data": [0, 1, 5, 6, 27, 28, 29, 30], "y_data_ful": 1, "y_decis": 8, "y_fit": [0, 28], "y_i": [0, 1, 5, 6, 7, 8, 9, 10, 11, 12, 13, 19, 21, 22, 27, 28, 29, 30], "y_if_": 10, "y_ix_": [0, 27], "y_ix_i": [7, 8, 13, 28, 29, 30], "y_iy_jk": 8, "y_j": [6, 8, 12, 22], "y_k": 12, "y_m": 21, "y_mean": [18, 30], "y_model": [0, 4, 5, 6, 27, 28, 29, 30], "y_n": [8, 13, 29, 30], "y_ny_1": 8, "y_ny_1k": 8, "y_ny_2": 8, "y_ny_2k": 8, "y_ny_n": 8, "y_ny_nk": 8, "y_offset": [6, 17, 28, 30], "y_plot": 9, "y_pred": [0, 1, 4, 6, 7, 8, 9, 10, 28, 30], "y_pred1": 9, "y_pred2": 9, "y_pred_rf": 10, "y_pred_tre": 10, "y_proba": [7, 10], "y_sampl": 19, "y_scaler": [6, 28, 30], "y_test": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 28, 29, 30], "y_test_onehot": 1, "y_test_predict": [], "y_tot": 4, "y_train": [0, 1, 3, 4, 5, 6, 7, 9, 10, 11, 15, 16, 17, 19, 27, 28, 29, 30], "y_train_mean": [6, 28, 30], "y_train_onehot": 1, "y_train_predict": [], "y_train_scal": [6, 28, 30], "y_val": 1, "ye": [3, 6, 7], "year": [0, 20, 27], "yet": [0, 1, 6, 8, 11, 13, 27], "yi": [13, 30], "yield": [0, 2, 5, 6, 8, 10, 12, 13, 14, 21, 24, 27, 29, 30], "yk": 8, "ylabel": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 24, 27, 28, 29, 30], "ylim": [3, 6], "ym": 9, "ymesh": 13, "yn": 0, "yo": [8, 9, 10], "yoshua": [1, 26], "you": [0, 1, 3, 4, 5, 6, 8, 9, 10, 11, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30], "young": 0, "your": [1, 2, 4, 5, 6, 8, 11, 13, 15, 17, 19, 20, 21, 27, 29, 30], "your_model_object": 16, "yourself": [11, 13, 27, 29], "youtu": [28, 29], "youtub": [20, 29, 30], "ypred": 6, "ypredict": [0, 13, 27, 28, 29, 30], "ypredict2": [13, 29, 30], "ypredictlasso": [5, 29], "ypredictol": [0, 5, 29], "ypredictown": [6, 28, 30], "ypredictownridg": [6, 28, 29, 30], "ypredictridg": [0, 5, 6, 28, 29, 30], "ypredictskl": [6, 28, 30], "ytest": 6, "ytick": [3, 6, 8, 9], "ytild": [0, 6, 27, 28], "ytildelasso": [5, 29], "ytildenp": [0, 27, 28], "ytildeol": [0, 5, 29], "ytildeownridg": [6, 28, 29, 30], "ytilderidg": [5, 6, 28, 29, 30], "ytrain": 6, "yuxi": 27, "yx": [21, 27], "yy": [21, 27], "yz": [21, 27], "z": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 21, 24, 27, 28], "z_": [1, 2, 12, 21, 27], "z_0": [21, 27], "z_1": [21, 27], "z_2": [21, 27], "z_c": 1, "z_h": 1, "z_hidden": 2, "z_i": [1, 12], "z_j": [1, 12], "z_k": [12, 28], "z_m": 1, "z_mod": 9, "z_o": 1, "z_output": 2, "zaman": 24, "zaxi": 6, "zero": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 27, 28, 29, 30], "zeros_lik": 4, "zeroth": 28, "zfill": 4, "zip": [4, 6], "zm_h": [0, 27], "zn": [], "zone": [], "zoom": 27, "zx": [21, 27], "zy": [21, 27], "zz": [21, 27], "\u00f8yvind": [6, 28, 30]}, "titles": ["3. Linear Regression", "14. Building a Feed Forward Neural Network", "15. Solving Differential Equations with Deep Learning", "16. Convolutional Neural Networks", "17. Recurrent neural networks: Overarching view", "4. Ridge and Lasso Regression", "5. Resampling Methods", "6. Logistic Regression", "8. Support Vector Machines, overarching aims", "9. Decision trees, overarching aims", "10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods", "11. Basic ideas of the Principal Component Analysis (PCA)", "13. Neural networks", "7. Optimization, the central part of any Machine Learning algortithm", "12. Clustering and Unsupervised Learning", "Exercises week 34", "Exercises week 35", "Exercises week 36", "Exercises week 37", "Exercises week 37", "Applied Data Analysis and Machine Learning", "2. Linear Algebra, Handling of Arrays and more Python Features", "Project 1 on Machine Learning, deadline October 6 (midnight), 2025", "Course setting", "1. Elements of Probability Theory and Statistical Data Analysis", "Teachers and Grading", "Textbooks", "Week 34: Introduction to the course, Logistics and Practicalities", "Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression", "Week 36: Linear Regression and Gradient descent", "Week 37: Statistical interpretations and Resampling Methods"], "titleterms": {"": [8, 10, 29, 30], "1": [0, 15, 16, 17, 18, 19, 22, 28], "15": 19, "19": 19, "1a": 18, "2": [0, 15, 16, 17, 18, 19, 27, 28, 29], "2023": 25, "2025": 22, "2a": [], "2b": [], "3": [0, 15, 16, 17, 18, 19, 28], "34": [15, 27], "35": [16, 28], "36": [17, 29], "37": [18, 19, 30], "3a": 18, "3b": 18, "4": [0, 15, 16, 17, 18, 19, 28], "4a": 18, "4b": 18, "5": [0, 16, 18], "6": 22, "8": 30, "A": [0, 1, 4, 8, 9, 27], "And": [27, 28, 30], "But": 30, "In": 25, "Ising": 6, "The": [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 15, 20, 27, 28, 29, 30], "To": 27, "With": [4, 29], "about": [27, 28, 29], "abov": 29, "accuraci": 30, "activ": [1, 12], "ad": [0, 6, 22, 27, 28], "adaboost": 10, "adagrad": [13, 30], "adam": [13, 30], "adapt": [10, 30], "adjust": 1, "advanc": 22, "adversari": 4, "again": [3, 9], "ai": [22, 27], "aim": [8, 9, 27], "aka": 27, "al": 30, "algebra": [21, 27], "algorithm": [9, 10, 11, 12, 27, 28, 29, 30], "algortithm": [13, 29], "all": 8, "an": [0, 4, 10, 15, 27], "analys": [5, 28, 29], "analysi": [0, 5, 6, 11, 20, 22, 24, 27, 28, 29], "analyt": [0, 16, 18], "ani": [13, 29], "anoth": [9, 29], "appli": 20, "approach": [0, 8, 14, 27, 30], "approxim": 12, "architectur": 1, "arrai": [21, 27], "assist": 25, "autocorrel": 24, "autograd": [2, 13, 30], "automat": [13, 30], "b": 22, "back": [1, 11, 12, 28, 29], "background": [20, 22], "bag": 10, "base": [13, 30], "basic": [0, 5, 7, 9, 10, 11, 21, 28, 29], "batch": [1, 30], "bay": 5, "befor": 11, "better": 8, "bia": [6, 19, 22], "binari": 1, "bind": 27, "bird": 10, "boldsymbol": [18, 28], "boost": 10, "bootstrap": [6, 10], "boston": [], "breast": 1, "brief": 27, "bring": 12, "build": [1, 3, 9], "c": [22, 27], "calcul": [18, 28, 29], "can": [27, 30], "cancer": [1, 7, 9, 11], "cart": 9, "case": [8, 10, 24, 28, 29], "central": [13, 20, 24, 29], "chain": 12, "challeng": 30, "chang": 10, "channel": 27, "chi": [0, 27], "choic": 17, "choos": [1, 30], "cifar01": 3, "classic": 11, "classif": [1, 9, 10], "classifi": 8, "clip": 1, "cluster": 14, "cnn": 3, "code": [1, 2, 5, 9, 11, 12, 13, 14, 15, 16, 22, 27, 28, 29, 30], "collect": [1, 3], "commun": 27, "compar": [2, 10, 16], "comparison": 29, "compet": 30, "complet": 28, "complex": [0, 6, 22, 28], "complic": 6, "compon": 11, "comput": [9, 19, 30], "computerlab": 27, "con": [9, 30], "concept": 24, "condit": 29, "conjug": 13, "contn": 27, "converg": 30, "convex": [8, 13, 29], "convolut": [3, 12], "correl": [11, 28], "correspond": [], "cost": [1, 10, 28, 29, 30], "cours": [20, 23, 26, 27], "covari": [5, 11, 24, 28], "cover": 27, "creat": 16, "cross": [6, 22], "cython": 27, "d": 22, "data": [0, 1, 3, 6, 7, 9, 11, 15, 17, 18, 20, 24, 27, 28], "dataset": [1, 3, 18], "david": 27, "deadlin": [22, 27], "deadllin": 25, "decai": [2, 30], "decis": [9, 10], "decomposit": [5, 11, 21, 28, 29], "deeep": [], "deep": [1, 2, 27], "defin": [1, 27], "degre": [0, 17, 28], "deliver": [15, 16], "deliveri": 22, "dens": 0, "deriv": [5, 12, 16, 17, 19, 28, 29, 30], "descent": [2, 10, 13, 18, 22, 29, 30], "design": 28, "detail": [3, 27], "develop": 1, "diagon": 11, "differ": [8, 30], "differenti": [2, 13, 30], "diffus": 2, "dimension": [2, 3, 8, 18], "disadvantag": 9, "discret": 24, "discrimin": 27, "distribut": [5, 24], "do": [1, 30], "doe": [28, 29], "domain": 24, "down": 1, "dropout": 1, "e": 22, "economi": [28, 29], "electron": 22, "element": [0, 24, 27], "elimin": 21, "energi": 27, "ensembl": 10, "entropi": 9, "environ": [0, 15], "equat": [0, 2, 12, 28, 29], "error": [0, 10, 27, 28, 29], "essenti": 27, "et": 30, "etc": 27, "euler": 2, "evalu": 1, "exampl": [1, 2, 3, 4, 6, 7, 8, 9, 10, 27, 28, 29, 30], "exercis": [0, 6, 15, 16, 17, 18, 19, 28], "expect": [19, 24], "experi": 24, "explor": 0, "exponenti": 2, "express": [16, 17, 19, 28], "extend": 29, "extrapol": 4, "extrem": [10, 27], "ey": 10, "f": 22, "fall": 25, "famili": [1, 27], "famou": 21, "fantast": [28, 29], "featur": [9, 16, 21, 28], "feed": [1, 12], "final": [12, 28, 30], "find": [16, 18], "fine": 1, "first": [4, 12, 27, 29], "fit": [0, 10, 15, 16, 27, 29], "fix": [28, 29, 30], "forc": 3, "forest": 10, "form": 18, "format": [22, 27], "formula": 18, "forward": [1, 2, 12], "foster": 27, "fourier": 3, "frank": 6, "freedom": [0, 17, 28], "frequent": [28, 30], "frequentist": [0, 27], "from": [5, 10, 12, 27, 28, 29, 30], "full": 2, "function": [0, 1, 6, 7, 8, 10, 11, 12, 13, 22, 24, 27, 28, 29, 30], "further": [3, 5, 28, 29], "g": 22, "gan": 4, "gaussian": 21, "gd": [13, 30], "gener": [4, 9, 27], "geometr": [11, 29], "gini": 9, "github": 15, "goal": [15, 16, 17, 18, 19], "good": [0, 27], "goodfellow": 30, "grade": [25, 27], "gradient": [1, 2, 10, 13, 18, 22, 29, 30], "growth": 2, "h": 22, "ha": 20, "handl": [21, 27], "hessian": [28, 29, 30], "hidden": 2, "hous": [], "how": 16, "hyperparamet": [1, 17], "hyperplan": 8, "i": [0, 1, 27], "id3": 9, "idea": 11, "ideal": 29, "ii": 27, "illustr": 29, "implement": [1, 16, 17, 18], "implic": [5, 28, 29], "import": [5, 21, 27, 28, 29], "improv": [1, 30], "includ": [13, 22, 30], "increment": 11, "index": 9, "inform": 25, "input": 2, "instal": [20, 22, 27], "instructor": 25, "interpret": [5, 11, 27, 28, 29, 30], "introduc": [11, 13, 28], "introduct": [0, 6, 20, 21, 22, 27], "invers": [5, 21], "invert": [28, 29], "iter": 10, "its": 28, "jacobian": 28, "jax": 13, "julia": 27, "jungl": 10, "kera": [1, 3], "kernel": [8, 11], "lab": [29, 30], "lagrangian": 8, "lasso": [5, 6, 22, 28, 29], "last": [28, 30], "later": [5, 28, 29], "layer": [1, 2, 3, 12], "learn": [0, 1, 2, 11, 13, 14, 15, 16, 17, 18, 19, 20, 22, 27, 28, 29, 30], "least": [5, 6, 16, 19, 22, 27, 28, 29, 30], "lectur": [27, 29, 30], "level": 10, "librari": [20, 27], "likelihood": 7, "limit": [1, 13, 24, 29, 30], "linear": [0, 8, 13, 15, 21, 27, 28, 29], "link": [5, 11, 26, 28], "literatur": 22, "logist": [7, 27], "loss": [28, 29, 30], "lu": 21, "machin": [0, 8, 13, 20, 22, 27, 29], "main": [24, 27], "make": [0, 9, 10, 28], "mani": [10, 12], "mass": 27, "materi": [22, 27, 28, 29, 30], "math": [5, 28, 29], "mathemat": [3, 5, 8, 28, 29], "matric": [5, 21, 27], "matrix": [1, 5, 11, 12, 16, 21, 27, 28, 29, 30], "matter": 0, "max": 28, "mean": [0, 28, 29], "meet": [5, 10, 24, 27, 28], "mercer": 8, "method": [6, 9, 10, 13, 22, 27, 29, 30], "midnight": 22, "min": 28, "mini": 30, "minibatch": 30, "minim": 27, "ml": 27, "mlp": 12, "mnist": [3, 4], "model": [0, 1, 4, 6, 12, 15, 17, 27], "moment": 30, "momentum": [13, 22, 30], "mondai": [29, 30], "moon": [8, 9], "more": [3, 6, 21, 22, 27, 28, 29, 30], "motiv": 30, "multilay": 12, "multipl": [1, 3, 17], "multipli": 8, "need": [22, 27], "network": [1, 2, 3, 4, 7, 12, 27], "neural": [1, 2, 3, 4, 7, 12, 27], "new": [4, 18], "newton": [29, 30], "non": 8, "none": 30, "normal": [0, 1], "notat": 12, "note": [22, 28, 29], "now": [1, 9, 13, 29, 30], "nuclear": [0, 27], "numba": 27, "number": [0, 2, 24, 28, 30], "numer": [2, 22, 24], "numpi": [21, 27], "object": 3, "obtain": 11, "octob": 22, "od": 2, "off": [6, 19, 22], "ol": [5, 6, 15, 16, 18, 22, 29], "one": [2, 12, 18, 29], "oper": 21, "optim": [1, 8, 13, 18, 20, 27, 28, 29, 30], "order": [13, 18, 30], "ordinari": [5, 6, 16, 19, 22, 27, 28, 29, 30], "organ": [0, 27], "oslo": 26, "other": [4, 9, 11, 12, 21, 22, 27], "our": [0, 4, 5, 11, 13, 22, 27, 28, 29], "outcom": [20, 27], "output": 2, "overarch": [0, 4, 8, 9, 27, 28], "overview": [10, 27, 30], "own": [0, 10, 11, 22, 27, 28], "packag": [21, 27], "panda": [27, 28], "paramet": [27, 28, 30], "paramt": 18, "part": [13, 20, 22, 29], "partial": 2, "pass": 1, "pca": 11, "pdf": 24, "perceptron": 12, "perform": [1, 9], "period": 3, "perspect": 1, "plan": [28, 29, 30], "plethora": 27, "point": 4, "poisson": 2, "polynomi": [3, 16, 18, 29], "popul": 2, "popular": 27, "practic": [13, 25, 27, 30], "pre": [1, 3], "preambl": 22, "predict": 4, "preprocess": [28, 30], "prerequisit": [3, 20, 27], "princip": 11, "principl": 3, "pro": [9, 30], "probabl": [5, 24], "problem": [1, 2, 13, 27, 28, 29, 30], "procedur": [9, 27], "process": [1, 3], "program": [2, 13, 22, 29, 30], "project": [6, 22, 25, 27], "prop": [13, 30], "propag": [1, 12], "properti": [5, 24, 28, 29, 30], "python": [0, 9, 15, 20, 21, 27], "quick": 8, "r": 27, "random": [10, 11, 24], "raphson": 29, "rate": [22, 30], "read": [9, 27, 28, 30], "real": [6, 27], "recommend": [27, 28], "recurr": [4, 12], "reduc": [0, 28], "reduct": 3, "refer": 22, "reformul": 2, "regress": [0, 5, 6, 7, 9, 10, 13, 15, 17, 18, 19, 22, 27, 28, 29, 30], "regular": 1, "relat": [], "relev": [26, 28], "relu": 1, "remark": 3, "remind": [6, 8, 27, 28, 29, 30], "replac": [13, 30], "report": 22, "repositori": 15, "requir": [2, 20], "resampl": [6, 19, 22, 30], "rescal": [6, 28], "residu": [28, 29], "resourc": 2, "result": [28, 29], "revisit": [13, 29, 30], "rewrit": [27, 28], "ridg": [0, 5, 6, 17, 18, 19, 22, 28, 29, 30], "rm": [13, 30], "rmsprop": 30, "rule": [12, 30], "rung": 22, "same": [13, 30], "sampl": 11, "scale": [17, 18, 28, 30], "schedul": 27, "schemat": 9, "scheme": 2, "scienc": 27, "scikit": [0, 1, 11, 27, 28, 29, 30], "second": [13, 18, 30], "semest": 25, "sensit": 29, "septemb": [19, 29, 30], "session": [29, 30], "set": [0, 2, 3, 9, 12, 15, 23, 27, 28, 29], "setup": 15, "sgd": [13, 30], "should": 1, "similar": [13, 30], "simpl": [0, 4, 9, 13, 18, 27, 28, 29, 30], "simplest": 18, "singl": 10, "singular": [5, 11, 28, 29], "size": [28, 29, 30], "sklearn": 16, "slightli": 30, "sneak": 30, "soft": 8, "softmax": 1, "softwar": [22, 27], "solv": [2, 29], "solver": 13, "some": [13, 21, 28, 29], "specifi": 2, "split": [0, 15, 28], "squar": [0, 5, 6, 10, 16, 19, 22, 27, 28, 29, 30], "standard": [13, 28], "state": 0, "statist": [5, 6, 20, 24, 27, 30], "steepest": [10, 13, 29], "step": 30, "stochast": [13, 22, 24, 30], "stop": 30, "strongli": 27, "suggest": 27, "summari": [25, 27], "superposit": 3, "supervis": 1, "support": 8, "svd": [5, 28, 29], "synthet": 18, "systemat": 3, "t": 28, "take": 16, "taken": [27, 30], "teach": 25, "teacher": [25, 27], "technic": 28, "techniqu": [6, 11, 22], "technologi": 20, "tensorflow": [1, 3], "tent": [25, 27], "test": [0, 1, 15, 17, 28], "text": 27, "textbook": [26, 27], "than": 29, "theorem": [5, 8, 11, 12, 24], "theori": 24, "theta": 18, "thi": 27, "time": 30, "tip": [13, 30], "togeth": 12, "tool": [22, 27], "top": 1, "topic": 27, "toward": 11, "trade": [6, 19, 22], "tradeoff": 6, "train": [0, 1, 4, 15, 27, 28], "transform": 3, "tree": [9, 10], "tuesdai": 29, "tune": 1, "two": [3, 8, 20], "type": [2, 4, 12, 27], "uio": 27, "univers": [12, 26], "unsupervis": 14, "up": [0, 2, 9, 12, 15, 27, 28, 29], "updat": [22, 30], "us": [0, 1, 2, 3, 7, 13, 16, 18, 20, 22, 27, 28, 29, 30], "v": 3, "valid": [6, 22], "valu": [5, 11, 19, 24, 28, 29], "vari": 30, "variabl": [24, 29], "varianc": [6, 19, 22], "variou": 0, "vector": [8, 12, 16, 21, 27, 28], "versu": 27, "video": 30, "view": [0, 4, 10, 28], "virtual": 15, "visual": [1, 9], "wai": [9, 22], "wave": 2, "we": [27, 30], "wednesdai": 29, "week": [15, 16, 17, 18, 19, 27, 28, 29, 30], "weekli": [], "what": [0, 27, 28, 29], "when": 30, "which": [1, 30], "why": 27, "wisconsin": 7, "write": [4, 11, 22, 29], "x": 28, "xgboost": 10, "yet": 29, "your": [0, 10, 16, 18, 22, 28]}}) \ No newline at end of file diff --git a/doc/LectureNotes/_build/html/statistics.html b/doc/LectureNotes/_build/html/statistics.html index 97a898137..1b311903a 100644 --- a/doc/LectureNotes/_build/html/statistics.html +++ b/doc/LectureNotes/_build/html/statistics.html @@ -230,6 +230,8 @@
    • Exercises week 36
    • Week 36: Linear Regression and Gradient descent
    • Exercises week 37
      • +
    • Week 37: Statistical interpretations and Resampling Methods
      • +
    • Exercises week 37

    Projects

    Projects

    Projects

    Projects

    Projects

    Projects

      diff --git a/doc/LectureNotes/_build/html/week37.html b/doc/LectureNotes/_build/html/week37.html new file mode 100644 index 000000000..fca68dada --- /dev/null +++ b/doc/LectureNotes/_build/html/week37.html @@ -0,0 +1,2756 @@ + + + + + + + + + + + Week 37: Statistical interpretations and Resampling Methods — Applied Data Analysis and Machine Learning + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + + + + + + + + + +
      +
      +
      +
      + + + Ctrl+K +
      +
      + +
      + +
      + + +
      +
      + + +
      +
      + + + +
      + + + +
      + + + + +
      + +
      +
      + + + + + +
      +
      + +
      +
      +
      + + +
      +
      + + + + +
      + +
      + + + +
      + +
      +
      + +
      +
      + +
      + +
      + +
      + + +
      + +
      + +
      + + + + + +
      + + +
      + + + + + + + + + + + + + +
      + +
      + +
      +
      + + + +
      +

      Week 37: Statistical interpretations and Resampling Methods

      + +
      +
      + +
      +

      Contents

      +
      + +
      +
      +
      + + + + +
      + + +
      +

      Week 37: Statistical interpretations and Resampling Methods#

      +

      Morten Hjorth-Jensen, Department of Physics, University of Oslo, Norway

      +

      Date: September 8-12, 2025

      +
      +

      Plans for week 37, lecture Monday#

      +

      Plans and material for the lecture on Monday September 8.

      +

      The family of gradient descent methods

      +
        +

      1. Plain gradient descent (constant learning rate), reminder from last week with examples using OLS and Ridge

        2. +

      2. Improving gradient descent with momentum

        3. +

      3. Introducing stochastic gradient descent

        4. +

      4. More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM

        5. +
      + +
      +
      +

      Readings and Videos:#

      +
        +

      1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at https://www.deeplearningbook.org/contents/optimization.html

        2. +

      2. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.

        3. +

      3. Video on gradient descent at https://www.youtube.com/watch?v=sDv4f4s2SB8

        4. +

      4. Video on Stochastic gradient descent at https://www.youtube.com/watch?v=vMh0zPT0tLI

        5. +
      +
      +
      +

      Material for lecture Monday September 8#

      +
      +
      +

      Gradient descent and revisiting Ordinary Least Squares from last week#

      +

      Last week we started with linear regression as a case study for the gradient descent +methods. Linear regression is a great test case for the gradient +descent methods discussed in the lectures since it has several +desirable properties such as:

      +
        +

      1. An analytical solution (recall homework sets for week 35).

        2. +

      2. The gradient can be computed analytically.

        3. +

      3. The cost function is convex which guarantees that gradient descent converges for small enough learning rates

        4. +
      +

      We revisit an example similar to what we had in the first homework set. We have a function of the type

      +
      +
      +
      x = 2*np.random.rand(m,1)
+y = 4+3*x+np.random.randn(m,1)
+
      +
      +
      +
      +

      with \(x_i \in [0,1] \) is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution \(\cal {N}(0,1)\). +The linear regression model is given by

      +
      +\[ +h_\theta(x) = \boldsymbol{y} = \theta_0 + \theta_1 x, +\]
      +

      such that

      +
      +\[ +\boldsymbol{y}_i = \theta_0 + \theta_1 x_i. +\]
      +
      +
      +

      Gradient descent example#

      +

      Let \(\mathbf{y} = (y_1,\cdots,y_n)^T\), \(\mathbf{\boldsymbol{y}} = (\boldsymbol{y}_1,\cdots,\boldsymbol{y}_n)^T\) and \(\theta = (\theta_0, \theta_1)^T\)

      +

      It is convenient to write \(\mathbf{\boldsymbol{y}} = X\theta\) where \(X \in \mathbb{R}^{100 \times 2} \) is the design matrix given by (we keep the intercept here)

      +
      +\[\begin{split} +X \equiv \begin{bmatrix} +1 & x_1 \\ +\vdots & \vdots \\ +1 & x_{100} & \\ +\end{bmatrix}. +\end{split}\]
      +

      The cost/loss/risk function is given by (

      +
      +\[ +C(\theta) = \frac{1}{n}||X\theta-\mathbf{y}||_{2}^{2} = \frac{1}{n}\sum_{i=1}^{100}\left[ (\theta_0 + \theta_1 x_i)^2 - 2 y_i (\theta_0 + \theta_1 x_i) + y_i^2\right] +\]
      +

      and we want to find \(\theta\) such that \(C(\theta)\) is minimized.

      +
      +
      +

      The derivative of the cost/loss function#

      +

      Computing \(\partial C(\theta) / \partial \theta_0\) and \(\partial C(\theta) / \partial \theta_1\) we can show that the gradient can be written as

      +
      +\[\begin{split} +\nabla_{\theta} C(\theta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\theta_0+\theta_1x_i-y_i\right) \\ +\sum_{i=1}^{100}\left( x_i (\theta_0+\theta_1x_i)-y_ix_i\right) \\ +\end{bmatrix} = \frac{2}{n}X^T(X\theta - \mathbf{y}), +\end{split}\]
      +

      where \(X\) is the design matrix defined above.

      +
      +
      +

      The Hessian matrix#

      +

      The Hessian matrix of \(C(\theta)\) is given by

      +
      +\[\begin{split} +\boldsymbol{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\theta)}{\partial \theta_0^2} & \frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} \\ +\frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} & \frac{\partial^2 C(\theta)}{\partial \theta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X. +\end{split}\]
      +

      This result implies that \(C(\theta)\) is a convex function since the matrix \(X^T X\) always is positive semi-definite.

      +
      +
      +

      Simple program#

      +

      We can now write a program that minimizes \(C(\theta)\) using the gradient descent method with a constant learning rate \(\gamma\) according to

      +
      +\[ +\theta_{k+1} = \theta_k - \gamma \nabla_\theta C(\theta_k), \ k=0,1,\cdots +\]
      +

      We can use the expression we computed for the gradient and let use a +\(\theta_0\) be chosen randomly and let \(\gamma = 0.001\). Stop iterating +when \(||\nabla_\theta C(\theta_k) || \leq \epsilon = 10^{-8}\). Note that the code below does not include the latter stop criterion.

      +

      And finally we can compare our solution for \(\theta\) with the analytic result given by +\(\theta= (X^TX)^{-1} X^T \mathbf{y}\).

      +
      +
      +

      Gradient Descent Example#

      +

      Here our simple example

      +
      +
      +
      %matplotlib inline
+
+
+# Importing various packages
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib import cm
+from matplotlib.ticker import LinearLocator, FormatStrFormatter
+import sys
+
+# the number of datapoints
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+# Hessian matrix
+H = (2.0/n)* X.T @ X
+# Get the eigenvalues
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y
+print(theta_linreg)
+theta = np.random.randn(2,1)
+
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
+
+for iter in range(Niterations):
+    gradient = (2.0/n)*X.T @ (X @ theta-y)
+    theta -= eta*gradient
+
+print(theta)
+xnew = np.array([[0],[2]])
+xbnew = np.c_[np.ones((2,1)), xnew]
+ypredict = xbnew.dot(theta)
+ypredict2 = xbnew.dot(theta_linreg)
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Gradient descent example')
+plt.show()
+
      +
      +
      +
      +
      +
      +

      Gradient descent and Ridge#

      +

      We have also discussed Ridge regression where the loss function contains a regularized term given by the \(L_2\) norm of \(\theta\),

      +
      +\[ +C_{\text{ridge}}(\theta) = \frac{1}{n}||X\theta -\mathbf{y}||^2 + \lambda ||\theta||^2, \ \lambda \geq 0. +\]
      +

      In order to minimize \(C_{\text{ridge}}(\theta)\) using GD we adjust the gradient as follows

      +
      +\[\begin{split} +\nabla_\theta C_{\text{ridge}}(\theta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\theta_0+\theta_1x_i-y_i\right) \\ +\sum_{i=1}^{100}\left( x_i (\theta_0+\theta_1x_i)-y_ix_i\right) \\ +\end{bmatrix} + 2\lambda\begin{bmatrix} \theta_0 \\ \theta_1\end{bmatrix} = 2 (\frac{1}{n}X^T(X\theta - \mathbf{y})+\lambda \theta). +\end{split}\]
      +

      We can easily extend our program to minimize \(C_{\text{ridge}}(\theta)\) using gradient descent and compare with the analytical solution given by

      +
      +\[ +\theta_{\text{ridge}} = \left(X^T X + n\lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}. +\]
      +
      +
      +

      The Hessian matrix for Ridge Regression#

      +

      The Hessian matrix of Ridge Regression for our simple example is given by

      +
      +\[\begin{split} +\boldsymbol{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\theta)}{\partial \theta_0^2} & \frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} \\ +\frac{\partial^2 C(\theta)}{\partial \theta_0 \partial \theta_1} & \frac{\partial^2 C(\theta)}{\partial \theta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X+2\lambda\boldsymbol{I}. +\end{split}\]
      +

      This implies that the Hessian matrix is positive definite, hence the stationary point is a +minimum. +Note that the Ridge cost function is convex being a sum of two convex +functions. Therefore, the stationary point is a global +minimum of this function.

      +
      +
      +

      Program example for gradient descent with Ridge Regression#

      +
      +
      +
      from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib import cm
+from matplotlib.ticker import LinearLocator, FormatStrFormatter
+import sys
+
+# the number of datapoints
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+
+#Ridge parameter lambda
+lmbda  = 0.001
+Id = n*lmbda* np.eye(XT_X.shape[0])
+
+# Hessian matrix
+H = (2.0/n)* XT_X+2*lmbda* np.eye(XT_X.shape[0])
+# Get the eigenvalues
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+
+theta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
+print(theta_linreg)
+# Start plain gradient descent
+theta = np.random.randn(2,1)
+
+eta = 1.0/np.max(EigValues)
+Niterations = 100
+
+for iter in range(Niterations):
+    gradients = 2.0/n*X.T @ (X @ (theta)-y)+2*lmbda*theta
+    theta -= eta*gradients
+
+print(theta)
+ypredict = X @ theta
+ypredict2 = X @ theta_linreg
+plt.plot(x, ypredict, "r-")
+plt.plot(x, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Gradient descent example for Ridge')
+plt.show()
+
      +
      +
      +
      +
      +
      +

      Using gradient descent methods, limitations#

      +
        +

      • Gradient descent (GD) finds local minima of our function. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.

        • +

      • GD is sensitive to initial conditions. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.

        • +

      • Gradients are computationally expensive to calculate for large datasets. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, \(E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2\); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over all \(n\) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called “mini batches”. This has the added benefit of introducing stochasticity into our algorithm.

        • +

      • GD is very sensitive to choices of learning rates. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would adaptively choose the learning rates to match the landscape.

        • +

      • GD treats all directions in parameter space uniformly. Another major drawback of GD is that unlike Newton’s method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.

        • +

      • GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.

        • +
      +
      +
      +

      Improving gradient descent with momentum#

      +

      We discuss here some simple examples where we introduce what is called ‘memory’about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.

      +
      +
      +
      from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# take a step
+		solution = solution - step_size * gradient
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# perform the gradient descent search
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
      +
      +
      +
      +
      +
      +

      Same code but now with momentum gradient descent#

      +
      +
      +
      from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# keep track of the change
+	change = 0.0
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# calculate update
+		new_change = step_size * gradient + momentum * change
+		# take a step
+		solution = solution - new_change
+		# save the change
+		change = new_change
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# define momentum
+momentum = 0.3
+# perform the gradient descent search with momentum
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
      +
      +
      +
      +
      +
      +

      Overview video on Stochastic Gradient Descent (SGD)#

      +

      What is Stochastic Gradient Descent +There are several reasons for using stochastic gradient descent. Some of these are:

      +
        +

      1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.

        2. +

      2. Hopefully avoid Local Minima

        3. +

      3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset.

        4. +
      +
      +
      +

      Batches and mini-batches#

      +

      In gradient descent we compute the cost function and its gradient for all data points we have.

      +

      In large-scale applications such as the ILSVRC challenge, the +training data can have on order of millions of examples. Hence, it +seems wasteful to compute the full cost function over the entire +training set in order to perform only a single parameter update. A +very common approach to addressing this challenge is to compute the +gradient over batches of the training data. For example, a typical batch could contain some thousand examples from +an entire training set of several millions. This batch is then used to +perform a parameter update.

      +
      +
      +

      Pros and cons#

      +
        +

      1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.

        2. +

      2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.

        3. +

      3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.

        4. +
      +
      +
      +

      Convergence rates#

      +
        +

      1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.

        2. +

      2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.

        3. +
      +
      +
      +

      Accuracy#

      +

      In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset.

      +

      There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence.

      +

      It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates

      +
      +
      +

      Stochastic Gradient Descent (SGD)#

      +

      In stochastic gradient descent, the extreme case is the case where we +have only one batch, that is we include the whole data set.

      +

      This process is called Stochastic Gradient +Descent (SGD) (or also sometimes on-line gradient descent). This is +relatively less common to see because in practice due to vectorized +code optimizations it can be computationally much more efficient to +evaluate the gradient for 100 examples, than the gradient for one +example 100 times. Even though SGD technically refers to using a +single example at a time to evaluate the gradient, you will hear +people use the term SGD even when referring to mini-batch gradient +descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD +for “Batch gradient descent” are rare to see), where it is usually +assumed that mini-batches are used. The size of the mini-batch is a +hyperparameter but it is not very common to cross-validate or bootstrap it. It is +usually based on memory constraints (if any), or set to some value, +e.g. 32, 64 or 128. We use powers of 2 in practice because many +vectorized operation implementations work faster when their inputs are +sized in powers of 2.

      +

      In our notes with SGD we mean stochastic gradient descent with mini-batches.

      +
      +
      +

      Stochastic Gradient Descent#

      +

      Stochastic gradient descent (SGD) and variants thereof address some of +the shortcomings of the Gradient descent method discussed above.

      +

      The underlying idea of SGD comes from the observation that the cost +function, which we want to minimize, can almost always be written as a +sum over \(n\) data points \(\{\mathbf{x}_i\}_{i=1}^n\),

      +
      +\[ +C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i, +\mathbf{\beta}). +\]
      +
      +
      +

      Computation of gradients#

      +

      This in turn means that the gradient can be +computed as a sum over \(i\)-gradients

      +
      +\[ +\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}). +\]
      +

      Stochasticity/randomness is introduced by only taking the +gradient on a subset of the data called minibatches. If there are \(n\) +data points and the size of each minibatch is \(M\), there will be \(n/M\) +minibatches. We denote these minibatches by \(B_k\) where +\(k=1,\cdots,n/M\).

      +
      +
      +

      SGD example#

      +

      As an example, suppose we have \(10\) data points \((\mathbf{x}_1,\cdots, \mathbf{x}_{10})\) +and we choose to have \(M=5\) minibathces, +then each minibatch contains two data points. In particular we have +\(B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 = +(\mathbf{x}_9,\mathbf{x}_{10})\). Note that if you choose \(M=1\) you +have only a single batch with all data points and on the other extreme, +you may choose \(M=n\) resulting in a minibatch for each datapoint, i.e +\(B_k = \mathbf{x}_k\).

      +

      The idea is now to approximate the gradient by replacing the sum over +all data points with a sum over the data points in one the minibatches +picked at random in each gradient descent step

      +
      +\[ +\nabla_{\beta} +C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta +c_i(\mathbf{x}_i, \mathbf{\beta}). +\]
      +
      +
      +

      The gradient step#

      +

      Thus a gradient descent step now looks like

      +
      +\[ +\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) +\]
      +

      where \(k\) is picked at random with equal +probability from \([1,n/M]\). An iteration over the number of +minibathces (n/M) is commonly referred to as an epoch. Thus it is +typical to choose a number of epochs and for each epoch iterate over +the number of minibatches, as exemplified in the code below.

      +
      +
      +

      Simple example code#

      +
      +
      +
      import numpy as np 
+
+n = 100 #100 datapoints 
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+n_epochs = 10 #number of epochs
+
+j = 0
+for epoch in range(1,n_epochs+1):
+    for i in range(m):
+        k = np.random.randint(m) #Pick the k-th minibatch at random
+        #Compute the gradient using the data in minibatch Bk
+        #Compute new suggestion for 
+        j += 1
+
      +
      +
      +
      +

      Taking the gradient only on a subset of the data has two important +benefits. First, it introduces randomness which decreases the chance +that our opmization scheme gets stuck in a local minima. Second, if +the size of the minibatches are small relative to the number of +datapoints (\(M < n\)), the computation of the gradient is much +cheaper since we sum over the datapoints in the \(k-th\) minibatch and not +all \(n\) datapoints.

      +
      +
      +

      When do we stop?#

      +

      A natural question is when do we stop the search for a new minimum? +One possibility is to compute the full gradient after a given number +of epochs and check if the norm of the gradient is smaller than some +threshold and stop if true. However, the condition that the gradient +is zero is valid also for local minima, so this would only tell us +that we are close to a local/global minimum. However, we could also +evaluate the cost function at this point, store the result and +continue the search. If the test kicks in at a later stage we can +compare the values of the cost function and keep the \(\beta\) that +gave the lowest value.

      +
      +
      +

      Slightly different approach#

      +

      Another approach is to let the step length \(\gamma_j\) depend on the +number of epochs in such a way that it becomes very small after a +reasonable time such that we do not move at all. Such approaches are +also called scaling. There are many such ways to scale the learning +rate +and discussions here. See +also +https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1 +for a discussion of different scaling functions for the learning rate.

      +
      +
      +

      Time decay rate#

      +

      As an example, let \(e = 0,1,2,3,\cdots\) denote the current epoch and let \(t_0, t_1 > 0\) be two fixed numbers. Furthermore, let \(t = e \cdot m + i\) where \(m\) is the number of minibatches and \(i=0,\cdots,m-1\). Then the function $\(\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} \)\( goes to zero as the number of epochs gets large. I.e. we start with a step length \)\gamma_j (0; t_0, t_1) = t_0/t_1\( which decays in *time* \)t$.

      +

      In this way we can fix the number of epochs, compute \(\beta\) and +evaluate the cost function at the end. Repeating the computation will +give a different result since the scheme is random by design. Then we +pick the final \(\beta\) that gives the lowest value of the cost +function.

      +
      +
      +
      import numpy as np 
+
+def step_length(t,t0,t1):
+    return t0/(t+t1)
+
+n = 100 #100 datapoints 
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+n_epochs = 500 #number of epochs
+t0 = 1.0
+t1 = 10
+
+gamma_j = t0/t1
+j = 0
+for epoch in range(1,n_epochs+1):
+    for i in range(m):
+        k = np.random.randint(m) #Pick the k-th minibatch at random
+        #Compute the gradient using the data in minibatch Bk
+        #Compute new suggestion for beta
+        t = epoch*m+i
+        gamma_j = step_length(t,t0,t1)
+        j += 1
+
+print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
+
      +
      +
      +
      +
      +
      +

      Code with a Number of Minibatches which varies#

      +

      In the code here we vary the number of mini-batches.

      +
      +
      +
      # Importing various packages
+from math import exp, sqrt
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
+
+
+for iter in range(Niterations):
+    gradients = 2.0/n*X.T @ ((X @ theta)-y)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+xnew = np.array([[0],[2]])
+Xnew = np.c_[np.ones((2,1)), xnew]
+ypredict = Xnew.dot(theta)
+ypredict2 = Xnew.dot(theta_linreg)
+
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+t0, t1 = 5, 50
+
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
+
+for epoch in range(n_epochs):
+# Can you figure out a better way of setting up the contributions to each batch?
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
+        eta = learning_schedule(epoch*m+i)
+        theta = theta - eta*gradients
+print("theta from own sdg")
+print(theta)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
      +
      +
      +
      +
      +
      +

      Replace or not#

      +

      In the above code, we have use replacement in setting up the +mini-batches. The discussion +here may be +useful.

      +
      +
      +

      Momentum based GD#

      +

      The stochastic gradient descent (SGD) is almost always used with a +momentum or inertia term that serves as a memory of the direction we +are moving in parameter space. This is typically implemented as +follows

      +
      +\[ +\mathbf{v}_{t}=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t) \nonumber +\]
      + +
      +
      +\[ +\begin{equation} +\boldsymbol{\theta}_{t+1}= \boldsymbol{\theta}_t -\mathbf{v}_{t}, +\label{_auto1} \tag{1} +\end{equation} +\]
      +

      where we have introduced a momentum parameter \(\gamma\), with +\(0\le\gamma\le 1\), and for brevity we dropped the explicit notation to +indicate the gradient is to be taken over a different mini-batch at +each step. We call this algorithm gradient descent with momentum +(GDM). From these equations, it is clear that \(\mathbf{v}_t\) is a +running average of recently encountered gradients and +\((1-\gamma)^{-1}\) sets the characteristic time scale for the memory +used in the averaging procedure. Consistent with this, when +\(\gamma=0\), this just reduces down to ordinary SGD as discussed +earlier. An equivalent way of writing the updates is

      +
      +\[ +\Delta \boldsymbol{\theta}_{t+1} = \gamma \Delta \boldsymbol{\theta}_t -\ \eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t), +\]
      +

      where we have defined \(\Delta \boldsymbol{\theta}_{t}= \boldsymbol{\theta}_t-\boldsymbol{\theta}_{t-1}\).

      +
      +
      +

      More on momentum based approaches#

      +

      Let us try to get more intuition from these equations. It is helpful +to consider a simple physical analogy with a particle of mass \(m\) +moving in a viscous medium with drag coefficient \(\mu\) and potential +\(E(\mathbf{w})\). If we denote the particle’s position by \(\mathbf{w}\), +then its motion is described by

      +
      +\[ +m {d^2 \mathbf{w} \over dt^2} + \mu {d \mathbf{w} \over dt }= -\nabla_w E(\mathbf{w}). +\]
      +

      We can discretize this equation in the usual way to get

      +
      +\[ +m { \mathbf{w}_{t+\Delta t}-2 \mathbf{w}_{t} +\mathbf{w}_{t-\Delta t} \over (\Delta t)^2}+\mu {\mathbf{w}_{t+\Delta t}- \mathbf{w}_{t} \over \Delta t} = -\nabla_w E(\mathbf{w}). +\]
      +

      Rearranging this equation, we can rewrite this as

      +
      +\[ +\Delta \mathbf{w}_{t +\Delta t}= - { (\Delta t)^2 \over m +\mu \Delta t} \nabla_w E(\mathbf{w})+ {m \over m +\mu \Delta t} \Delta \mathbf{w}_t. +\]
      +
      +
      +

      Momentum parameter#

      +

      Notice that this equation is identical to previous one if we identify +the position of the particle, \(\mathbf{w}\), with the parameters +\(\boldsymbol{\theta}\). This allows us to identify the momentum +parameter and learning rate with the mass of the particle and the +viscous drag as:

      +
      +\[ +\gamma= {m \over m +\mu \Delta t }, \qquad \eta = {(\Delta t)^2 \over m +\mu \Delta t}. +\]
      +

      Thus, as the name suggests, the momentum parameter is proportional to +the mass of the particle and effectively provides inertia. +Furthermore, in the large viscosity/small learning rate limit, our +memory time scales as \((1-\gamma)^{-1} \approx m/(\mu \Delta t)\).

      +

      Why is momentum useful? SGD momentum helps the gradient descent +algorithm gain speed in directions with persistent but small gradients +even in the presence of stochasticity, while suppressing oscillations +in high-curvature directions. This becomes especially important in +situations where the landscape is shallow and flat in some directions +and narrow and steep in others. It has been argued that first-order +methods (with appropriate initial conditions) can perform comparable +to more expensive second order methods, especially in the context of +complex deep learning models.

      +

      These beneficial properties of momentum can sometimes become even more +pronounced by using a slight modification of the classical momentum +algorithm called Nesterov Accelerated Gradient (NAG).

      +

      In the NAG algorithm, rather than calculating the gradient at the +current parameters, \(\nabla_\theta E(\boldsymbol{\theta}_t)\), one +calculates the gradient at the expected value of the parameters given +our current momentum, \(\nabla_\theta E(\boldsymbol{\theta}_t +\gamma +\mathbf{v}_{t-1})\). This yields the NAG update rule

      +
      +\[ +\mathbf{v}_{t}=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \mathbf{v}_{t-1}) \nonumber +\]
      + +
      +
      +\[ +\begin{equation} +\boldsymbol{\theta}_{t+1}= \boldsymbol{\theta}_t -\mathbf{v}_{t}. +\label{_auto2} \tag{2} +\end{equation} +\]
      +

      One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of \(\gamma\).

      +
      +
      +

      Second moment of the gradient#

      +

      In stochastic gradient descent, with and without momentum, we still +have to specify a schedule for tuning the learning rates \(\eta_t\) +as a function of time. As discussed in the context of Newton’s +method, this presents a number of dilemmas. The learning rate is +limited by the steepest direction which can change depending on the +current position in the landscape. To circumvent this problem, ideally +our algorithm would keep track of curvature and take large steps in +shallow, flat directions and small steps in steep, narrow directions. +Second-order methods accomplish this by calculating or approximating +the Hessian and normalizing the learning rate by the +curvature. However, this is very computationally expensive for +extremely large models. Ideally, we would like to be able to +adaptively change the step size to match the landscape without paying +the steep computational price of calculating or approximating +Hessians.

      +

      During the last decade a number of methods have been introduced that accomplish +this by tracking not only the gradient, but also the second moment of +the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and +ADAM.

      +
      +
      +

      Challenge: Choosing a Fixed Learning Rate#

      +

      A fixed \(\eta\) is hard to get right:

      +
        +

      1. If \(\eta\) is too large, the updates can overshoot the minimum, causing oscillations or divergence

        2. +

      2. If \(\eta\) is too small, convergence is very slow (many iterations to make progress)

        3. +
      +

      In practice, one often uses trial-and-error or schedules (decaying \(\eta\) over time) to find a workable balance. +For a function with steep directions and flat directions, a single global \(\eta\) may be inappropriate:

      +
        +

      1. Steep coordinates require a smaller step size to avoid oscillation.

        2. +

      2. Flat/shallow coordinates could use a larger step to speed up progress.

        3. +

      3. This issue is pronounced in high-dimensional problems with sparse or varying-scale features – we need a method to adjust step sizesper feature.

        4. +
      +
      +
      +

      Motivation for Adaptive Step Sizes#

      +
        +

      1. Instead of a fixed global \(\eta\), use an \textbf{adaptive learning rate} for each parameter that depends on the history of gradients.

        2. +

      2. Parameters that have large accumulated gradient magnitude should get smaller steps (they’ve been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.

        3. +

      3. This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected

        4. +

      4. Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates

        5. +

      5. Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive AdaGrad, one of the first adaptive methods.

        6. +
      +
      +
      +

      Derivation of the AdaGrad Algorithm#

      +

      Accumulating Gradient History.

      +
        +

      1. AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)

        2. +

      2. Let \(g_t = \nabla \ell_{i_t}(x_t)\) be the gradient at step \(t\) (or a subgradient for nondifferentiable cases).

        3. +

      3. Initialize \(h_0 = 0\) (an all-zero vector in \(\mathbb{R}^d\)).

        4. +

      4. At each iteration \(t\), update the accumulation:

        5. +
      +
      +\[ +h_t \;=\; h_{t-1} + g_t \circ g_t, +\]
      +
        +

      1. Here \(g_t \circ g_t\) denotes element-wise square of the gradient vector. \(h_t^{(j)} = h_{t-1}^{(j)} + (g_{t,j})^2\) for each parameter \(j\).

        2. +

      2. We can view \(H_t = \mathrm{diag}(h_t)\) as a diagonal matrix of past squared gradients. Initially \(H_0 = 0\).

        3. +
      +
      +
      +

      AdaGrad Update Rule Derivation#

      +

      We scale the gradient by the inverse square root of the accumulated matrix \(H_t\). The AdaGrad update at step \(t\) is:

      +
      +\[ +x_{t+1} \;=\; x_t \;-\; \eta\, H_t^{-1/2}\, g_t, +\]
      +

      where \(H_t^{-1/2}\) is the diagonal matrix with entries \((h_{t}^{(1)})^{-1/2}, \dots, (h_{t}^{(d)})^{-1/2}\) +In coordinates, this means each parameter \(j\) has an individual step size:

      +
      +\[ +x_{t+1,j} =x_{t,j} -\frac{\eta}{\sqrt{h_{t,j}}}g_{t,j}. +\]
      +

      In practice we add a small constant \(\epsilon\) in the denominator for numerical stability to avoid division by zero:

      +
      +\[ +x_{t+1,j}= x_{t,j}-\frac{\eta}{\sqrt{\epsilon + h_{t,j}}}g_{t,j}. +\]
      +

      Equivalently, the effective learning rate for parameter \(j\) at time \(t\) is \(\displaystyle \alpha_{t,j} = \frac{\eta}{\sqrt{\epsilon + h_{t,j}}}\). This decreases over time as \(h_{t,j}\) grows.

      +
      +
      +

      AdaGrad Properties#

      +
        +

      1. AdaGrad automatically tunes the step size for each parameter. Parameters with more volatile or large gradients get smaller steps, and those with small or infrequent gradients get relatively larger steps

        2. +

      2. No manual schedule needed: The accumulation \(h_t\) keeps increasing (or stays the same if gradient is zero), so step sizes \(\eta/\sqrt{h_t}\) are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.

        3. +

      3. Sparse data benefit: For very sparse features, \(h_{t,j}\) grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal

        4. +

      4. Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate (e.g. \(O(1/\sqrt{T})\) regret bound) comparable to the best fixed learning rate tuned for the problem

        5. +
      +

      It effectively reduces the need to tune \(\eta\) by hand.

      +
        +

      1. Limitations: Because \(h_t\) accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)

        2. +
      +
      +
      +

      RMSProp: Adaptive Learning Rates#

      +

      Addresses AdaGrad’s diminishing learning rate issue. +Uses a decaying average of squared gradients (instead of a cumulative sum):

      +
      +\[ +v_t = \beta_2\, v_{t-1} + (1-\beta_2)\, (\nabla L(w_t))^2, +\]
      +

      with \(\beta_2\) typically \(0.9\) (or \(0.99\)).

      +
        +

      1. Update: \(w_{t+1} = w_t - \frac{\alpha}{\sqrt{v_t + \epsilon}} \nabla L(w_t)\).

        2. +

      2. Recent gradients have more weight, so \(v_t\) adapts to the current landscape.

        3. +

      3. Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.

        4. +
      +

      RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 - unpublished.)

      +
      +
      +

      Adam Optimizer#

      +

      Why Combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.

      +
        +

      1. Fast convergence by smoothing gradients (accelerates in long-term gradient direction).

        2. +

      2. Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).

        3. +

      3. Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)

        4. +

      4. Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)

        5. +
      +

      Result: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice.

      +
      +
      +

      RMS prop#

      +

      In RMS prop, in addition to keeping a running average of the first +moment of the gradient, we also keep track of the second moment +denoted by \(\mathbf{s}_t=\mathbb{E}[\mathbf{g}_t^2]\). The update rule +for RMS prop is given by

      + +
      +
      +\[ +\begin{equation} +\mathbf{g}_t = \nabla_\theta E(\boldsymbol{\theta}) +\label{_auto3} \tag{3} +\end{equation} +\]
      +
      +\[ +\mathbf{s}_t =\beta \mathbf{s}_{t-1} +(1-\beta)\mathbf{g}_t^2 \nonumber +\]
      +
      +\[ +\boldsymbol{\theta}_{t+1}=\boldsymbol{\theta}_t - \eta_t { \mathbf{g}_t \over \sqrt{\mathbf{s}_t +\epsilon}}, \nonumber +\]
      +

      where \(\beta\) controls the averaging time of the second moment and is +typically taken to be about \(\beta=0.9\), \(\eta_t\) is a learning rate +typically chosen to be \(10^{-3}\), and \(\epsilon\sim 10^{-8} \) is a +small regularization constant to prevent divergences. Multiplication +and division by vectors is understood as an element-wise operation. It +is clear from this formula that the learning rate is reduced in +directions where the norm of the gradient is consistently large. This +greatly speeds up the convergence by allowing us to use a larger +learning rate for flat directions.

      +
      +
      +

      ADAM optimizer#

      +

      A related algorithm is the ADAM optimizer. In +ADAM, we keep a running average of +both the first and second moment of the gradient and use this +information to adaptively change the learning rate for different +parameters. The method isefficient when working with large +problems involving lots data and/or parameters. It is a combination of the +gradient descent with momentum algorithm and the RMSprop algorithm +discussed above.

      +

      In addition to keeping a running average of the first and +second moments of the gradient +(i.e. \(\mathbf{m}_t=\mathbb{E}[\mathbf{g}_t]\) and +\(\mathbf{s}_t=\mathbb{E}[\mathbf{g}^2_t]\), respectively), ADAM +performs an additional bias correction to account for the fact that we +are estimating the first two moments of the gradient using a running +average (denoted by the hats in the update rule below). The update +rule for ADAM is given by (where multiplication and division are once +again understood to be element-wise operations below)

      + +
      +
      +\[ +\begin{equation} +\mathbf{g}_t = \nabla_\theta E(\boldsymbol{\theta}) +\label{_auto4} \tag{4} +\end{equation} +\]
      +
      +\[ +\mathbf{m}_t = \beta_1 \mathbf{m}_{t-1} + (1-\beta_1) \mathbf{g}_t \nonumber +\]
      +
      +\[ +\mathbf{s}_t =\beta_2 \mathbf{s}_{t-1} +(1-\beta_2)\mathbf{g}_t^2 \nonumber +\]
      +
      +\[ +\boldsymbol{\mathbf{m}}_t={\mathbf{m}_t \over 1-\beta_1^t} \nonumber +\]
      +
      +\[ +\boldsymbol{\mathbf{s}}_t ={\mathbf{s}_t \over1-\beta_2^t} \nonumber +\]
      +
      +\[ +\boldsymbol{\theta}_{t+1}=\boldsymbol{\theta}_t - \eta_t { \boldsymbol{\mathbf{m}}_t \over \sqrt{\boldsymbol{\mathbf{s}}_t} +\epsilon}, \nonumber +\]
      + +
      +
      +\[ +\begin{equation} +\label{_auto5} \tag{5} +\end{equation} +\]
      +

      where \(\beta_1\) and \(\beta_2\) set the memory lifetime of the first and +second moment and are typically taken to be \(0.9\) and \(0.99\) +respectively, and \(\eta\) and \(\epsilon\) are identical to RMSprop.

      +

      Like in RMSprop, the effective step size of a parameter depends on the +magnitude of its gradient squared. To understand this better, let us +rewrite this expression in terms of the variance +\(\boldsymbol{\sigma}_t^2 = \boldsymbol{\mathbf{s}}_t - +(\boldsymbol{\mathbf{m}}_t)^2\). Consider a single parameter \(\theta_t\). The +update rule for this parameter is given by

      +
      +\[ +\Delta \theta_{t+1}= -\eta_t { \boldsymbol{m}_t \over \sqrt{\sigma_t^2 + m_t^2 }+\epsilon}. +\]
      +
      +
      +

      Algorithms and codes for Adagrad, RMSprop and Adam#

      +

      The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.

      +

      The codes which implement these algorithms are discussed after our presentation of automatic differentiation.

      +
      +
      +

      AdaGrad algorithm, taken from Goodfellow et al#

      + + +

      Figure 1:

      +
      +
      +

      RMSProp algorithm, taken from Goodfellow et al#

      + + +

      Figure 1:

      +
      +
      +

      ADAM algorithm, taken from Goodfellow et al#

      + + +

      Figure 1:

      +
      +
      +

      Practical tips#

      +
        +

      • Randomize the data when making mini-batches. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.

        • +

      • Transform your inputs. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.

        • +

      • Monitor the out-of-sample performance. Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This early stopping significantly improves performance in many settings.

        • +

      • Adaptive optimization methods don’t always have good generalization. Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.

        • +
      +
      +
      +

      Sneaking in automatic differentiation using Autograd#

      +

      In the examples here we take the liberty of sneaking in automatic +differentiation (without having discussed the mathematics). In +project 1 you will write the gradients as discussed above, that is +hard-coding the gradients. By introducing automatic differentiation +via the library autograd, which is now replaced by JAX, we have +more flexibility in setting up alternative cost functions.

      +

      The +first example shows results with ordinary leats squares.

      +
      +
      +
      # Using Autograd to calculate gradients for OLS
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+def CostOLS(beta):
+    return (1.0/n)*np.sum((y-X @ beta)**2)
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
+# define the gradient
+training_gradient = grad(CostOLS)
+
+for iter in range(Niterations):
+    gradients = training_gradient(theta)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+xnew = np.array([[0],[2]])
+Xnew = np.c_[np.ones((2,1)), xnew]
+ypredict = Xnew.dot(theta)
+ypredict2 = Xnew.dot(theta_linreg)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
      +
      +
      +
      +
      +
      +

      Same code but now with momentum gradient descent#

      +
      +
      +
      # Using Autograd to calculate gradients for OLS
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+def CostOLS(beta):
+    return (1.0/n)*np.sum((y-X @ beta)**2)
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x#+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 30
+
+# define the gradient
+training_gradient = grad(CostOLS)
+
+for iter in range(Niterations):
+    gradients = training_gradient(theta)
+    theta -= eta*gradients
+    print(iter,gradients[0],gradients[1])
+print("theta from own gd")
+print(theta)
+
+# Now improve with momentum gradient descent
+change = 0.0
+delta_momentum = 0.3
+for iter in range(Niterations):
+    # calculate gradient
+    gradients = training_gradient(theta)
+    # calculate update
+    new_change = eta*gradients+delta_momentum*change
+    # take a step
+    theta -= new_change
+    # save the change
+    change = new_change
+    print(iter,gradients[0],gradients[1])
+print("theta from own gd wth momentum")
+print(theta)
+
      +
      +
      +
      +
      +
      +

      Including Stochastic Gradient Descent with Autograd#

      +

      In this code we include the stochastic gradient descent approach +discussed above. Note here that we specify which argument we are +taking the derivative with respect to when using autograd.

      +
      +
      +
      # Using Autograd to calculate gradients using SGD
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+
+for iter in range(Niterations):
+    gradients = (1.0/n)*training_gradient(y, X, theta)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+xnew = np.array([[0],[2]])
+Xnew = np.c_[np.ones((2,1)), xnew]
+ypredict = Xnew.dot(theta)
+ypredict2 = Xnew.dot(theta_linreg)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+t0, t1 = 5, 50
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
+
+for epoch in range(n_epochs):
+# Can you figure out a better way of setting up the contributions to each batch?
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        eta = learning_schedule(epoch*m+i)
+        theta = theta - eta*gradients
+print("theta from own sdg")
+print(theta)
+
      +
      +
      +
      +
      +
      +

      Same code but now with momentum gradient descent#

      +
      +
      +
      # Using Autograd to calculate gradients using SGD
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
+
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 100
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+
+for iter in range(Niterations):
+    gradients = (1.0/n)*training_gradient(y, X, theta)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+t0, t1 = 5, 50
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
+
+change = 0.0
+delta_momentum = 0.3
+
+for epoch in range(n_epochs):
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        eta = learning_schedule(epoch*m+i)
+        # calculate update
+        new_change = eta*gradients+delta_momentum*change
+        # take a step
+        theta -= new_change
+        # save the change
+        change = new_change
+print("theta from own sdg with momentum")
+print(theta)
+
      +
      +
      +
      +
      +
      +

      But none of these can compete with Newton’s method#

      +

      Note that we here have introduced automatic differentiation

      +
      +
      +
      # Using Newton's method
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+from autograd import grad
+
+def CostOLS(beta):
+    return (1.0/n)*np.sum((y-X @ beta)**2)
+
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+5*x*x
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(beta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+# Note that here the Hessian does not depend on the parameters beta
+invH = np.linalg.pinv(H)
+beta = np.random.randn(3,1)
+Niterations = 5
+# define the gradient
+training_gradient = grad(CostOLS)
+
+for iter in range(Niterations):
+    gradients = training_gradient(beta)
+    beta -= invH @ gradients
+    print(iter,gradients[0],gradients[1])
+print("beta from own Newton code")
+print(beta)
+
      +
      +
      +
      +
      +
      +

      Similar (second order function now) problem but now with AdaGrad#

      +
      +
      +
      # Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = 0.0
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        Giter += gradients*gradients
+        update = gradients*eta/(delta+np.sqrt(Giter))
+        theta -= update
+print("theta from own AdaGrad")
+print(theta)
+
      +
      +
      +
      +

      Running this code we note an almost perfect agreement with the results from matrix inversion.

      +
      +
      +

      RMSprop for adaptive learning rate with Stochastic Gradient Descent#

      +
      +
      +
      # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = 0.0
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Accumulated gradient
+	# Scaling with rho the new and the previous results
+        Giter = (rho*Giter+(1-rho)*gradients*gradients)
+	# Taking the diagonal only and inverting
+        update = gradients*eta/(delta+np.sqrt(Giter))
+	# Hadamard product
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+
      +
      +
      +
      +
      +
      +

      And finally ADAM#

      +
      +
      +
      # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
+
      +
      +
      +
      +
      +
      +

      Material for the lab sessions#

      +
        +

      1. Exercise set for week 37 and reminder on scaling (from lab sessions of week 35)

        2. +

      2. Work on project 1

        3. +
      + +
        +

      • For more discussions of Ridge regression and calculation of averages, Wessel van Wieringen’s article is highly recommended.

        • +
      +
      +
      +

      Reminder on different scaling methods#

      +

      Before fitting a regression model, it is good practice to normalize or +standardize the features. This ensures all features are on a +comparable scale, which is especially important when using +regularization. In the exercises this week we will perform standardization, scaling each +feature to have mean 0 and standard deviation 1.

      +

      Here we compute the mean and standard deviation of each column (feature) in our design/feature matrix \(\boldsymbol{X}\). +Then we subtract the mean and divide by the standard deviation for each feature.

      +

      In the example here we +we will also center the target \(\boldsymbol{y}\) to mean \(0\). Centering \(\boldsymbol{y}\) +(and each feature) means the model does not require a separate intercept +term, the data is shifted such that the intercept is effectively 0 +. (In practice, one could include an intercept in the model and not +penalize it, but here we simplify by centering.) +Choose \(n=100\) data points and set up \(\boldsymbol{x}, \)\boldsymbol{y}\( and the design matrix \)\boldsymbol{X}$.

      +
      +
      +
      # Standardize features (zero mean, unit variance for each feature)
+X_mean = X.mean(axis=0)
+X_std = X.std(axis=0)
+X_std[X_std == 0] = 1  # safeguard to avoid division by zero for constant features
+X_norm = (X - X_mean) / X_std
+
+# Center the target to zero mean (optional, to simplify intercept handling)
+y_mean = ?
+y_centered = ?
+
      +
      +
      +
      +

      Do we need to center the values of \(y\)?

      +

      After this preprocessing, each column of \(\boldsymbol{X}_{\mathrm{norm}}\) has mean zero and standard deviation \(1\) +and \(\boldsymbol{y}_{\mathrm{centered}}\) has mean 0. This can make the optimization landscape +nicer and ensures the regularization penalty \(\lambda \sum_j +\theta_j^2\) in Ridge regression treats each coefficient fairly (since features are on the +same scale).

      +
      +
      +

      Functionality in Scikit-Learn#

      +

      Scikit-Learn has several functions which allow us to rescale the +data, normally resulting in much better results in terms of various +accuracy scores. The StandardScaler function in Scikit-Learn +ensures that for each feature/predictor we study the mean value is +zero and the variance is one (every column in the design/feature +matrix). This scaling has the drawback that it does not ensure that +we have a particular maximum or minimum in our data set. Another +function included in Scikit-Learn is the MinMaxScaler which +ensures that all features are exactly between \(0\) and \(1\). The

      +
      +
      +

      More preprocessing#

      +

      The Normalizer scales each data +point such that the feature vector has a euclidean length of one. In other words, it +projects a data point on the circle (or sphere in the case of higher dimensions) with a +radius of 1. This means every data point is scaled by a different number (by the +inverse of it’s length). +This normalization is often used when only the direction (or angle) of the data matters, +not the length of the feature vector.

      +

      The RobustScaler works similarly to the StandardScaler in that it +ensures statistical properties for each feature that guarantee that +they are on the same scale. However, the RobustScaler uses the median +and quartiles, instead of mean and variance. This makes the +RobustScaler ignore data points that are very different from the rest +(like measurement errors). These odd data points are also called +outliers, and might often lead to trouble for other scaling +techniques.

      +
      +
      +

      Frequently used scaling functions#

      +

      Many features are often scaled using standardization to improve performance. In Scikit-Learn this is given by the StandardScaler function as discussed above. It is easy however to write your own. +Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:

      +
      +\[ +x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)}, +\]
      +

      where \(\overline{x}_j\) and \(\sigma(x_j)\) are the mean and standard deviation, respectively, of the feature \(x_j\). +This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don’t wish to calculate it, it is then common to simply set it to one.

      +

      Keep in mind that when you transform your data set before training a model, the same transformation needs to be done +on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as

      +
      +
      +
      """
+#Model training, we compute the mean value of y and X
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+# The we fit our model with the training data
+trained_model = some_model.fit(X_train,y_train)
+
+
+#Model prediction, we need also to transform our data set used for the prediction.
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+"""
+
      +
      +
      +
      +

      Let us try to understand what this may imply mathematically when we +subtract the mean values, also known as zero centering. For +simplicity, we will focus on ordinary regression, as done in the above example.

      +

      The cost/loss function for regression is

      +
      +\[ +C(\theta_0, \theta_1, ... , \theta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \theta_0 - \sum_{j=1}^{p-1} X_{ij}\theta_j\right)^2,. +\]
      +

      Recall also that we use the squared value. This expression can lead to an +increased penalty for higher differences between predicted and +output/target values.

      +

      What we have done is to single out the \(\theta_0\) term in the +definition of the mean squared error (MSE). The design matrix \(X\) +does in this case not contain any intercept column. When we take the +derivative with respect to \(\theta_0\), we want the derivative to obey

      +
      +\[ +\frac{\partial C}{\partial \theta_j} = 0, +\]
      +

      for all \(j\). For \(\theta_0\) we have

      +
      +\[ +\frac{\partial C}{\partial \theta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \theta_0 - \sum_{j=1}^{p-1} X_{ij} \theta_j\right). +\]
      +

      Multiplying away the constant \(2/n\), we obtain

      +
      +\[ +\sum_{i=0}^{n-1} \theta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \theta_j. +\]
      +

      Let us specialize first to the case where we have only two parameters \(\theta_0\) and \(\theta_1\). +Our result for \(\theta_0\) simplifies then to

      +
      +\[ +n\theta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \theta_1. +\]
      +

      We obtain then

      +
      +\[ +\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \theta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. +\]
      +

      If we define

      +
      +\[ +\mu_{\boldsymbol{x}_1}=\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}, +\]
      +

      and the mean value of the outputs as

      +
      +\[ +\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, +\]
      +

      we have

      +
      +\[ +\theta_0 = \mu_y - \theta_1\mu_{\boldsymbol{x}_1}. +\]
      +

      In the general case with more parameters than \(\theta_0\) and \(\theta_1\), we have

      +
      +\[ +\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\theta_j. +\]
      +

      We can rewrite the latter equation as

      +
      +\[ +\theta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \sum_{j=1}^{p-1} \mu_{\boldsymbol{x}_j}\theta_j, +\]
      +

      where we have defined

      +
      +\[ +\mu_{\boldsymbol{x}_j}=\frac{1}{n}\sum_{i=0}^{n-1} X_{ij}, +\]
      +

      the mean value for all elements of the column vector \(\boldsymbol{x}_j\).

      +

      Replacing \(y_i\) with \(y_i - y_i - \overline{\boldsymbol{y}}\) and centering also our design matrix results in a cost function (in vector-matrix disguise)

      +
      +\[ +C(\boldsymbol{\theta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\theta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\theta}). +\]
      +

      If we minimize with respect to \(\boldsymbol{\theta}\) we have then

      +
      +\[ +\hat{\boldsymbol{\theta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, +\]
      +

      where \(\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}}\) +and \(\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}\).

      +

      For Ridge regression we need to add \(\lambda \boldsymbol{\theta}^T\boldsymbol{\theta}\) to the cost function and get then

      +
      +\[ +\hat{\boldsymbol{\theta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. +\]
      +

      What does this mean? And why do we insist on all this? Let us look at some examples.

      +

      This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. +Note also that we do not split the data into training and test.

      +
      +
      +
      import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+def fit_theta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_theta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_theta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+theta = fit_theta(X, y)
+
+# Intercept is included in the design matrix
+skl = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True theta: {true_theta}")
+print(f"Fitted theta: {theta}")
+print(f"Sklearn fitted theta: {skl.coef_}")
+ypredictOwn = X @ theta
+ypredictSKL = skl.predict(X)
+print(f"MSE with intercept column")
+print(MSE(y,ypredictOwn))
+print(f"MSE with intercept column from SKL")
+print(MSE(y,ypredictSKL))
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ theta, label="Fit")
+plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+skl = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+theta = fit_theta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ theta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted theta (without intercept): {theta}")
+print(f"Sklearn intercept: {skl.intercept_}")
+print(f"Sklearn fitted theta (without intercept): {skl.coef_}")
+ypredictOwn = X @ theta
+ypredictSKL = skl.predict(X)
+print(f"MSE with Manual intercept")
+print(MSE(y,ypredictOwn+intercept))
+print(f"MSE with Sklearn intercept")
+print(MSE(y,ypredictSKL))
+
+plt.plot(x, X @ theta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
+plt.show()
+
      +
      +
      +
      +

      The intercept is the value of our output/target variable +when all our features are zero and our function crosses the \(y\)-axis (for a one-dimensional case).

      +

      Printing the MSE, we see first that both methods give the same MSE, as +they should. However, when we move to for example Ridge regression, +the way we treat the intercept may give a larger or smaller MSE, +meaning that the MSE can be penalized by the value of the +intercept. Not including the intercept in the fit, means that the +regularization term does not include \(\theta_0\). For different values +of \(\lambda\), this may lead to different MSE values.

      +

      To remind the reader, the regularization term, with the intercept in Ridge regression, is given by

      +
      +\[ +\lambda \vert\vert \boldsymbol{\theta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\theta_j^2, +\]
      +

      but when we take out the intercept, this equation becomes

      +
      +\[ +\lambda \vert\vert \boldsymbol{\theta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\theta_j^2. +\]
      +

      For Lasso regression we have

      +
      +\[ +\lambda \vert\vert \boldsymbol{\theta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\theta_j\vert. +\]
      +

      It means that, when scaling the design matrix and the outputs/targets, +by subtracting the mean values, we have an optimization problem which +is not penalized by the intercept. The MSE value can then be smaller +since it focuses only on the remaining quantities. If we however bring +back the intercept, we will get a MSE which then contains the +intercept.

      +

      Armed with this wisdom, we attempt first to simply set the intercept equal to False in our implementation of Ridge regression for our well-known vanilla data set.

      +
      +
      +
      import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 20
+X = np.zeros((n,Maxpolydegree))
+#We include explicitely the intercept column
+for degree in range(Maxpolydegree):
+    X[:,degree] = x**degree
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+p = Maxpolydegree
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 6
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 2, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeTheta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
+    # Note: we include the intercept column and no scaling
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    # and then make the prediction
+    ytildeOwnRidge = X_train @ OwnRidgeTheta
+    ypredictOwnRidge = X_test @ OwnRidgeTheta
+    ytildeRidge = RegRidge.predict(X_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    print("Theta values for own Ridge implementation")
+    print(OwnRidgeTheta)
+    print("Theta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print("MSE values for own Ridge implementation")
+    print(MSEOwnRidgePredict[i])
+    print("MSE values for Scikit-Learn Ridge implementation")
+    print(MSERidgePredict[i])
+
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
      +
      +
      +
      +

      The results here agree when we force Scikit-Learn’s Ridge function to include the first column in our design matrix. +We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix. +What happens if we do not include the intercept in our fit? +Let us see how we can change this code by zero centering.

      +
      +
      +
      import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 20
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+#Center by removing mean from each feature
+X_train_scaled = X_train - X_train_mean 
+X_test_scaled = X_test - X_train_mean
+#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)
+#Remove the intercept from the training data.
+y_scaler = np.mean(y_train)           
+y_train_scaled = y_train - y_scaler   
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 6
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 2, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeTheta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeTheta #The intercept can be shifted so the model can predict on uncentered data
+    #Add intercept to prediction
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeTheta + y_scaler 
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    print("Theta values for own Ridge implementation")
+    print(OwnRidgeTheta) #Intercept is given by mean of target variable
+    print("Theta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+    print("MSE values for own Ridge implementation")
+    print(MSEOwnRidgePredict[i])
+    print("MSE values for Scikit-Learn Ridge implementation")
+    print(MSERidgePredict[i])
+
+
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
      +
      +
      +
      +

      We see here, when compared to the code which includes explicitely the +intercept column, that our MSE value is actually smaller. This is +because the regularization term does not include the intercept value +\(\theta_0\) in the fitting. This applies to Lasso regularization as +well. It means that our optimization is now done only with the +centered matrix and/or vector that enter the fitting procedure.

      +
      +
      + + + + +
      + + + + + + + + +
      + + + +
      + + +
      +
      + Contents +
      +
      + +
      + + +
      +
      + +
      + +
      + +

      +By Morten Hjorth-Jensen +

      + +
      + +
      + + +

      + + © Copyright 2023. +
      + +

      + +
      + +
      + +
      + +
      + +
      + +
      +
      + + +
      +
      +
      + + + + + +
      +
      + + \ No newline at end of file diff --git a/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb b/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb index 23a5a9d27..81ba6bd1e 100644 --- a/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb +++ b/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "1b941c35", + "id": "8e6632a0", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "dc05b096", + "id": "82705c4f", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "2cf07405", + "id": "921bf331", "metadata": { "editable": true }, @@ -46,7 +46,7 @@ }, { "cell_type": "markdown", - "id": "3c139edb", + "id": "adff65d5", "metadata": { "editable": true }, @@ -58,7 +58,7 @@ }, { "cell_type": "markdown", - "id": "aad4cfac", + "id": "70418b3d", "metadata": { "editable": true }, @@ -70,7 +70,7 @@ }, { "cell_type": "markdown", - "id": "6682282f", + "id": "11a3cf73", "metadata": { "editable": true }, @@ -83,7 +83,7 @@ }, { "cell_type": "markdown", - "id": "89e2f4c4", + "id": "04a06b51", "metadata": { "editable": true }, @@ -99,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "b06d4e53", + "id": "408db3d9", "metadata": { "editable": true }, @@ -120,7 +120,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "63796480", + "id": "37fb732c", "metadata": { "collapsed": false, "editable": true @@ -140,12 +140,12 @@ }, { "cell_type": "markdown", - "id": "80748600", + "id": "d861e1e3", "metadata": { "editable": true }, "source": [ - "Fill in the necessary details.\n", + "Fill in the necessary details. Do we need to center the $y$-values? \n", "\n", "After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n", "and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This makes the optimization landscape\n", @@ -156,7 +156,7 @@ }, { "cell_type": "markdown", - "id": "92751e5f", + "id": "b3e774d0", "metadata": { "editable": true }, @@ -168,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "aedfbd7a", + "id": "d5dc7708", "metadata": { "editable": true }, @@ -179,7 +179,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "5d1288fa", + "id": "4c9c86ac", "metadata": { "collapsed": false, "editable": true @@ -187,7 +187,9 @@ "outputs": [], "source": [ "# Set regularization parameter, either a single value or a vector of values\n", - "lambda = ?\n", + "# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called \"anonymous functions\" or \"lambda functions.\"\n", + "lam = ?\n", + "\n", "\n", "# Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y\n", "I = np.eye(n_features)\n", @@ -200,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "628f5e89", + "id": "eeae00fd", "metadata": { "editable": true }, @@ -214,7 +216,7 @@ }, { "cell_type": "markdown", - "id": "f115ba4e", + "id": "e1c215d5", "metadata": { "editable": true }, @@ -226,7 +228,7 @@ }, { "cell_type": "markdown", - "id": "a9b5189c", + "id": "587dd3dc", "metadata": { "editable": true }, @@ -238,7 +240,7 @@ }, { "cell_type": "markdown", - "id": "a3969ff6", + "id": "bfa34697", "metadata": { "editable": true }, @@ -258,7 +260,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "34d87303", + "id": "49245f55", "metadata": { "collapsed": false, "editable": true @@ -273,19 +275,8 @@ "# Initialize weights for gradient descent\n", "theta = np.zeros(n_features)\n", "\n", - "# Arrays to store history for plotting\n", - "cost_history = np.zeros(num_iters)\n", - "\n", "# Gradient descent loop\n", - "m = n_samples # number of data points\n", "for t in range(num_iters):\n", - " # Compute prediction error\n", - " error = X_norm.dot(theta) - y_centered \n", - " # Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring\n", - " cost_OLS = ?\n", - " cost_Ridge = ?\n", - " # You could add a history for both methods (optional)\n", - " cost_history[t] = ?\n", " # Compute gradients for OSL and Ridge\n", " grad_OLS = ?\n", " grad_Ridge = ?\n", @@ -302,31 +293,33 @@ }, { "cell_type": "markdown", - "id": "989f70bb", + "id": "f3f43f2c", "metadata": { "editable": true }, "source": [ "### 4a)\n", "\n", - "Discuss the results as function of the learning rate parameters and the number of iterations." + "Write first a gradient descent code for OLS only using the above template.\n", + "Discuss the results as function of the learning rate parameters and the number of iterations" ] }, { "cell_type": "markdown", - "id": "370b2dad", + "id": "9ba303be", "metadata": { "editable": true }, "source": [ "### 4b)\n", "\n", + "Write then a similar code for Ridge regression using the above template.\n", "Try to add a stopping parameter as function of the number iterations and the difference between the new and old $\\theta$ values. How would you define a stopping criterion?" ] }, { "cell_type": "markdown", - "id": "ef197cd7", + "id": "78362c6c", "metadata": { "editable": true }, @@ -352,7 +345,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "4ccc2f65", + "id": "8be1cebe", "metadata": { "collapsed": false, "editable": true @@ -381,7 +374,7 @@ }, { "cell_type": "markdown", - "id": "00e279ef", + "id": "e2693666", "metadata": { "editable": true }, @@ -395,7 +388,7 @@ }, { "cell_type": "markdown", - "id": "c910b3f4", + "id": "bc954d12", "metadata": { "editable": true }, @@ -407,7 +400,7 @@ }, { "cell_type": "markdown", - "id": "89e6e040", + "id": "6534b610", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/jupyter_execute/exercisesweek38.ipynb b/doc/LectureNotes/_build/jupyter_execute/exercisesweek38.ipynb new file mode 100644 index 000000000..a8c3a0e50 --- /dev/null +++ b/doc/LectureNotes/_build/jupyter_execute/exercisesweek38.ipynb @@ -0,0 +1,429 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1da77599", + "metadata": {}, + "source": [ + "# Exercises week 37\n", + "## September 15-19\n", + "\n", + "## Resampling and the Bias-Variance Trade-off" + ] + }, + { + "cell_type": "markdown", + "id": "e9f27b0e", + "metadata": {}, + "source": [ + "## Learning goals\n", + "\n", + "After completing these exercises, you will know how to\n", + "- Derive expectation and variances values related to linear regression\n", + "- Compute expectation and variances values related to linear regression\n", + "- Compute and evaluate the trade-off between bias and variance of a model" + ] + }, + { + "cell_type": "markdown", + "id": "984af8e3", + "metadata": {}, + "source": [ + "This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercises are also a part of project 1 and can be reused in the theory part of the project.\n", + "\n", + "For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "\n", + "We assume that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$ which describes our data" + ] + }, + { + "cell_type": "markdown", + "id": "c16f7d0e", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9fcf981a", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "source": [ + "We further assume that this continous function can be modeled with a linear model $\\mathbf{\\tilde{y}}$ of some features $\\mathbf{X}$." + ] + }, + { + "cell_type": "markdown", + "id": "d4189366", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{\\tilde{y}} + \\boldsymbol{\\varepsilon} = \\boldsymbol{X}\\boldsymbol{\\beta} +\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f4fca21b", + "metadata": {}, + "source": [ + "We therefore get that our data $\\boldsymbol{y}$ has an expectation value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$, that is $\\boldsymbol{y}$ follows a normal distribution with mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$." + ] + }, + { + "cell_type": "markdown", + "id": "5de0c7e6", + "metadata": {}, + "source": [ + "## Exercise 1: Expectation values for ordinary least squares expressions" + ] + }, + { + "cell_type": "markdown", + "id": "d878c699", + "metadata": {}, + "source": [ + "**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ show that" + ] + }, + { + "cell_type": "markdown", + "id": "08b7007d", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "46e93394", + "metadata": {}, + "source": [ + "**b)** Show that the variance of $\\boldsymbol{\\hat{\\beta}_{OLS}}$ is" + ] + }, + { + "cell_type": "markdown", + "id": "be1b65be", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{Var}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d2143684", + "metadata": {}, + "source": [ + "We can use the last expression when we define a [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval) for the parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$. \n", + "A given parameter ${\\boldsymbol{\\hat{\\beta}_{OLS}}}_j$ is given by the diagonal matrix element of the above matrix." + ] + }, + { + "cell_type": "markdown", + "id": "f5c2dc22", + "metadata": {}, + "source": [ + "## Exercise 2: Expectation values for Ridge regression" + ] + }, + { + "cell_type": "markdown", + "id": "3893e3e7", + "metadata": {}, + "source": [ + "**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{Ridge}}$ show that" + ] + }, + { + "cell_type": "markdown", + "id": "79dc571f", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "028209a1", + "metadata": {}, + "source": [ + "We see that $\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big] \\not= \\mathbb{E} \\big[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}\\big ]$ for any $\\lambda > 0$." + ] + }, + { + "cell_type": "markdown", + "id": "65f6f914", + "metadata": {}, + "source": [ + "**b)** Why do we say that Ridge regression gives a biased estimate? Is this a problem?" + ] + }, + { + "cell_type": "markdown", + "id": "b4e721fc", + "metadata": {}, + "source": [ + "**c)** Show that the variance is" + ] + }, + { + "cell_type": "markdown", + "id": "090eb1e1", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6b8e8697", + "metadata": {}, + "source": [ + "We see that if the parameter $\\lambda$ goes to infinity then the variance of the Ridge parameters $\\boldsymbol{\\beta}$ goes to zero." + ] + }, + { + "cell_type": "markdown", + "id": "74bc300b", + "metadata": {}, + "source": [ + "## Exercise 3: Deriving the expression for the Bias-Variance Trade-off" + ] + }, + { + "cell_type": "markdown", + "id": "eeb86010", + "metadata": {}, + "source": [ + "The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.\n", + "\n", + "The parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ are found by optimizing the mean squared error via the so-called cost function" + ] + }, + { + "cell_type": "markdown", + "id": "522a0d1d", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "831db06c", + "metadata": {}, + "source": [ + "**a)** Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a\n", + "term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function $f$ with the model ${\\bf y}$.\n", + "show that" + ] + }, + { + "cell_type": "markdown", + "id": "8cc52b3c", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8cb50416", + "metadata": {}, + "source": [ + "with" + ] + }, + { + "cell_type": "markdown", + "id": "e49bdbb4", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "eca5554a", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "id": "b1054343", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{var}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\tilde{\\boldsymbol{y}}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "70fbfcd7", + "metadata": {}, + "source": [ + "**b)** Explain what the terms mean and discuss their interpretations." + ] + }, + { + "cell_type": "markdown", + "id": "b8f8b9d1", + "metadata": {}, + "source": [ + "## Exercise 4: Computing the Bias and Variance" + ] + }, + { + "cell_type": "markdown", + "id": "9e012430", + "metadata": {}, + "source": [ + "Before you compute the bias and variance of a real model for different complexities, let's for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.\n", + "\n", + "**a)** Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "b5bf581c", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "n = 100\n", + "bootstraps = 1000\n", + "\n", + "predictions = np.random.rand(bootstraps, n) * 10 + 10\n", + "targets = np.random.rand(bootstraps, n)\n", + "\n", + "mse = ...\n", + "bias = ...\n", + "variance = ..." + ] + }, + { + "cell_type": "markdown", + "id": "7b1dc621", + "metadata": {}, + "source": [ + "**b)** Change the prediction values in some way to increase the bias while decreasing the variance.\n", + "\n", + "**c)** Change the prediction values in some way to increase the variance while decreasing the bias." + ] + }, + { + "cell_type": "markdown", + "id": "8da63362", + "metadata": {}, + "source": [ + "**d)** Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "dd5855e4", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.preprocessing import PolynomialFeatures # use the fit_transform method of the created object!\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.utils import resample" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e35fa37", + "metadata": {}, + "outputs": [], + "source": [ + "n = 100\n", + "bootstraps = 1000\n", + "\n", + "x = np.linspace(-3, 3, n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1)\n", + "\n", + "biases = []\n", + "variances = []\n", + "mses = []\n", + "\n", + "#for p in range(1, 5):\n", + "# predictions = ...\n", + "# targets = ...\n", + "# for b in range(bootstraps):\n", + "# x_sample, y_sample = ...\n", + "# X = ...\n", + "# X_train, X_test, y_train, y_test = ...\n", + "#\n", + "# predictions[b, :] = \n", + "# targets[b, :] = \n", + "# \n", + "# biases.append(...)\n", + "# variances.append(...)\n", + "# mses.append(...)" + ] + }, + { + "cell_type": "markdown", + "id": "253b8461", + "metadata": {}, + "source": [ + "**e)** Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).\n", + "\n", + "**f)** Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/doc/LectureNotes/_build/jupyter_execute/week37.ipynb b/doc/LectureNotes/_build/jupyter_execute/week37.ipynb new file mode 100644 index 000000000..7d4bf87cb --- /dev/null +++ b/doc/LectureNotes/_build/jupyter_execute/week37.ipynb @@ -0,0 +1,3736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c543a71a", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "4ff6b050", + "metadata": { + "editable": true + }, + "source": [ + "# Week 37: Statistical interpretations and Resampling Methods\n", + "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo, Norway\n", + "\n", + "Date: **September 8-12, 2025**\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "a7079d3e", + "metadata": { + "editable": true + }, + "source": [ + "## Plans for week 37, lecture Monday\n", + "\n", + "**Plans and material for the lecture on Monday September 8.**\n", + "\n", + "The family of gradient descent methods\n", + "1. Plain gradient descent (constant learning rate), reminder from last week with examples using OLS and Ridge\n", + "\n", + "2. Improving gradient descent with momentum\n", + "\n", + "3. Introducing stochastic gradient descent\n", + "\n", + "4. More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "db3af8cc", + "metadata": { + "editable": true + }, + "source": [ + "## Readings and Videos:\n", + "1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at and chapter 8.3-8.5 at \n", + "\n", + "2. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.\n", + "\n", + "3. Video on gradient descent at \n", + "\n", + "4. Video on Stochastic gradient descent at " + ] + }, + { + "cell_type": "markdown", + "id": "79d6d5be", + "metadata": { + "editable": true + }, + "source": [ + "## Material for lecture Monday September 8" + ] + }, + { + "cell_type": "markdown", + "id": "fa427fc3", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and revisiting Ordinary Least Squares from last week\n", + "\n", + "Last week we started with linear regression as a case study for the gradient descent\n", + "methods. Linear regression is a great test case for the gradient\n", + "descent methods discussed in the lectures since it has several\n", + "desirable properties such as:\n", + "\n", + "1. An analytical solution (recall homework sets for week 35).\n", + "\n", + "2. The gradient can be computed analytically.\n", + "\n", + "3. The cost function is convex which guarantees that gradient descent converges for small enough learning rates\n", + "\n", + "We revisit an example similar to what we had in the first homework set. We have a function of the type" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1185629e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "x = 2*np.random.rand(m,1)\n", + "y = 4+3*x+np.random.randn(m,1)" + ] + }, + { + "cell_type": "markdown", + "id": "42dfe293", + "metadata": { + "editable": true + }, + "source": [ + "with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n", + "The linear regression model is given by" + ] + }, + { + "cell_type": "markdown", + "id": "a726cc6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_\\theta(x) = \\boldsymbol{y} = \\theta_0 + \\theta_1 x,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "991c8919", + "metadata": { + "editable": true + }, + "source": [ + "such that" + ] + }, + { + "cell_type": "markdown", + "id": "ba895122", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y}_i = \\theta_0 + \\theta_1 x_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5c2b9317", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent example\n", + "\n", + "Let $\\mathbf{y} = (y_1,\\cdots,y_n)^T$, $\\mathbf{\\boldsymbol{y}} = (\\boldsymbol{y}_1,\\cdots,\\boldsymbol{y}_n)^T$ and $\\theta = (\\theta_0, \\theta_1)^T$\n", + "\n", + "It is convenient to write $\\mathbf{\\boldsymbol{y}} = X\\theta$ where $X \\in \\mathbb{R}^{100 \\times 2} $ is the design matrix given by (we keep the intercept here)" + ] + }, + { + "cell_type": "markdown", + "id": "264eef67", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "X \\equiv \\begin{bmatrix}\n", + "1 & x_1 \\\\\n", + "\\vdots & \\vdots \\\\\n", + "1 & x_{100} & \\\\\n", + "\\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a451a9ce", + "metadata": { + "editable": true + }, + "source": [ + "The cost/loss/risk function is given by (" + ] + }, + { + "cell_type": "markdown", + "id": "e7fa569b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta) = \\frac{1}{n}||X\\theta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\theta_0 + \\theta_1 x_i)^2 - 2 y_i (\\theta_0 + \\theta_1 x_i) + y_i^2\\right]\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a2655bc6", + "metadata": { + "editable": true + }, + "source": [ + "and we want to find $\\theta$ such that $C(\\theta)$ is minimized." + ] + }, + { + "cell_type": "markdown", + "id": "277ace2e", + "metadata": { + "editable": true + }, + "source": [ + "## The derivative of the cost/loss function\n", + "\n", + "Computing $\\partial C(\\theta) / \\partial \\theta_0$ and $\\partial C(\\theta) / \\partial \\theta_1$ we can show that the gradient can be written as" + ] + }, + { + "cell_type": "markdown", + "id": "75bbf03c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\theta} C(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T(X\\theta - \\mathbf{y}),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5bb2db2", + "metadata": { + "editable": true + }, + "source": [ + "where $X$ is the design matrix defined above." + ] + }, + { + "cell_type": "markdown", + "id": "53430d47", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix\n", + "The Hessian matrix of $C(\\theta)$ is given by" + ] + }, + { + "cell_type": "markdown", + "id": "89dce328", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "542b8278", + "metadata": { + "editable": true + }, + "source": [ + "This result implies that $C(\\theta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite." + ] + }, + { + "cell_type": "markdown", + "id": "2b94baca", + "metadata": { + "editable": true + }, + "source": [ + "## Simple program\n", + "\n", + "We can now write a program that minimizes $C(\\theta)$ using the gradient descent method with a constant learning rate $\\gamma$ according to" + ] + }, + { + "cell_type": "markdown", + "id": "bfc6b93f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{k+1} = \\theta_k - \\gamma \\nabla_\\theta C(\\theta_k), \\ k=0,1,\\cdots\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "703de92e", + "metadata": { + "editable": true + }, + "source": [ + "We can use the expression we computed for the gradient and let use a\n", + "$\\theta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n", + "when $||\\nabla_\\theta C(\\theta_k) || \\leq \\epsilon = 10^{-8}$. **Note that the code below does not include the latter stop criterion**.\n", + "\n", + "And finally we can compare our solution for $\\theta$ with the analytic result given by \n", + "$\\theta= (X^TX)^{-1} X^T \\mathbf{y}$." + ] + }, + { + "cell_type": "markdown", + "id": "f843aae1", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient Descent Example\n", + "\n", + "Here our simple example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b76443c9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "\n", + "# Importing various packages\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "# Hessian matrix\n", + "H = (2.0/n)* X.T @ X\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y\n", + "print(theta_linreg)\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "for iter in range(Niterations):\n", + " gradient = (2.0/n)*X.T @ (X @ theta-y)\n", + " theta -= eta*gradient\n", + "\n", + "print(theta)\n", + "xnew = np.array([[0],[2]])\n", + "xbnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = xbnew.dot(theta)\n", + "ypredict2 = xbnew.dot(theta_linreg)\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7b75f4ec", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and Ridge\n", + "\n", + "We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\\theta$," + ] + }, + { + "cell_type": "markdown", + "id": "47ecdee7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C_{\\text{ridge}}(\\theta) = \\frac{1}{n}||X\\theta -\\mathbf{y}||^2 + \\lambda ||\\theta||^2, \\ \\lambda \\geq 0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff25299b", + "metadata": { + "editable": true + }, + "source": [ + "In order to minimize $C_{\\text{ridge}}(\\theta)$ using GD we adjust the gradient as follows" + ] + }, + { + "cell_type": "markdown", + "id": "21a88a84", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\theta C_{\\text{ridge}}(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} + 2\\lambda\\begin{bmatrix} \\theta_0 \\\\ \\theta_1\\end{bmatrix} = 2 (\\frac{1}{n}X^T(X\\theta - \\mathbf{y})+\\lambda \\theta).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67491c15", + "metadata": { + "editable": true + }, + "source": [ + "We can easily extend our program to minimize $C_{\\text{ridge}}(\\theta)$ using gradient descent and compare with the analytical solution given by" + ] + }, + { + "cell_type": "markdown", + "id": "f1ac13f1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67d83802", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix for Ridge Regression\n", + "The Hessian matrix of Ridge Regression for our simple example is given by" + ] + }, + { + "cell_type": "markdown", + "id": "75935d70", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X+2\\lambda\\boldsymbol{I}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4707a65d", + "metadata": { + "editable": true + }, + "source": [ + "This implies that the Hessian matrix is positive definite, hence the stationary point is a\n", + "minimum.\n", + "Note that the Ridge cost function is convex being a sum of two convex\n", + "functions. Therefore, the stationary point is a global\n", + "minimum of this function." + ] + }, + { + "cell_type": "markdown", + "id": "ab16b163", + "metadata": { + "editable": true + }, + "source": [ + "## Program example for gradient descent with Ridge Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fe7ee1f7", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "\n", + "#Ridge parameter lambda\n", + "lmbda = 0.001\n", + "Id = n*lmbda* np.eye(XT_X.shape[0])\n", + "\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X+2*lmbda* np.eye(XT_X.shape[0])\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "\n", + "theta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y\n", + "print(theta_linreg)\n", + "# Start plain gradient descent\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ (X @ (theta)-y)+2*lmbda*theta\n", + " theta -= eta*gradients\n", + "\n", + "print(theta)\n", + "ypredict = X @ theta\n", + "ypredict2 = X @ theta_linreg\n", + "plt.plot(x, ypredict, \"r-\")\n", + "plt.plot(x, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example for Ridge')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "823fdaf7", + "metadata": { + "editable": true + }, + "source": [ + "## Using gradient descent methods, limitations\n", + "\n", + "* **Gradient descent (GD) finds local minima of our function**. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.\n", + "\n", + "* **GD is sensitive to initial conditions**. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.\n", + "\n", + "* **Gradients are computationally expensive to calculate for large datasets**. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, $E \\propto \\sum_{i=1}^n (y_i - \\mathbf{w}^T\\cdot\\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called \"mini batches\". This has the added benefit of introducing stochasticity into our algorithm.\n", + "\n", + "* **GD is very sensitive to choices of learning rates**. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.\n", + "\n", + "* **GD treats all directions in parameter space uniformly.** Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. \n", + "\n", + "* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points." + ] + }, + { + "cell_type": "markdown", + "id": "21ea3f2a", + "metadata": { + "editable": true + }, + "source": [ + "## Improving gradient descent with momentum\n", + "\n", + "We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5b729a7e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# take a step\n", + "\t\tsolution = solution - step_size * gradient\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# perform the gradient descent search\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "209fc69c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "984c4202", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# keep track of the change\n", + "\tchange = 0.0\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# calculate update\n", + "\t\tnew_change = step_size * gradient + momentum * change\n", + "\t\t# take a step\n", + "\t\tsolution = solution - new_change\n", + "\t\t# save the change\n", + "\t\tchange = new_change\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# define momentum\n", + "momentum = 0.3\n", + "# perform the gradient descent search with momentum\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "8577b238", + "metadata": { + "editable": true + }, + "source": [ + "## Overview video on Stochastic Gradient Descent (SGD)\n", + "\n", + "[What is Stochastic Gradient Descent](https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer)\n", + "There are several reasons for using stochastic gradient descent. Some of these are:\n", + "\n", + "1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.\n", + "\n", + "2. Hopefully avoid Local Minima\n", + "\n", + "3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c11d0874", + "metadata": { + "editable": true + }, + "source": [ + "## Batches and mini-batches\n", + "\n", + "In gradient descent we compute the cost function and its gradient for all data points we have.\n", + "\n", + "In large-scale applications such as the [ILSVRC challenge](https://www.image-net.org/challenges/LSVRC/), the\n", + "training data can have on order of millions of examples. Hence, it\n", + "seems wasteful to compute the full cost function over the entire\n", + "training set in order to perform only a single parameter update. A\n", + "very common approach to addressing this challenge is to compute the\n", + "gradient over batches of the training data. For example, a typical batch could contain some thousand examples from\n", + "an entire training set of several millions. This batch is then used to\n", + "perform a parameter update." + ] + }, + { + "cell_type": "markdown", + "id": "64ad52d0", + "metadata": { + "editable": true + }, + "source": [ + "## Pros and cons\n", + "\n", + "1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.\n", + "\n", + "2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.\n", + "\n", + "3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient." + ] + }, + { + "cell_type": "markdown", + "id": "5dd4c467", + "metadata": { + "editable": true + }, + "source": [ + "## Convergence rates\n", + "\n", + "1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.\n", + "\n", + "2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration." + ] + }, + { + "cell_type": "markdown", + "id": "42904823", + "metadata": { + "editable": true + }, + "source": [ + "## Accuracy\n", + "\n", + "In general, stochastic Gradient Descent is Less accurate than gradient\n", + "descent, as it calculates the gradient on single examples, which may\n", + "not accurately represent the overall dataset. Gradient Descent is\n", + "more accurate because it uses the average gradient calculated over the\n", + "entire dataset.\n", + "\n", + "There are other disadvantages to using SGD. The main drawback is that\n", + "its convergence behaviour can be more erratic due to the random\n", + "sampling of individual training examples. This can lead to less\n", + "accurate results, as the algorithm may not converge to the true\n", + "minimum of the cost function. Additionally, the learning rate, which\n", + "determines the step size of each update to the model’s parameters,\n", + "must be carefully chosen to ensure convergence.\n", + "\n", + "It is however the method of choice in deep learning algorithms where\n", + "SGD is often used in combination with other optimization techniques,\n", + "such as momentum or adaptive learning rates" + ] + }, + { + "cell_type": "markdown", + "id": "49dfaf2e", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent (SGD)\n", + "\n", + "In stochastic gradient descent, the extreme case is the case where we\n", + "have only one batch, that is we include the whole data set.\n", + "\n", + "This process is called Stochastic Gradient\n", + "Descent (SGD) (or also sometimes on-line gradient descent). This is\n", + "relatively less common to see because in practice due to vectorized\n", + "code optimizations it can be computationally much more efficient to\n", + "evaluate the gradient for 100 examples, than the gradient for one\n", + "example 100 times. Even though SGD technically refers to using a\n", + "single example at a time to evaluate the gradient, you will hear\n", + "people use the term SGD even when referring to mini-batch gradient\n", + "descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD\n", + "for “Batch gradient descent” are rare to see), where it is usually\n", + "assumed that mini-batches are used. The size of the mini-batch is a\n", + "hyperparameter but it is not very common to cross-validate or bootstrap it. It is\n", + "usually based on memory constraints (if any), or set to some value,\n", + "e.g. 32, 64 or 128. We use powers of 2 in practice because many\n", + "vectorized operation implementations work faster when their inputs are\n", + "sized in powers of 2.\n", + "\n", + "In our notes with SGD we mean stochastic gradient descent with mini-batches." + ] + }, + { + "cell_type": "markdown", + "id": "284b8663", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent\n", + "\n", + "Stochastic gradient descent (SGD) and variants thereof address some of\n", + "the shortcomings of the Gradient descent method discussed above.\n", + "\n", + "The underlying idea of SGD comes from the observation that the cost\n", + "function, which we want to minimize, can almost always be written as a\n", + "sum over $n$ data points $\\{\\mathbf{x}_i\\}_{i=1}^n$," + ] + }, + { + "cell_type": "markdown", + "id": "5e8cc1b8", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "510505a0", + "metadata": { + "editable": true + }, + "source": [ + "## Computation of gradients\n", + "\n", + "This in turn means that the gradient can be\n", + "computed as a sum over $i$-gradients" + ] + }, + { + "cell_type": "markdown", + "id": "32fc501f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\beta C(\\mathbf{\\beta}) = \\sum_i^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9dcac7ef", + "metadata": { + "editable": true + }, + "source": [ + "Stochasticity/randomness is introduced by only taking the\n", + "gradient on a subset of the data called minibatches. If there are $n$\n", + "data points and the size of each minibatch is $M$, there will be $n/M$\n", + "minibatches. We denote these minibatches by $B_k$ where\n", + "$k=1,\\cdots,n/M$." + ] + }, + { + "cell_type": "markdown", + "id": "c9464120", + "metadata": { + "editable": true + }, + "source": [ + "## SGD example\n", + "As an example, suppose we have $10$ data points $(\\mathbf{x}_1,\\cdots, \\mathbf{x}_{10})$ \n", + "and we choose to have $M=5$ minibathces,\n", + "then each minibatch contains two data points. In particular we have\n", + "$B_1 = (\\mathbf{x}_1,\\mathbf{x}_2), \\cdots, B_5 =\n", + "(\\mathbf{x}_9,\\mathbf{x}_{10})$. Note that if you choose $M=1$ you\n", + "have only a single batch with all data points and on the other extreme,\n", + "you may choose $M=n$ resulting in a minibatch for each datapoint, i.e\n", + "$B_k = \\mathbf{x}_k$.\n", + "\n", + "The idea is now to approximate the gradient by replacing the sum over\n", + "all data points with a sum over the data points in one the minibatches\n", + "picked at random in each gradient descent step" + ] + }, + { + "cell_type": "markdown", + "id": "59bc847c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\beta}\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}) \\rightarrow \\sum_{i \\in B_k}^n \\nabla_\\beta\n", + "c_i(\\mathbf{x}_i, \\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ba220299", + "metadata": { + "editable": true + }, + "source": [ + "## The gradient step\n", + "\n", + "Thus a gradient descent step now looks like" + ] + }, + { + "cell_type": "markdown", + "id": "8608ba34", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\beta_{j+1} = \\beta_j - \\gamma_j \\sum_{i \\in B_k}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta})\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0faa1782", + "metadata": { + "editable": true + }, + "source": [ + "where $k$ is picked at random with equal\n", + "probability from $[1,n/M]$. An iteration over the number of\n", + "minibathces (n/M) is commonly referred to as an epoch. Thus it is\n", + "typical to choose a number of epochs and for each epoch iterate over\n", + "the number of minibatches, as exemplified in the code below." + ] + }, + { + "cell_type": "markdown", + "id": "97eddb68", + "metadata": { + "editable": true + }, + "source": [ + "## Simple example code" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "309da17e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 10 #number of epochs\n", + "\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for \n", + " j += 1" + ] + }, + { + "cell_type": "markdown", + "id": "7aea4344", + "metadata": { + "editable": true + }, + "source": [ + "Taking the gradient only on a subset of the data has two important\n", + "benefits. First, it introduces randomness which decreases the chance\n", + "that our opmization scheme gets stuck in a local minima. Second, if\n", + "the size of the minibatches are small relative to the number of\n", + "datapoints ($M < n$), the computation of the gradient is much\n", + "cheaper since we sum over the datapoints in the $k-th$ minibatch and not\n", + "all $n$ datapoints." + ] + }, + { + "cell_type": "markdown", + "id": "52f8d549", + "metadata": { + "editable": true + }, + "source": [ + "## When do we stop?\n", + "\n", + "A natural question is when do we stop the search for a new minimum?\n", + "One possibility is to compute the full gradient after a given number\n", + "of epochs and check if the norm of the gradient is smaller than some\n", + "threshold and stop if true. However, the condition that the gradient\n", + "is zero is valid also for local minima, so this would only tell us\n", + "that we are close to a local/global minimum. However, we could also\n", + "evaluate the cost function at this point, store the result and\n", + "continue the search. If the test kicks in at a later stage we can\n", + "compare the values of the cost function and keep the $\\beta$ that\n", + "gave the lowest value." + ] + }, + { + "cell_type": "markdown", + "id": "dcd04de7", + "metadata": { + "editable": true + }, + "source": [ + "## Slightly different approach\n", + "\n", + "Another approach is to let the step length $\\gamma_j$ depend on the\n", + "number of epochs in such a way that it becomes very small after a\n", + "reasonable time such that we do not move at all. Such approaches are\n", + "also called scaling. There are many such ways to [scale the learning\n", + "rate](https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1)\n", + "and [discussions here](https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf). See\n", + "also\n", + "\n", + "for a discussion of different scaling functions for the learning rate." + ] + }, + { + "cell_type": "markdown", + "id": "4ecf0b16", + "metadata": { + "editable": true + }, + "source": [ + "## Time decay rate\n", + "\n", + "As an example, let $e = 0,1,2,3,\\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \\cdot m + i$ where $m$ is the number of minibatches and $i=0,\\cdots,m-1$. Then the function $$\\gamma_j(t; t_0, t_1) = \\frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$.\n", + "\n", + "In this way we can fix the number of epochs, compute $\\beta$ and\n", + "evaluate the cost function at the end. Repeating the computation will\n", + "give a different result since the scheme is random by design. Then we\n", + "pick the final $\\beta$ that gives the lowest value of the cost\n", + "function." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "71eb89db", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "def step_length(t,t0,t1):\n", + " return t0/(t+t1)\n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 500 #number of epochs\n", + "t0 = 1.0\n", + "t1 = 10\n", + "\n", + "gamma_j = t0/t1\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for beta\n", + " t = epoch*m+i\n", + " gamma_j = step_length(t,t0,t1)\n", + " j += 1\n", + "\n", + "print(\"gamma_j after %d epochs: %g\" % (n_epochs,gamma_j))" + ] + }, + { + "cell_type": "markdown", + "id": "10ce3fc7", + "metadata": { + "editable": true + }, + "source": [ + "## Code with a Number of Minibatches which varies\n", + "\n", + "In the code here we vary the number of mini-batches." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f2d402a8", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "from math import exp, sqrt\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ ((X @ theta)-y)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ffd1e92d", + "metadata": { + "editable": true + }, + "source": [ + "## Replace or not\n", + "\n", + "In the above code, we have use replacement in setting up the\n", + "mini-batches. The discussion\n", + "[here](https://sebastianraschka.com/faq/docs/sgd-methods.html) may be\n", + "useful." + ] + }, + { + "cell_type": "markdown", + "id": "4b72d3f9", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum based GD\n", + "\n", + "The stochastic gradient descent (SGD) is almost always used with a\n", + "*momentum* or inertia term that serves as a memory of the direction we\n", + "are moving in parameter space. This is typically implemented as\n", + "follows" + ] + }, + { + "cell_type": "markdown", + "id": "1b60ec02", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e1fa24ba", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t},\n", + "\\label{_auto1} \\tag{1}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "7a1fc9fc", + "metadata": { + "editable": true + }, + "source": [ + "where we have introduced a momentum parameter $\\gamma$, with\n", + "$0\\le\\gamma\\le 1$, and for brevity we dropped the explicit notation to\n", + "indicate the gradient is to be taken over a different mini-batch at\n", + "each step. We call this algorithm gradient descent with momentum\n", + "(GDM). From these equations, it is clear that $\\mathbf{v}_t$ is a\n", + "running average of recently encountered gradients and\n", + "$(1-\\gamma)^{-1}$ sets the characteristic time scale for the memory\n", + "used in the averaging procedure. Consistent with this, when\n", + "$\\gamma=0$, this just reduces down to ordinary SGD as discussed\n", + "earlier. An equivalent way of writing the updates is" + ] + }, + { + "cell_type": "markdown", + "id": "23abda5d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\boldsymbol{\\theta}_{t+1} = \\gamma \\Delta \\boldsymbol{\\theta}_t -\\ \\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0f0067a9", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined $\\Delta \\boldsymbol{\\theta}_{t}= \\boldsymbol{\\theta}_t-\\boldsymbol{\\theta}_{t-1}$." + ] + }, + { + "cell_type": "markdown", + "id": "b86ec959", + "metadata": { + "editable": true + }, + "source": [ + "## More on momentum based approaches\n", + "\n", + "Let us try to get more intuition from these equations. It is helpful\n", + "to consider a simple physical analogy with a particle of mass $m$\n", + "moving in a viscous medium with drag coefficient $\\mu$ and potential\n", + "$E(\\mathbf{w})$. If we denote the particle's position by $\\mathbf{w}$,\n", + "then its motion is described by" + ] + }, + { + "cell_type": "markdown", + "id": "7685ff97", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m {d^2 \\mathbf{w} \\over dt^2} + \\mu {d \\mathbf{w} \\over dt }= -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "28203c1e", + "metadata": { + "editable": true + }, + "source": [ + "We can discretize this equation in the usual way to get" + ] + }, + { + "cell_type": "markdown", + "id": "1495b23f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m { \\mathbf{w}_{t+\\Delta t}-2 \\mathbf{w}_{t} +\\mathbf{w}_{t-\\Delta t} \\over (\\Delta t)^2}+\\mu {\\mathbf{w}_{t+\\Delta t}- \\mathbf{w}_{t} \\over \\Delta t} = -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "15bbc6e6", + "metadata": { + "editable": true + }, + "source": [ + "Rearranging this equation, we can rewrite this as" + ] + }, + { + "cell_type": "markdown", + "id": "ffeffc05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\mathbf{w}_{t +\\Delta t}= - { (\\Delta t)^2 \\over m +\\mu \\Delta t} \\nabla_w E(\\mathbf{w})+ {m \\over m +\\mu \\Delta t} \\Delta \\mathbf{w}_t.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "166cfafc", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum parameter\n", + "\n", + "Notice that this equation is identical to previous one if we identify\n", + "the position of the particle, $\\mathbf{w}$, with the parameters\n", + "$\\boldsymbol{\\theta}$. This allows us to identify the momentum\n", + "parameter and learning rate with the mass of the particle and the\n", + "viscous drag as:" + ] + }, + { + "cell_type": "markdown", + "id": "486fadf0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\gamma= {m \\over m +\\mu \\Delta t }, \\qquad \\eta = {(\\Delta t)^2 \\over m +\\mu \\Delta t}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cdb0769d", + "metadata": { + "editable": true + }, + "source": [ + "Thus, as the name suggests, the momentum parameter is proportional to\n", + "the mass of the particle and effectively provides inertia.\n", + "Furthermore, in the large viscosity/small learning rate limit, our\n", + "memory time scales as $(1-\\gamma)^{-1} \\approx m/(\\mu \\Delta t)$.\n", + "\n", + "Why is momentum useful? SGD momentum helps the gradient descent\n", + "algorithm gain speed in directions with persistent but small gradients\n", + "even in the presence of stochasticity, while suppressing oscillations\n", + "in high-curvature directions. This becomes especially important in\n", + "situations where the landscape is shallow and flat in some directions\n", + "and narrow and steep in others. It has been argued that first-order\n", + "methods (with appropriate initial conditions) can perform comparable\n", + "to more expensive second order methods, especially in the context of\n", + "complex deep learning models.\n", + "\n", + "These beneficial properties of momentum can sometimes become even more\n", + "pronounced by using a slight modification of the classical momentum\n", + "algorithm called Nesterov Accelerated Gradient (NAG).\n", + "\n", + "In the NAG algorithm, rather than calculating the gradient at the\n", + "current parameters, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t)$, one\n", + "calculates the gradient at the expected value of the parameters given\n", + "our current momentum, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma\n", + "\\mathbf{v}_{t-1})$. This yields the NAG update rule" + ] + }, + { + "cell_type": "markdown", + "id": "e65569bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma \\mathbf{v}_{t-1}) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d72d316", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t}.\n", + "\\label{_auto2} \\tag{2}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b7b9811e", + "metadata": { + "editable": true + }, + "source": [ + "One of the major advantages of NAG is that it allows for the use of a\n", + "larger learning rate than GDM for the same choice of $\\gamma$." + ] + }, + { + "cell_type": "markdown", + "id": "d8f3c523", + "metadata": { + "editable": true + }, + "source": [ + "## Second moment of the gradient\n", + "\n", + "In stochastic gradient descent, with and without momentum, we still\n", + "have to specify a schedule for tuning the learning rates $\\eta_t$\n", + "as a function of time. As discussed in the context of Newton's\n", + "method, this presents a number of dilemmas. The learning rate is\n", + "limited by the steepest direction which can change depending on the\n", + "current position in the landscape. To circumvent this problem, ideally\n", + "our algorithm would keep track of curvature and take large steps in\n", + "shallow, flat directions and small steps in steep, narrow directions.\n", + "Second-order methods accomplish this by calculating or approximating\n", + "the Hessian and normalizing the learning rate by the\n", + "curvature. However, this is very computationally expensive for\n", + "extremely large models. Ideally, we would like to be able to\n", + "adaptively change the step size to match the landscape without paying\n", + "the steep computational price of calculating or approximating\n", + "Hessians.\n", + "\n", + "During the last decade a number of methods have been introduced that accomplish\n", + "this by tracking not only the gradient, but also the second moment of\n", + "the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and\n", + "[ADAM](https://arxiv.org/abs/1412.6980)." + ] + }, + { + "cell_type": "markdown", + "id": "22d6be74", + "metadata": { + "editable": true + }, + "source": [ + "## Challenge: Choosing a Fixed Learning Rate\n", + "A fixed $\\eta$ is hard to get right:\n", + "1. If $\\eta$ is too large, the updates can overshoot the minimum, causing oscillations or divergence\n", + "\n", + "2. If $\\eta$ is too small, convergence is very slow (many iterations to make progress)\n", + "\n", + "In practice, one often uses trial-and-error or schedules (decaying $\\eta$ over time) to find a workable balance.\n", + "For a function with steep directions and flat directions, a single global $\\eta$ may be inappropriate:\n", + "1. Steep coordinates require a smaller step size to avoid oscillation.\n", + "\n", + "2. Flat/shallow coordinates could use a larger step to speed up progress.\n", + "\n", + "3. This issue is pronounced in high-dimensional problems with **sparse or varying-scale features** – we need a method to adjust step sizesper feature." + ] + }, + { + "cell_type": "markdown", + "id": "091ace5a", + "metadata": { + "editable": true + }, + "source": [ + "## Motivation for Adaptive Step Sizes\n", + "\n", + "1. Instead of a fixed global $\\eta$, use an \\textbf{adaptive learning rate} for each parameter that depends on the history of gradients.\n", + "\n", + "2. Parameters that have large accumulated gradient magnitude should get smaller steps (they've been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.\n", + "\n", + "3. This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected\n", + "\n", + "4. Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates\n", + "\n", + "5. Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive **AdaGrad**, one of the first adaptive methods." + ] + }, + { + "cell_type": "markdown", + "id": "5c05fb8b", + "metadata": { + "editable": true + }, + "source": [ + "## Derivation of the AdaGrad Algorithm\n", + "\n", + "**Accumulating Gradient History.**\n", + "\n", + "1. AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)\n", + "\n", + "2. Let $g_t = \\nabla \\ell_{i_t}(x_t)$ be the gradient at step $t$ (or a subgradient for nondifferentiable cases).\n", + "\n", + "3. Initialize $h_0 = 0$ (an all-zero vector in $\\mathbb{R}^d$).\n", + "\n", + "4. At each iteration $t$, update the accumulation:" + ] + }, + { + "cell_type": "markdown", + "id": "21fb8bd0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_t \\;=\\; h_{t-1} + g_t \\circ g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f71dfafa", + "metadata": { + "editable": true + }, + "source": [ + "1. Here $g_t \\circ g_t$ denotes element-wise square of the gradient vector. $h_t^{(j)} = h_{t-1}^{(j)} + (g_{t,j})^2$ for each parameter $j$.\n", + "\n", + "2. We can view $H_t = \\mathrm{diag}(h_t)$ as a diagonal matrix of past squared gradients. Initially $H_0 = 0$." + ] + }, + { + "cell_type": "markdown", + "id": "64869e2e", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Update Rule Derivation\n", + "\n", + "We scale the gradient by the inverse square root of the accumulated matrix $H_t$. The AdaGrad update at step $t$ is:" + ] + }, + { + "cell_type": "markdown", + "id": "c2d8d6b1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1} \\;=\\; x_t \\;-\\; \\eta\\, H_t^{-1/2}\\, g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2171198a", + "metadata": { + "editable": true + }, + "source": [ + "where $H_t^{-1/2}$ is the diagonal matrix with entries $(h_{t}^{(1)})^{-1/2}, \\dots, (h_{t}^{(d)})^{-1/2}$\n", + "In coordinates, this means each parameter $j$ has an individual step size:" + ] + }, + { + "cell_type": "markdown", + "id": "353e21d3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j} =x_{t,j} -\\frac{\\eta}{\\sqrt{h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b59e6add", + "metadata": { + "editable": true + }, + "source": [ + "In practice we add a small constant $\\epsilon$ in the denominator for numerical stability to avoid division by zero:" + ] + }, + { + "cell_type": "markdown", + "id": "707caefe", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j}= x_{t,j}-\\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "62dafce4", + "metadata": { + "editable": true + }, + "source": [ + "Equivalently, the effective learning rate for parameter $j$ at time $t$ is $\\displaystyle \\alpha_{t,j} = \\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}$. This decreases over time as $h_{t,j}$ grows." + ] + }, + { + "cell_type": "markdown", + "id": "01a2ba61", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Properties\n", + "1. AdaGrad automatically tunes the step size for each parameter. Parameters with more **volatile or large gradients** get smaller steps, and those with **small or infrequent gradients** get relatively larger steps\n", + "\n", + "2. No manual schedule needed: The accumulation $h_t$ keeps increasing (or stays the same if gradient is zero), so step sizes $\\eta/\\sqrt{h_t}$ are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.\n", + "\n", + "3. Sparse data benefit: For very sparse features, $h_{t,j}$ grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal\n", + "\n", + "4. Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate (e.g. $O(1/\\sqrt{T})$ regret bound) comparable to the best fixed learning rate tuned for the problem\n", + "\n", + "It effectively reduces the need to tune $\\eta$ by hand.\n", + "1. Limitations: Because $h_t$ accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)" + ] + }, + { + "cell_type": "markdown", + "id": "b4bc88e0", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp: Adaptive Learning Rates\n", + "Addresses AdaGrad’s diminishing learning rate issue.\n", + "Uses a decaying average of squared gradients (instead of a cumulative sum):" + ] + }, + { + "cell_type": "markdown", + "id": "4933fddb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "v_t = \\beta_2\\, v_{t-1} + (1-\\beta_2)\\, (\\nabla L(w_t))^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0dd8edf7", + "metadata": { + "editable": true + }, + "source": [ + "with $\\beta_2$ typically $0.9$ (or $0.99$).\n", + "1. Update: $w_{t+1} = w_t - \\frac{\\alpha}{\\sqrt{v_t + \\epsilon}} \\nabla L(w_t)$.\n", + "\n", + "2. Recent gradients have more weight, so $v_t$ adapts to the current landscape.\n", + "\n", + "3. Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.\n", + "\n", + "RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 - unpublished.)" + ] + }, + { + "cell_type": "markdown", + "id": "1ce98f28", + "metadata": { + "editable": true + }, + "source": [ + "## Adam Optimizer\n", + "\n", + "Why Combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.\n", + "\n", + "1. Fast convergence by smoothing gradients (accelerates in long-term gradient direction).\n", + "\n", + "2. Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).\n", + "\n", + "3. Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)\n", + "\n", + "4. Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)\n", + "\n", + "**Result**: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice." + ] + }, + { + "cell_type": "markdown", + "id": "d7d646bc", + "metadata": { + "editable": true + }, + "source": [ + "## RMS prop\n", + "\n", + "In RMS prop, in addition to keeping a running average of the first\n", + "moment of the gradient, we also keep track of the second moment\n", + "denoted by $\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}_t^2]$. The update rule\n", + "for RMS prop is given by" + ] + }, + { + "cell_type": "markdown", + "id": "3a212292", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto3} \\tag{3}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "88c0ec96", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta \\mathbf{s}_{t-1} +(1-\\beta)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9cfb794f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\mathbf{g}_t \\over \\sqrt{\\mathbf{s}_t +\\epsilon}}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "aaa4936c", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta$ controls the averaging time of the second moment and is\n", + "typically taken to be about $\\beta=0.9$, $\\eta_t$ is a learning rate\n", + "typically chosen to be $10^{-3}$, and $\\epsilon\\sim 10^{-8} $ is a\n", + "small regularization constant to prevent divergences. Multiplication\n", + "and division by vectors is understood as an element-wise operation. It\n", + "is clear from this formula that the learning rate is reduced in\n", + "directions where the norm of the gradient is consistently large. This\n", + "greatly speeds up the convergence by allowing us to use a larger\n", + "learning rate for flat directions." + ] + }, + { + "cell_type": "markdown", + "id": "5493a6cd", + "metadata": { + "editable": true + }, + "source": [ + "## [ADAM optimizer](https://arxiv.org/abs/1412.6980)\n", + "\n", + "A related algorithm is the ADAM optimizer. In\n", + "[ADAM](https://arxiv.org/abs/1412.6980), we keep a running average of\n", + "both the first and second moment of the gradient and use this\n", + "information to adaptively change the learning rate for different\n", + "parameters. The method isefficient when working with large\n", + "problems involving lots data and/or parameters. It is a combination of the\n", + "gradient descent with momentum algorithm and the RMSprop algorithm\n", + "discussed above.\n", + "\n", + "In addition to keeping a running average of the first and\n", + "second moments of the gradient\n", + "(i.e. $\\mathbf{m}_t=\\mathbb{E}[\\mathbf{g}_t]$ and\n", + "$\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}^2_t]$, respectively), ADAM\n", + "performs an additional bias correction to account for the fact that we\n", + "are estimating the first two moments of the gradient using a running\n", + "average (denoted by the hats in the update rule below). The update\n", + "rule for ADAM is given by (where multiplication and division are once\n", + "again understood to be element-wise operations below)" + ] + }, + { + "cell_type": "markdown", + "id": "35795070", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto4} \\tag{4}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1e67e6dd", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{m}_t = \\beta_1 \\mathbf{m}_{t-1} + (1-\\beta_1) \\mathbf{g}_t \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39d582bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta_2 \\mathbf{s}_{t-1} +(1-\\beta_2)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a7e2423", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{m}}_t={\\mathbf{m}_t \\over 1-\\beta_1^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "53da77c2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{s}}_t ={\\mathbf{s}_t \\over1-\\beta_2^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a4c7df45", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\boldsymbol{\\mathbf{m}}_t \\over \\sqrt{\\boldsymbol{\\mathbf{s}}_t} +\\epsilon}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f645a0aa", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\label{_auto5} \\tag{5}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "505619bb", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta_1$ and $\\beta_2$ set the memory lifetime of the first and\n", + "second moment and are typically taken to be $0.9$ and $0.99$\n", + "respectively, and $\\eta$ and $\\epsilon$ are identical to RMSprop.\n", + "\n", + "Like in RMSprop, the effective step size of a parameter depends on the\n", + "magnitude of its gradient squared. To understand this better, let us\n", + "rewrite this expression in terms of the variance\n", + "$\\boldsymbol{\\sigma}_t^2 = \\boldsymbol{\\mathbf{s}}_t -\n", + "(\\boldsymbol{\\mathbf{m}}_t)^2$. Consider a single parameter $\\theta_t$. The\n", + "update rule for this parameter is given by" + ] + }, + { + "cell_type": "markdown", + "id": "6cb4c9a9", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\theta_{t+1}= -\\eta_t { \\boldsymbol{m}_t \\over \\sqrt{\\sigma_t^2 + m_t^2 }+\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39627950", + "metadata": { + "editable": true + }, + "source": [ + "## Algorithms and codes for Adagrad, RMSprop and Adam\n", + "\n", + "The algorithms we have implemented are well described in the text by [Goodfellow, Bengio and Courville, chapter 8](https://www.deeplearningbook.org/contents/optimization.html).\n", + "\n", + "The codes which implement these algorithms are discussed after our presentation of automatic differentiation." + ] + }, + { + "cell_type": "markdown", + "id": "f240057d", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "428ee09d", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "9ce3cc3e", + "metadata": { + "editable": true + }, + "source": [ + "## ADAM algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "193b3778", + "metadata": { + "editable": true + }, + "source": [ + "## Practical tips\n", + "\n", + "* **Randomize the data when making mini-batches**. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.\n", + "\n", + "* **Transform your inputs**. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.\n", + "\n", + "* **Monitor the out-of-sample performance.** Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This *early stopping* significantly improves performance in many settings.\n", + "\n", + "* **Adaptive optimization methods don't always have good generalization.** Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications." + ] + }, + { + "cell_type": "markdown", + "id": "8d7187f5", + "metadata": { + "editable": true + }, + "source": [ + "## Sneaking in automatic differentiation using Autograd\n", + "\n", + "In the examples here we take the liberty of sneaking in automatic\n", + "differentiation (without having discussed the mathematics). In\n", + "project 1 you will write the gradients as discussed above, that is\n", + "hard-coding the gradients. By introducing automatic differentiation\n", + "via the library **autograd**, which is now replaced by **JAX**, we have\n", + "more flexibility in setting up alternative cost functions.\n", + "\n", + "The\n", + "first example shows results with ordinary leats squares." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "68d35b76", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f39b471c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "265059ef", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x#+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 30\n", + "\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "# Now improve with momentum gradient descent\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "for iter in range(Niterations):\n", + " # calculate gradient\n", + " gradients = training_gradient(theta)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd wth momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "18183fb0", + "metadata": { + "editable": true + }, + "source": [ + "## Including Stochastic Gradient Descent with Autograd\n", + "\n", + "In this code we include the stochastic gradient descent approach\n", + "discussed above. Note here that we specify which argument we are\n", + "taking the derivative with respect to when using **autograd**." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4d9d93b3", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "1652a471", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6092712c", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "\n", + "for epoch in range(n_epochs):\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + "print(\"theta from own sdg with momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "4d2f230d", + "metadata": { + "editable": true + }, + "source": [ + "## But none of these can compete with Newton's method\n", + "\n", + "Note that we here have introduced automatic differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b46cedcc", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Newton's method\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+5*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(beta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "# Note that here the Hessian does not depend on the parameters beta\n", + "invH = np.linalg.pinv(H)\n", + "beta = np.random.randn(3,1)\n", + "Niterations = 5\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(beta)\n", + " beta -= invH @ gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"beta from own Newton code\")\n", + "print(beta)" + ] + }, + { + "cell_type": "markdown", + "id": "f048d5c7", + "metadata": { + "editable": true + }, + "source": [ + "## Similar (second order function now) problem but now with AdaGrad" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d8bdd180", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " Giter += gradients*gradients\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + " theta -= update\n", + "print(\"theta from own AdaGrad\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "145ce394", + "metadata": { + "editable": true + }, + "source": [ + "Running this code we note an almost perfect agreement with the results from matrix inversion." + ] + }, + { + "cell_type": "markdown", + "id": "ec0ee110", + "metadata": { + "editable": true + }, + "source": [ + "## RMSprop for adaptive learning rate with Stochastic Gradient Descent" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b6e8e3e4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameter rho\n", + "rho = 0.99\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + "\t# Accumulated gradient\n", + "\t# Scaling with rho the new and the previous results\n", + " Giter = (rho*Giter+(1-rho)*gradients*gradients)\n", + "\t# Taking the diagonal only and inverting\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + "\t# Hadamard product\n", + " theta -= update\n", + "print(\"theta from own RMSprop\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "79d8c0ae", + "metadata": { + "editable": true + }, + "source": [ + "## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c16ec5ac", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980\n", + "beta1 = 0.9\n", + "beta2 = 0.999\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-7\n", + "iter = 0\n", + "for epoch in range(n_epochs):\n", + " first_moment = 0.0\n", + " second_moment = 0.0\n", + " iter += 1\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " # Computing moments first\n", + " first_moment = beta1*first_moment + (1-beta1)*gradients\n", + " second_moment = beta2*second_moment+(1-beta2)*gradients*gradients\n", + " first_term = first_moment/(1.0-beta1**iter)\n", + " second_term = second_moment/(1.0-beta2**iter)\n", + "\t# Scaling with rho the new and the previous results\n", + " update = eta*first_term/(np.sqrt(second_term)+delta)\n", + " theta -= update\n", + "print(\"theta from own ADAM\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "7c581f8b", + "metadata": { + "editable": true + }, + "source": [ + "## Material for the lab sessions\n", + "\n", + "1. Exercise set for week 37 and reminder on scaling (from lab sessions of week 35)\n", + "\n", + "2. Work on project 1\n", + "\n", + "\n", + " * For more discussions of Ridge regression and calculation of averages, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended." + ] + }, + { + "cell_type": "markdown", + "id": "d8a82e01", + "metadata": { + "editable": true + }, + "source": [ + "## Reminder on different scaling methods\n", + "\n", + "Before fitting a regression model, it is good practice to normalize or\n", + "standardize the features. This ensures all features are on a\n", + "comparable scale, which is especially important when using\n", + "regularization. In the exercises this week we will perform standardization, scaling each\n", + "feature to have mean 0 and standard deviation 1.\n", + "\n", + "Here we compute the mean and standard deviation of each column (feature) in our design/feature matrix $\\boldsymbol{X}$.\n", + "Then we subtract the mean and divide by the standard deviation for each feature.\n", + "\n", + "In the example here we\n", + "we will also center the target $\\boldsymbol{y}$ to mean $0$. Centering $\\boldsymbol{y}$\n", + "(and each feature) means the model does not require a separate intercept\n", + "term, the data is shifted such that the intercept is effectively 0\n", + ". (In practice, one could include an intercept in the model and not\n", + "penalize it, but here we simplify by centering.)\n", + "Choose $n=100$ data points and set up $\\boldsymbol{x}, $\\boldsymbol{y}$ and the design matrix $\\boldsymbol{X}$." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "3fa174a9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Standardize features (zero mean, unit variance for each feature)\n", + "X_mean = X.mean(axis=0)\n", + "X_std = X.std(axis=0)\n", + "X_std[X_std == 0] = 1 # safeguard to avoid division by zero for constant features\n", + "X_norm = (X - X_mean) / X_std\n", + "\n", + "# Center the target to zero mean (optional, to simplify intercept handling)\n", + "y_mean = ?\n", + "y_centered = ?" + ] + }, + { + "cell_type": "markdown", + "id": "b02de7a6", + "metadata": { + "editable": true + }, + "source": [ + "Do we need to center the values of $y$?\n", + "\n", + "After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n", + "and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This can make the optimization landscape\n", + "nicer and ensures the regularization penalty $\\lambda \\sum_j\n", + "\\theta_j^2$ in Ridge regression treats each coefficient fairly (since features are on the\n", + "same scale)." + ] + }, + { + "cell_type": "markdown", + "id": "cd3f85ef", + "metadata": { + "editable": true + }, + "source": [ + "## Functionality in Scikit-Learn\n", + "\n", + "**Scikit-Learn** has several functions which allow us to rescale the\n", + "data, normally resulting in much better results in terms of various\n", + "accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n", + "ensures that for each feature/predictor we study the mean value is\n", + "zero and the variance is one (every column in the design/feature\n", + "matrix). This scaling has the drawback that it does not ensure that\n", + "we have a particular maximum or minimum in our data set. Another\n", + "function included in **Scikit-Learn** is the **MinMaxScaler** which\n", + "ensures that all features are exactly between $0$ and $1$. The" + ] + }, + { + "cell_type": "markdown", + "id": "1e0198a0", + "metadata": { + "editable": true + }, + "source": [ + "## More preprocessing\n", + "\n", + "The **Normalizer** scales each data\n", + "point such that the feature vector has a euclidean length of one. In other words, it\n", + "projects a data point on the circle (or sphere in the case of higher dimensions) with a\n", + "radius of 1. This means every data point is scaled by a different number (by the\n", + "inverse of it’s length).\n", + "This normalization is often used when only the direction (or angle) of the data matters,\n", + "not the length of the feature vector.\n", + "\n", + "The **RobustScaler** works similarly to the StandardScaler in that it\n", + "ensures statistical properties for each feature that guarantee that\n", + "they are on the same scale. However, the RobustScaler uses the median\n", + "and quartiles, instead of mean and variance. This makes the\n", + "RobustScaler ignore data points that are very different from the rest\n", + "(like measurement errors). These odd data points are also called\n", + "outliers, and might often lead to trouble for other scaling\n", + "techniques." + ] + }, + { + "cell_type": "markdown", + "id": "04da94ca", + "metadata": { + "editable": true + }, + "source": [ + "## Frequently used scaling functions\n", + "\n", + "Many features are often scaled using standardization to improve performance. In **Scikit-Learn** this is given by the **StandardScaler** function as discussed above. It is easy however to write your own. \n", + "Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:" + ] + }, + { + "cell_type": "markdown", + "id": "aa1ad257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6587016d", + "metadata": { + "editable": true + }, + "source": [ + "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard deviation, respectively, of the feature $x_j$.\n", + "This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2e0350b4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "8179b037", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "3ebed257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta_0, \\theta_1, ... , \\theta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij}\\theta_j\\right)^2,.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "51b239a4", + "metadata": { + "editable": true + }, + "source": [ + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", + "\n", + "What we have done is to single out the $\\theta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\theta_0$, we want the derivative to obey" + ] + }, + { + "cell_type": "markdown", + "id": "295790f3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_j} = 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b68338d6", + "metadata": { + "editable": true + }, + "source": [ + "for all $j$. For $\\theta_0$ we have" + ] + }, + { + "cell_type": "markdown", + "id": "c015cb3b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij} \\theta_j\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5e4917fa", + "metadata": { + "editable": true + }, + "source": [ + "Multiplying away the constant $2/n$, we obtain" + ] + }, + { + "cell_type": "markdown", + "id": "150fedf5", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\sum_{i=0}^{n-1} \\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6d782b1", + "metadata": { + "editable": true + }, + "source": [ + "Let us specialize first to the case where we have only two parameters $\\theta_0$ and $\\theta_1$.\n", + "Our result for $\\theta_0$ simplifies then to" + ] + }, + { + "cell_type": "markdown", + "id": "a4065dd3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "n\\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\theta_1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "06324d79", + "metadata": { + "editable": true + }, + "source": [ + "We obtain then" + ] + }, + { + "cell_type": "markdown", + "id": "9e3e6b35", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\theta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cd9c3754", + "metadata": { + "editable": true + }, + "source": [ + "If we define" + ] + }, + { + "cell_type": "markdown", + "id": "8361829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8f5f46fd", + "metadata": { + "editable": true + }, + "source": [ + "and the mean value of the outputs as" + ] + }, + { + "cell_type": "markdown", + "id": "cf210a6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d3af09fc", + "metadata": { + "editable": true + }, + "source": [ + "we have" + ] + }, + { + "cell_type": "markdown", + "id": "ec91f8c1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\mu_y - \\theta_1\\mu_{\\boldsymbol{x}_1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0c9f35e6", + "metadata": { + "editable": true + }, + "source": [ + "In the general case with more parameters than $\\theta_0$ and $\\theta_1$, we have" + ] + }, + { + "cell_type": "markdown", + "id": "de80fa4b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "34c99b80", + "metadata": { + "editable": true + }, + "source": [ + "We can rewrite the latter equation as" + ] + }, + { + "cell_type": "markdown", + "id": "758ade4d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\theta_j,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff0f4725", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined" + ] + }, + { + "cell_type": "markdown", + "id": "997b829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "621e7d85", + "metadata": { + "editable": true + }, + "source": [ + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", + "\n", + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" + ] + }, + { + "cell_type": "markdown", + "id": "5530e459", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\theta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "fd9bace8", + "metadata": { + "editable": true + }, + "source": [ + "If we minimize with respect to $\\boldsymbol{\\theta}$ we have then" + ] + }, + { + "cell_type": "markdown", + "id": "d949be75", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ad52617", + "metadata": { + "editable": true + }, + "source": [ + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\theta}^T\\boldsymbol{\\theta}$ to the cost function and get then" + ] + }, + { + "cell_type": "markdown", + "id": "41cabc72", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1fd416a6", + "metadata": { + "editable": true + }, + "source": [ + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f13cacb4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_theta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_theta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_theta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "theta = fit_theta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True theta: {true_theta}\")\n", + "print(f\"Fitted theta: {theta}\")\n", + "print(f\"Sklearn fitted theta: {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ theta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "theta = fit_theta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ theta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted theta (without intercept): {theta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted theta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ theta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ff4b28ed", + "metadata": { + "editable": true + }, + "source": [ + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", + "\n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\theta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", + "\n", + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" + ] + }, + { + "cell_type": "markdown", + "id": "03b4dc3c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\theta_j^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6cff2148", + "metadata": { + "editable": true + }, + "source": [ + "but when we take out the intercept, this equation becomes" + ] + }, + { + "cell_type": "markdown", + "id": "51d823c4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\theta_j^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5ab270f9", + "metadata": { + "editable": true + }, + "source": [ + "For Lasso regression we have" + ] + }, + { + "cell_type": "markdown", + "id": "2b2e1492", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\theta_j\\vert.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "063be947", + "metadata": { + "editable": true + }, + "source": [ + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", + "\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "5e841a10", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeTheta\n", + " ypredictOwnRidge = X_test @ OwnRidgeTheta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta)\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "40e54083", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "edd7dd42", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", + "\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", + "\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeTheta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeTheta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta) #Intercept is given by mean of target variable\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f9cc046b", + "metadata": { + "editable": true + }, + "source": [ + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\theta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/doc/LectureNotes/week37.ipynb b/doc/LectureNotes/week37.ipynb new file mode 100644 index 000000000..acd6b4e52 --- /dev/null +++ b/doc/LectureNotes/week37.ipynb @@ -0,0 +1,3736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c543a71a", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "4ff6b050", + "metadata": { + "editable": true + }, + "source": [ + "# Week 37: Statistical interpretations and Resampling Methods\n", + "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo, Norway\n", + "\n", + "Date: **September 8-12, 2025**\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "a7079d3e", + "metadata": { + "editable": true + }, + "source": [ + "## Plans for week 37, lecture Monday\n", + "\n", + "**Plans and material for the lecture on Monday September 8.**\n", + "\n", + "The family of gradient descent methods\n", + "1. Plain gradient descent (constant learning rate), reminder from last week with examples using OLS and Ridge\n", + "\n", + "2. Improving gradient descent with momentum\n", + "\n", + "3. Introducing stochastic gradient descent\n", + "\n", + "4. More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "db3af8cc", + "metadata": { + "editable": true + }, + "source": [ + "## Readings and Videos:\n", + "1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at and chapter 8.3-8.5 at \n", + "\n", + "2. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.\n", + "\n", + "3. Video on gradient descent at \n", + "\n", + "4. Video on Stochastic gradient descent at " + ] + }, + { + "cell_type": "markdown", + "id": "79d6d5be", + "metadata": { + "editable": true + }, + "source": [ + "## Material for lecture Monday September 8" + ] + }, + { + "cell_type": "markdown", + "id": "fa427fc3", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and revisiting Ordinary Least Squares from last week\n", + "\n", + "Last week we started with linear regression as a case study for the gradient descent\n", + "methods. Linear regression is a great test case for the gradient\n", + "descent methods discussed in the lectures since it has several\n", + "desirable properties such as:\n", + "\n", + "1. An analytical solution (recall homework sets for week 35).\n", + "\n", + "2. The gradient can be computed analytically.\n", + "\n", + "3. The cost function is convex which guarantees that gradient descent converges for small enough learning rates\n", + "\n", + "We revisit an example similar to what we had in the first homework set. We have a function of the type" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1185629e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "x = 2*np.random.rand(m,1)\n", + "y = 4+3*x+np.random.randn(m,1)" + ] + }, + { + "cell_type": "markdown", + "id": "42dfe293", + "metadata": { + "editable": true + }, + "source": [ + "with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n", + "The linear regression model is given by" + ] + }, + { + "cell_type": "markdown", + "id": "a726cc6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_\\theta(x) = \\boldsymbol{y} = \\theta_0 + \\theta_1 x,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "991c8919", + "metadata": { + "editable": true + }, + "source": [ + "such that" + ] + }, + { + "cell_type": "markdown", + "id": "ba895122", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y}_i = \\theta_0 + \\theta_1 x_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5c2b9317", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent example\n", + "\n", + "Let $\\mathbf{y} = (y_1,\\cdots,y_n)^T$, $\\mathbf{\\boldsymbol{y}} = (\\boldsymbol{y}_1,\\cdots,\\boldsymbol{y}_n)^T$ and $\\theta = (\\theta_0, \\theta_1)^T$\n", + "\n", + "It is convenient to write $\\mathbf{\\boldsymbol{y}} = X\\theta$ where $X \\in \\mathbb{R}^{100 \\times 2} $ is the design matrix given by (we keep the intercept here)" + ] + }, + { + "cell_type": "markdown", + "id": "264eef67", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "X \\equiv \\begin{bmatrix}\n", + "1 & x_1 \\\\\n", + "\\vdots & \\vdots \\\\\n", + "1 & x_{100} & \\\\\n", + "\\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a451a9ce", + "metadata": { + "editable": true + }, + "source": [ + "The cost/loss/risk function is given by (" + ] + }, + { + "cell_type": "markdown", + "id": "e7fa569b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta) = \\frac{1}{n}||X\\theta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\theta_0 + \\theta_1 x_i)^2 - 2 y_i (\\theta_0 + \\theta_1 x_i) + y_i^2\\right]\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a2655bc6", + "metadata": { + "editable": true + }, + "source": [ + "and we want to find $\\theta$ such that $C(\\theta)$ is minimized." + ] + }, + { + "cell_type": "markdown", + "id": "277ace2e", + "metadata": { + "editable": true + }, + "source": [ + "## The derivative of the cost/loss function\n", + "\n", + "Computing $\\partial C(\\theta) / \\partial \\theta_0$ and $\\partial C(\\theta) / \\partial \\theta_1$ we can show that the gradient can be written as" + ] + }, + { + "cell_type": "markdown", + "id": "75bbf03c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\theta} C(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T(X\\theta - \\mathbf{y}),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5bb2db2", + "metadata": { + "editable": true + }, + "source": [ + "where $X$ is the design matrix defined above." + ] + }, + { + "cell_type": "markdown", + "id": "53430d47", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix\n", + "The Hessian matrix of $C(\\theta)$ is given by" + ] + }, + { + "cell_type": "markdown", + "id": "89dce328", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "542b8278", + "metadata": { + "editable": true + }, + "source": [ + "This result implies that $C(\\theta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite." + ] + }, + { + "cell_type": "markdown", + "id": "2b94baca", + "metadata": { + "editable": true + }, + "source": [ + "## Simple program\n", + "\n", + "We can now write a program that minimizes $C(\\theta)$ using the gradient descent method with a constant learning rate $\\gamma$ according to" + ] + }, + { + "cell_type": "markdown", + "id": "bfc6b93f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{k+1} = \\theta_k - \\gamma \\nabla_\\theta C(\\theta_k), \\ k=0,1,\\cdots\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "703de92e", + "metadata": { + "editable": true + }, + "source": [ + "We can use the expression we computed for the gradient and let use a\n", + "$\\theta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n", + "when $||\\nabla_\\theta C(\\theta_k) || \\leq \\epsilon = 10^{-8}$. **Note that the code below does not include the latter stop criterion**.\n", + "\n", + "And finally we can compare our solution for $\\theta$ with the analytic result given by \n", + "$\\theta= (X^TX)^{-1} X^T \\mathbf{y}$." + ] + }, + { + "cell_type": "markdown", + "id": "f843aae1", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient Descent Example\n", + "\n", + "Here our simple example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b76443c9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "\n", + "# Importing various packages\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "# Hessian matrix\n", + "H = (2.0/n)* X.T @ X\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y\n", + "print(theta_linreg)\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "for iter in range(Niterations):\n", + " gradient = (2.0/n)*X.T @ (X @ theta-y)\n", + " theta -= eta*gradient\n", + "\n", + "print(theta)\n", + "xnew = np.array([[0],[2]])\n", + "xbnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = xbnew.dot(theta)\n", + "ypredict2 = xbnew.dot(theta_linreg)\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7b75f4ec", + "metadata": { + "editable": true + }, + "source": [ + "## Gradient descent and Ridge\n", + "\n", + "We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\\theta$," + ] + }, + { + "cell_type": "markdown", + "id": "47ecdee7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C_{\\text{ridge}}(\\theta) = \\frac{1}{n}||X\\theta -\\mathbf{y}||^2 + \\lambda ||\\theta||^2, \\ \\lambda \\geq 0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff25299b", + "metadata": { + "editable": true + }, + "source": [ + "In order to minimize $C_{\\text{ridge}}(\\theta)$ using GD we adjust the gradient as follows" + ] + }, + { + "cell_type": "markdown", + "id": "21a88a84", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\theta C_{\\text{ridge}}(\\theta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\theta_0+\\theta_1x_i-y_i\\right) \\\\\n", + "\\sum_{i=1}^{100}\\left( x_i (\\theta_0+\\theta_1x_i)-y_ix_i\\right) \\\\\n", + "\\end{bmatrix} + 2\\lambda\\begin{bmatrix} \\theta_0 \\\\ \\theta_1\\end{bmatrix} = 2 (\\frac{1}{n}X^T(X\\theta - \\mathbf{y})+\\lambda \\theta).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67491c15", + "metadata": { + "editable": true + }, + "source": [ + "We can easily extend our program to minimize $C_{\\text{ridge}}(\\theta)$ using gradient descent and compare with the analytical solution given by" + ] + }, + { + "cell_type": "markdown", + "id": "f1ac13f1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67d83802", + "metadata": { + "editable": true + }, + "source": [ + "## The Hessian matrix for Ridge Regression\n", + "The Hessian matrix of Ridge Regression for our simple example is given by" + ] + }, + { + "cell_type": "markdown", + "id": "75935d70", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{H} \\equiv \\begin{bmatrix}\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0^2} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} \\\\\n", + "\\frac{\\partial^2 C(\\theta)}{\\partial \\theta_0 \\partial \\theta_1} & \\frac{\\partial^2 C(\\theta)}{\\partial \\theta_1^2} & \\\\\n", + "\\end{bmatrix} = \\frac{2}{n}X^T X+2\\lambda\\boldsymbol{I}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4707a65d", + "metadata": { + "editable": true + }, + "source": [ + "This implies that the Hessian matrix is positive definite, hence the stationary point is a\n", + "minimum.\n", + "Note that the Ridge cost function is convex being a sum of two convex\n", + "functions. Therefore, the stationary point is a global\n", + "minimum of this function." + ] + }, + { + "cell_type": "markdown", + "id": "ab16b163", + "metadata": { + "editable": true + }, + "source": [ + "## Program example for gradient descent with Ridge Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fe7ee1f7", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import cm\n", + "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n", + "import sys\n", + "\n", + "# the number of datapoints\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "\n", + "#Ridge parameter lambda\n", + "lmbda = 0.001\n", + "Id = n*lmbda* np.eye(XT_X.shape[0])\n", + "\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X+2*lmbda* np.eye(XT_X.shape[0])\n", + "# Get the eigenvalues\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "\n", + "theta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y\n", + "print(theta_linreg)\n", + "# Start plain gradient descent\n", + "theta = np.random.randn(2,1)\n", + "\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ (X @ (theta)-y)+2*lmbda*theta\n", + " theta -= eta*gradients\n", + "\n", + "print(theta)\n", + "ypredict = X @ theta\n", + "ypredict2 = X @ theta_linreg\n", + "plt.plot(x, ypredict, \"r-\")\n", + "plt.plot(x, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Gradient descent example for Ridge')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "823fdaf7", + "metadata": { + "editable": true + }, + "source": [ + "## Using gradient descent methods, limitations\n", + "\n", + "* **Gradient descent (GD) finds local minima of our function**. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.\n", + "\n", + "* **GD is sensitive to initial conditions**. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.\n", + "\n", + "* **Gradients are computationally expensive to calculate for large datasets**. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, $E \\propto \\sum_{i=1}^n (y_i - \\mathbf{w}^T\\cdot\\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called \"mini batches\". This has the added benefit of introducing stochasticity into our algorithm.\n", + "\n", + "* **GD is very sensitive to choices of learning rates**. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.\n", + "\n", + "* **GD treats all directions in parameter space uniformly.** Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. \n", + "\n", + "* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points." + ] + }, + { + "cell_type": "markdown", + "id": "21ea3f2a", + "metadata": { + "editable": true + }, + "source": [ + "## Improving gradient descent with momentum\n", + "\n", + "We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5b729a7e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# take a step\n", + "\t\tsolution = solution - step_size * gradient\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# perform the gradient descent search\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "209fc69c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "984c4202", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import asarray\n", + "from numpy import arange\n", + "from numpy.random import rand\n", + "from numpy.random import seed\n", + "from matplotlib import pyplot\n", + " \n", + "# objective function\n", + "def objective(x):\n", + "\treturn x**2.0\n", + " \n", + "# derivative of objective function\n", + "def derivative(x):\n", + "\treturn x * 2.0\n", + " \n", + "# gradient descent algorithm\n", + "def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):\n", + "\t# track all solutions\n", + "\tsolutions, scores = list(), list()\n", + "\t# generate an initial point\n", + "\tsolution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])\n", + "\t# keep track of the change\n", + "\tchange = 0.0\n", + "\t# run the gradient descent\n", + "\tfor i in range(n_iter):\n", + "\t\t# calculate gradient\n", + "\t\tgradient = derivative(solution)\n", + "\t\t# calculate update\n", + "\t\tnew_change = step_size * gradient + momentum * change\n", + "\t\t# take a step\n", + "\t\tsolution = solution - new_change\n", + "\t\t# save the change\n", + "\t\tchange = new_change\n", + "\t\t# evaluate candidate point\n", + "\t\tsolution_eval = objective(solution)\n", + "\t\t# store solution\n", + "\t\tsolutions.append(solution)\n", + "\t\tscores.append(solution_eval)\n", + "\t\t# report progress\n", + "\t\tprint('>%d f(%s) = %.5f' % (i, solution, solution_eval))\n", + "\treturn [solutions, scores]\n", + " \n", + "# seed the pseudo random number generator\n", + "seed(4)\n", + "# define range for input\n", + "bounds = asarray([[-1.0, 1.0]])\n", + "# define the total iterations\n", + "n_iter = 30\n", + "# define the step size\n", + "step_size = 0.1\n", + "# define momentum\n", + "momentum = 0.3\n", + "# perform the gradient descent search with momentum\n", + "solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)\n", + "# sample input range uniformly at 0.1 increments\n", + "inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)\n", + "# compute targets\n", + "results = objective(inputs)\n", + "# create a line plot of input vs result\n", + "pyplot.plot(inputs, results)\n", + "# plot the solutions found\n", + "pyplot.plot(solutions, scores, '.-', color='red')\n", + "# show the plot\n", + "pyplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "8577b238", + "metadata": { + "editable": true + }, + "source": [ + "## Overview video on Stochastic Gradient Descent (SGD)\n", + "\n", + "[What is Stochastic Gradient Descent](https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer)\n", + "There are several reasons for using stochastic gradient descent. Some of these are:\n", + "\n", + "1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.\n", + "\n", + "2. Hopefully avoid Local Minima\n", + "\n", + "3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c11d0874", + "metadata": { + "editable": true + }, + "source": [ + "## Batches and mini-batches\n", + "\n", + "In gradient descent we compute the cost function and its gradient for all data points we have.\n", + "\n", + "In large-scale applications such as the [ILSVRC challenge](https://www.image-net.org/challenges/LSVRC/), the\n", + "training data can have on order of millions of examples. Hence, it\n", + "seems wasteful to compute the full cost function over the entire\n", + "training set in order to perform only a single parameter update. A\n", + "very common approach to addressing this challenge is to compute the\n", + "gradient over batches of the training data. For example, a typical batch could contain some thousand examples from\n", + "an entire training set of several millions. This batch is then used to\n", + "perform a parameter update." + ] + }, + { + "cell_type": "markdown", + "id": "64ad52d0", + "metadata": { + "editable": true + }, + "source": [ + "## Pros and cons\n", + "\n", + "1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.\n", + "\n", + "2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.\n", + "\n", + "3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient." + ] + }, + { + "cell_type": "markdown", + "id": "5dd4c467", + "metadata": { + "editable": true + }, + "source": [ + "## Convergence rates\n", + "\n", + "1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.\n", + "\n", + "2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration." + ] + }, + { + "cell_type": "markdown", + "id": "42904823", + "metadata": { + "editable": true + }, + "source": [ + "## Accuracy\n", + "\n", + "In general, stochastic Gradient Descent is Less accurate than gradient\n", + "descent, as it calculates the gradient on single examples, which may\n", + "not accurately represent the overall dataset. Gradient Descent is\n", + "more accurate because it uses the average gradient calculated over the\n", + "entire dataset.\n", + "\n", + "There are other disadvantages to using SGD. The main drawback is that\n", + "its convergence behaviour can be more erratic due to the random\n", + "sampling of individual training examples. This can lead to less\n", + "accurate results, as the algorithm may not converge to the true\n", + "minimum of the cost function. Additionally, the learning rate, which\n", + "determines the step size of each update to the model’s parameters,\n", + "must be carefully chosen to ensure convergence.\n", + "\n", + "It is however the method of choice in deep learning algorithms where\n", + "SGD is often used in combination with other optimization techniques,\n", + "such as momentum or adaptive learning rates" + ] + }, + { + "cell_type": "markdown", + "id": "49dfaf2e", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent (SGD)\n", + "\n", + "In stochastic gradient descent, the extreme case is the case where we\n", + "have only one batch, that is we include the whole data set.\n", + "\n", + "This process is called Stochastic Gradient\n", + "Descent (SGD) (or also sometimes on-line gradient descent). This is\n", + "relatively less common to see because in practice due to vectorized\n", + "code optimizations it can be computationally much more efficient to\n", + "evaluate the gradient for 100 examples, than the gradient for one\n", + "example 100 times. Even though SGD technically refers to using a\n", + "single example at a time to evaluate the gradient, you will hear\n", + "people use the term SGD even when referring to mini-batch gradient\n", + "descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD\n", + "for “Batch gradient descent” are rare to see), where it is usually\n", + "assumed that mini-batches are used. The size of the mini-batch is a\n", + "hyperparameter but it is not very common to cross-validate or bootstrap it. It is\n", + "usually based on memory constraints (if any), or set to some value,\n", + "e.g. 32, 64 or 128. We use powers of 2 in practice because many\n", + "vectorized operation implementations work faster when their inputs are\n", + "sized in powers of 2.\n", + "\n", + "In our notes with SGD we mean stochastic gradient descent with mini-batches." + ] + }, + { + "cell_type": "markdown", + "id": "284b8663", + "metadata": { + "editable": true + }, + "source": [ + "## Stochastic Gradient Descent\n", + "\n", + "Stochastic gradient descent (SGD) and variants thereof address some of\n", + "the shortcomings of the Gradient descent method discussed above.\n", + "\n", + "The underlying idea of SGD comes from the observation that the cost\n", + "function, which we want to minimize, can almost always be written as a\n", + "sum over $n$ data points $\\{\\mathbf{x}_i\\}_{i=1}^n$," + ] + }, + { + "cell_type": "markdown", + "id": "5e8cc1b8", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "510505a0", + "metadata": { + "editable": true + }, + "source": [ + "## Computation of gradients\n", + "\n", + "This in turn means that the gradient can be\n", + "computed as a sum over $i$-gradients" + ] + }, + { + "cell_type": "markdown", + "id": "32fc501f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_\\beta C(\\mathbf{\\beta}) = \\sum_i^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9dcac7ef", + "metadata": { + "editable": true + }, + "source": [ + "Stochasticity/randomness is introduced by only taking the\n", + "gradient on a subset of the data called minibatches. If there are $n$\n", + "data points and the size of each minibatch is $M$, there will be $n/M$\n", + "minibatches. We denote these minibatches by $B_k$ where\n", + "$k=1,\\cdots,n/M$." + ] + }, + { + "cell_type": "markdown", + "id": "c9464120", + "metadata": { + "editable": true + }, + "source": [ + "## SGD example\n", + "As an example, suppose we have $10$ data points $(\\mathbf{x}_1,\\cdots, \\mathbf{x}_{10})$ \n", + "and we choose to have $M=5$ minibathces,\n", + "then each minibatch contains two data points. In particular we have\n", + "$B_1 = (\\mathbf{x}_1,\\mathbf{x}_2), \\cdots, B_5 =\n", + "(\\mathbf{x}_9,\\mathbf{x}_{10})$. Note that if you choose $M=1$ you\n", + "have only a single batch with all data points and on the other extreme,\n", + "you may choose $M=n$ resulting in a minibatch for each datapoint, i.e\n", + "$B_k = \\mathbf{x}_k$.\n", + "\n", + "The idea is now to approximate the gradient by replacing the sum over\n", + "all data points with a sum over the data points in one the minibatches\n", + "picked at random in each gradient descent step" + ] + }, + { + "cell_type": "markdown", + "id": "59bc847c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\nabla_{\\beta}\n", + "C(\\mathbf{\\beta}) = \\sum_{i=1}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta}) \\rightarrow \\sum_{i \\in B_k}^n \\nabla_\\beta\n", + "c_i(\\mathbf{x}_i, \\mathbf{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ba220299", + "metadata": { + "editable": true + }, + "source": [ + "## The gradient step\n", + "\n", + "Thus a gradient descent step now looks like" + ] + }, + { + "cell_type": "markdown", + "id": "8608ba34", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\beta_{j+1} = \\beta_j - \\gamma_j \\sum_{i \\in B_k}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", + "\\mathbf{\\beta})\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0faa1782", + "metadata": { + "editable": true + }, + "source": [ + "where $k$ is picked at random with equal\n", + "probability from $[1,n/M]$. An iteration over the number of\n", + "minibathces (n/M) is commonly referred to as an epoch. Thus it is\n", + "typical to choose a number of epochs and for each epoch iterate over\n", + "the number of minibatches, as exemplified in the code below." + ] + }, + { + "cell_type": "markdown", + "id": "97eddb68", + "metadata": { + "editable": true + }, + "source": [ + "## Simple example code" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "309da17e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 10 #number of epochs\n", + "\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for \n", + " j += 1" + ] + }, + { + "cell_type": "markdown", + "id": "7aea4344", + "metadata": { + "editable": true + }, + "source": [ + "Taking the gradient only on a subset of the data has two important\n", + "benefits. First, it introduces randomness which decreases the chance\n", + "that our opmization scheme gets stuck in a local minima. Second, if\n", + "the size of the minibatches are small relative to the number of\n", + "datapoints ($M < n$), the computation of the gradient is much\n", + "cheaper since we sum over the datapoints in the $k-th$ minibatch and not\n", + "all $n$ datapoints." + ] + }, + { + "cell_type": "markdown", + "id": "52f8d549", + "metadata": { + "editable": true + }, + "source": [ + "## When do we stop?\n", + "\n", + "A natural question is when do we stop the search for a new minimum?\n", + "One possibility is to compute the full gradient after a given number\n", + "of epochs and check if the norm of the gradient is smaller than some\n", + "threshold and stop if true. However, the condition that the gradient\n", + "is zero is valid also for local minima, so this would only tell us\n", + "that we are close to a local/global minimum. However, we could also\n", + "evaluate the cost function at this point, store the result and\n", + "continue the search. If the test kicks in at a later stage we can\n", + "compare the values of the cost function and keep the $\\beta$ that\n", + "gave the lowest value." + ] + }, + { + "cell_type": "markdown", + "id": "dcd04de7", + "metadata": { + "editable": true + }, + "source": [ + "## Slightly different approach\n", + "\n", + "Another approach is to let the step length $\\gamma_j$ depend on the\n", + "number of epochs in such a way that it becomes very small after a\n", + "reasonable time such that we do not move at all. Such approaches are\n", + "also called scaling. There are many such ways to [scale the learning\n", + "rate](https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1)\n", + "and [discussions here](https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf). See\n", + "also\n", + "\n", + "for a discussion of different scaling functions for the learning rate." + ] + }, + { + "cell_type": "markdown", + "id": "4ecf0b16", + "metadata": { + "editable": true + }, + "source": [ + "## Time decay rate\n", + "\n", + "As an example, let $e = 0,1,2,3,\\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \\cdot m + i$ where $m$ is the number of minibatches and $i=0,\\cdots,m-1$. Then the function $$\\gamma_j(t; t_0, t_1) = \\frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$.\n", + "\n", + "In this way we can fix the number of epochs, compute $\\beta$ and\n", + "evaluate the cost function at the end. Repeating the computation will\n", + "give a different result since the scheme is random by design. Then we\n", + "pick the final $\\beta$ that gives the lowest value of the cost\n", + "function." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "71eb89db", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "\n", + "def step_length(t,t0,t1):\n", + " return t0/(t+t1)\n", + "\n", + "n = 100 #100 datapoints \n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "n_epochs = 500 #number of epochs\n", + "t0 = 1.0\n", + "t1 = 10\n", + "\n", + "gamma_j = t0/t1\n", + "j = 0\n", + "for epoch in range(1,n_epochs+1):\n", + " for i in range(m):\n", + " k = np.random.randint(m) #Pick the k-th minibatch at random\n", + " #Compute the gradient using the data in minibatch Bk\n", + " #Compute new suggestion for beta\n", + " t = epoch*m+i\n", + " gamma_j = step_length(t,t0,t1)\n", + " j += 1\n", + "\n", + "print(\"gamma_j after %d epochs: %g\" % (n_epochs,gamma_j))" + ] + }, + { + "cell_type": "markdown", + "id": "10ce3fc7", + "metadata": { + "editable": true + }, + "source": [ + "## Code with a Number of Minibatches which varies\n", + "\n", + "In the code here we vary the number of mini-batches." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f2d402a8", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "from math import exp, sqrt\n", + "from random import random, seed\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = 2.0/n*X.T @ ((X @ theta)-y)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ffd1e92d", + "metadata": { + "editable": true + }, + "source": [ + "## Replace or not\n", + "\n", + "In the above code, we have use replacement in setting up the\n", + "mini-batches. The discussion\n", + "[here](https://sebastianraschka.com/faq/docs/sgd-methods.html) may be\n", + "useful." + ] + }, + { + "cell_type": "markdown", + "id": "4b72d3f9", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum based GD\n", + "\n", + "The stochastic gradient descent (SGD) is almost always used with a\n", + "*momentum* or inertia term that serves as a memory of the direction we\n", + "are moving in parameter space. This is typically implemented as\n", + "follows" + ] + }, + { + "cell_type": "markdown", + "id": "1b60ec02", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e1fa24ba", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t},\n", + "\\label{_auto1} \\tag{1}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "7a1fc9fc", + "metadata": { + "editable": true + }, + "source": [ + "where we have introduced a momentum parameter $\\gamma$, with\n", + "$0\\le\\gamma\\le 1$, and for brevity we dropped the explicit notation to\n", + "indicate the gradient is to be taken over a different mini-batch at\n", + "each step. We call this algorithm gradient descent with momentum\n", + "(GDM). From these equations, it is clear that $\\mathbf{v}_t$ is a\n", + "running average of recently encountered gradients and\n", + "$(1-\\gamma)^{-1}$ sets the characteristic time scale for the memory\n", + "used in the averaging procedure. Consistent with this, when\n", + "$\\gamma=0$, this just reduces down to ordinary SGD as discussed\n", + "earlier. An equivalent way of writing the updates is" + ] + }, + { + "cell_type": "markdown", + "id": "23abda5d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\boldsymbol{\\theta}_{t+1} = \\gamma \\Delta \\boldsymbol{\\theta}_t -\\ \\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0f0067a9", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined $\\Delta \\boldsymbol{\\theta}_{t}= \\boldsymbol{\\theta}_t-\\boldsymbol{\\theta}_{t-1}$." + ] + }, + { + "cell_type": "markdown", + "id": "b86ec959", + "metadata": { + "editable": true + }, + "source": [ + "## More on momentum based approaches\n", + "\n", + "Let us try to get more intuition from these equations. It is helpful\n", + "to consider a simple physical analogy with a particle of mass $m$\n", + "moving in a viscous medium with drag coefficient $\\mu$ and potential\n", + "$E(\\mathbf{w})$. If we denote the particle's position by $\\mathbf{w}$,\n", + "then its motion is described by" + ] + }, + { + "cell_type": "markdown", + "id": "7685ff97", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m {d^2 \\mathbf{w} \\over dt^2} + \\mu {d \\mathbf{w} \\over dt }= -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "28203c1e", + "metadata": { + "editable": true + }, + "source": [ + "We can discretize this equation in the usual way to get" + ] + }, + { + "cell_type": "markdown", + "id": "1495b23f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "m { \\mathbf{w}_{t+\\Delta t}-2 \\mathbf{w}_{t} +\\mathbf{w}_{t-\\Delta t} \\over (\\Delta t)^2}+\\mu {\\mathbf{w}_{t+\\Delta t}- \\mathbf{w}_{t} \\over \\Delta t} = -\\nabla_w E(\\mathbf{w}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "15bbc6e6", + "metadata": { + "editable": true + }, + "source": [ + "Rearranging this equation, we can rewrite this as" + ] + }, + { + "cell_type": "markdown", + "id": "ffeffc05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\mathbf{w}_{t +\\Delta t}= - { (\\Delta t)^2 \\over m +\\mu \\Delta t} \\nabla_w E(\\mathbf{w})+ {m \\over m +\\mu \\Delta t} \\Delta \\mathbf{w}_t.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "166cfafc", + "metadata": { + "editable": true + }, + "source": [ + "## Momentum parameter\n", + "\n", + "Notice that this equation is identical to previous one if we identify\n", + "the position of the particle, $\\mathbf{w}$, with the parameters\n", + "$\\boldsymbol{\\theta}$. This allows us to identify the momentum\n", + "parameter and learning rate with the mass of the particle and the\n", + "viscous drag as:" + ] + }, + { + "cell_type": "markdown", + "id": "486fadf0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\gamma= {m \\over m +\\mu \\Delta t }, \\qquad \\eta = {(\\Delta t)^2 \\over m +\\mu \\Delta t}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cdb0769d", + "metadata": { + "editable": true + }, + "source": [ + "Thus, as the name suggests, the momentum parameter is proportional to\n", + "the mass of the particle and effectively provides inertia.\n", + "Furthermore, in the large viscosity/small learning rate limit, our\n", + "memory time scales as $(1-\\gamma)^{-1} \\approx m/(\\mu \\Delta t)$.\n", + "\n", + "Why is momentum useful? SGD momentum helps the gradient descent\n", + "algorithm gain speed in directions with persistent but small gradients\n", + "even in the presence of stochasticity, while suppressing oscillations\n", + "in high-curvature directions. This becomes especially important in\n", + "situations where the landscape is shallow and flat in some directions\n", + "and narrow and steep in others. It has been argued that first-order\n", + "methods (with appropriate initial conditions) can perform comparable\n", + "to more expensive second order methods, especially in the context of\n", + "complex deep learning models.\n", + "\n", + "These beneficial properties of momentum can sometimes become even more\n", + "pronounced by using a slight modification of the classical momentum\n", + "algorithm called Nesterov Accelerated Gradient (NAG).\n", + "\n", + "In the NAG algorithm, rather than calculating the gradient at the\n", + "current parameters, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t)$, one\n", + "calculates the gradient at the expected value of the parameters given\n", + "our current momentum, $\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma\n", + "\\mathbf{v}_{t-1})$. This yields the NAG update rule" + ] + }, + { + "cell_type": "markdown", + "id": "e65569bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma \\mathbf{v}_{t-1}) \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d72d316", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\boldsymbol{\\theta}_{t+1}= \\boldsymbol{\\theta}_t -\\mathbf{v}_{t}.\n", + "\\label{_auto2} \\tag{2}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b7b9811e", + "metadata": { + "editable": true + }, + "source": [ + "One of the major advantages of NAG is that it allows for the use of a\n", + "larger learning rate than GDM for the same choice of $\\gamma$." + ] + }, + { + "cell_type": "markdown", + "id": "d8f3c523", + "metadata": { + "editable": true + }, + "source": [ + "## Second moment of the gradient\n", + "\n", + "In stochastic gradient descent, with and without momentum, we still\n", + "have to specify a schedule for tuning the learning rates $\\eta_t$\n", + "as a function of time. As discussed in the context of Newton's\n", + "method, this presents a number of dilemmas. The learning rate is\n", + "limited by the steepest direction which can change depending on the\n", + "current position in the landscape. To circumvent this problem, ideally\n", + "our algorithm would keep track of curvature and take large steps in\n", + "shallow, flat directions and small steps in steep, narrow directions.\n", + "Second-order methods accomplish this by calculating or approximating\n", + "the Hessian and normalizing the learning rate by the\n", + "curvature. However, this is very computationally expensive for\n", + "extremely large models. Ideally, we would like to be able to\n", + "adaptively change the step size to match the landscape without paying\n", + "the steep computational price of calculating or approximating\n", + "Hessians.\n", + "\n", + "During the last decade a number of methods have been introduced that accomplish\n", + "this by tracking not only the gradient, but also the second moment of\n", + "the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and\n", + "[ADAM](https://arxiv.org/abs/1412.6980)." + ] + }, + { + "cell_type": "markdown", + "id": "22d6be74", + "metadata": { + "editable": true + }, + "source": [ + "## Challenge: Choosing a Fixed Learning Rate\n", + "A fixed $\\eta$ is hard to get right:\n", + "1. If $\\eta$ is too large, the updates can overshoot the minimum, causing oscillations or divergence\n", + "\n", + "2. If $\\eta$ is too small, convergence is very slow (many iterations to make progress)\n", + "\n", + "In practice, one often uses trial-and-error or schedules (decaying $\\eta$ over time) to find a workable balance.\n", + "For a function with steep directions and flat directions, a single global $\\eta$ may be inappropriate:\n", + "1. Steep coordinates require a smaller step size to avoid oscillation.\n", + "\n", + "2. Flat/shallow coordinates could use a larger step to speed up progress.\n", + "\n", + "3. This issue is pronounced in high-dimensional problems with **sparse or varying-scale features** – we need a method to adjust step sizesper feature." + ] + }, + { + "cell_type": "markdown", + "id": "091ace5a", + "metadata": { + "editable": true + }, + "source": [ + "## Motivation for Adaptive Step Sizes\n", + "\n", + "1. Instead of a fixed global $\\eta$, use an \\textbf{adaptive learning rate} for each parameter that depends on the history of gradients.\n", + "\n", + "2. Parameters that have large accumulated gradient magnitude should get smaller steps (they've been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.\n", + "\n", + "3. This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected\n", + "\n", + "4. Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates\n", + "\n", + "5. Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive **AdaGrad**, one of the first adaptive methods." + ] + }, + { + "cell_type": "markdown", + "id": "5c05fb8b", + "metadata": { + "editable": true + }, + "source": [ + "## Derivation of the AdaGrad Algorithm\n", + "\n", + "**Accumulating Gradient History.**\n", + "\n", + "1. AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)\n", + "\n", + "2. Let $g_t = \\nabla \\ell_{i_t}(x_t)$ be the gradient at step $t$ (or a subgradient for nondifferentiable cases).\n", + "\n", + "3. Initialize $h_0 = 0$ (an all-zero vector in $\\mathbb{R}^d$).\n", + "\n", + "4. At each iteration $t$, update the accumulation:" + ] + }, + { + "cell_type": "markdown", + "id": "21fb8bd0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "h_t \\;=\\; h_{t-1} + g_t \\circ g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f71dfafa", + "metadata": { + "editable": true + }, + "source": [ + "1. Here $g_t \\circ g_t$ denotes element-wise square of the gradient vector. $h_t^{(j)} = h_{t-1}^{(j)} + (g_{t,j})^2$ for each parameter $j$.\n", + "\n", + "2. We can view $H_t = \\mathrm{diag}(h_t)$ as a diagonal matrix of past squared gradients. Initially $H_0 = 0$." + ] + }, + { + "cell_type": "markdown", + "id": "64869e2e", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Update Rule Derivation\n", + "\n", + "We scale the gradient by the inverse square root of the accumulated matrix $H_t$. The AdaGrad update at step $t$ is:" + ] + }, + { + "cell_type": "markdown", + "id": "c2d8d6b1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1} \\;=\\; x_t \\;-\\; \\eta\\, H_t^{-1/2}\\, g_t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2171198a", + "metadata": { + "editable": true + }, + "source": [ + "where $H_t^{-1/2}$ is the diagonal matrix with entries $(h_{t}^{(1)})^{-1/2}, \\dots, (h_{t}^{(d)})^{-1/2}$\n", + "In coordinates, this means each parameter $j$ has an individual step size:" + ] + }, + { + "cell_type": "markdown", + "id": "353e21d3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j} =x_{t,j} -\\frac{\\eta}{\\sqrt{h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b59e6add", + "metadata": { + "editable": true + }, + "source": [ + "In practice we add a small constant $\\epsilon$ in the denominator for numerical stability to avoid division by zero:" + ] + }, + { + "cell_type": "markdown", + "id": "707caefe", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_{t+1,j}= x_{t,j}-\\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}g_{t,j}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "62dafce4", + "metadata": { + "editable": true + }, + "source": [ + "Equivalently, the effective learning rate for parameter $j$ at time $t$ is $\\displaystyle \\alpha_{t,j} = \\frac{\\eta}{\\sqrt{\\epsilon + h_{t,j}}}$. This decreases over time as $h_{t,j}$ grows." + ] + }, + { + "cell_type": "markdown", + "id": "01a2ba61", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad Properties\n", + "1. AdaGrad automatically tunes the step size for each parameter. Parameters with more **volatile or large gradients** get smaller steps, and those with **small or infrequent gradients** get relatively larger steps\n", + "\n", + "2. No manual schedule needed: The accumulation $h_t$ keeps increasing (or stays the same if gradient is zero), so step sizes $\\eta/\\sqrt{h_t}$ are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.\n", + "\n", + "3. Sparse data benefit: For very sparse features, $h_{t,j}$ grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal\n", + "\n", + "4. Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate (e.g. $O(1/\\sqrt{T})$ regret bound) comparable to the best fixed learning rate tuned for the problem\n", + "\n", + "It effectively reduces the need to tune $\\eta$ by hand.\n", + "1. Limitations: Because $h_t$ accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)" + ] + }, + { + "cell_type": "markdown", + "id": "b4bc88e0", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp: Adaptive Learning Rates\n", + "Addresses AdaGrad’s diminishing learning rate issue.\n", + "Uses a decaying average of squared gradients (instead of a cumulative sum):" + ] + }, + { + "cell_type": "markdown", + "id": "4933fddb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "v_t = \\beta_2\\, v_{t-1} + (1-\\beta_2)\\, (\\nabla L(w_t))^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0dd8edf7", + "metadata": { + "editable": true + }, + "source": [ + "with $\\beta_2$ typically $0.9$ (or $0.99$).\n", + "1. Update: $w_{t+1} = w_t - \\frac{\\alpha}{\\sqrt{v_t + \\epsilon}} \\nabla L(w_t)$.\n", + "\n", + "2. Recent gradients have more weight, so $v_t$ adapts to the current landscape.\n", + "\n", + "3. Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.\n", + "\n", + "RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 - unpublished.)" + ] + }, + { + "cell_type": "markdown", + "id": "1ce98f28", + "metadata": { + "editable": true + }, + "source": [ + "## Adam Optimizer\n", + "\n", + "Why Combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.\n", + "\n", + "1. Fast convergence by smoothing gradients (accelerates in long-term gradient direction).\n", + "\n", + "2. Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).\n", + "\n", + "3. Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)\n", + "\n", + "4. Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)\n", + "\n", + "**Result**: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice." + ] + }, + { + "cell_type": "markdown", + "id": "d7d646bc", + "metadata": { + "editable": true + }, + "source": [ + "## RMS prop\n", + "\n", + "In RMS prop, in addition to keeping a running average of the first\n", + "moment of the gradient, we also keep track of the second moment\n", + "denoted by $\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}_t^2]$. The update rule\n", + "for RMS prop is given by" + ] + }, + { + "cell_type": "markdown", + "id": "3a212292", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto3} \\tag{3}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "88c0ec96", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta \\mathbf{s}_{t-1} +(1-\\beta)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9cfb794f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\mathbf{g}_t \\over \\sqrt{\\mathbf{s}_t +\\epsilon}}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "aaa4936c", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta$ controls the averaging time of the second moment and is\n", + "typically taken to be about $\\beta=0.9$, $\\eta_t$ is a learning rate\n", + "typically chosen to be $10^{-3}$, and $\\epsilon\\sim 10^{-8} $ is a\n", + "small regularization constant to prevent divergences. Multiplication\n", + "and division by vectors is understood as an element-wise operation. It\n", + "is clear from this formula that the learning rate is reduced in\n", + "directions where the norm of the gradient is consistently large. This\n", + "greatly speeds up the convergence by allowing us to use a larger\n", + "learning rate for flat directions." + ] + }, + { + "cell_type": "markdown", + "id": "5493a6cd", + "metadata": { + "editable": true + }, + "source": [ + "## [ADAM optimizer](https://arxiv.org/abs/1412.6980)\n", + "\n", + "A related algorithm is the ADAM optimizer. In\n", + "[ADAM](https://arxiv.org/abs/1412.6980), we keep a running average of\n", + "both the first and second moment of the gradient and use this\n", + "information to adaptively change the learning rate for different\n", + "parameters. The method isefficient when working with large\n", + "problems involving lots data and/or parameters. It is a combination of the\n", + "gradient descent with momentum algorithm and the RMSprop algorithm\n", + "discussed above.\n", + "\n", + "In addition to keeping a running average of the first and\n", + "second moments of the gradient\n", + "(i.e. $\\mathbf{m}_t=\\mathbb{E}[\\mathbf{g}_t]$ and\n", + "$\\mathbf{s}_t=\\mathbb{E}[\\mathbf{g}^2_t]$, respectively), ADAM\n", + "performs an additional bias correction to account for the fact that we\n", + "are estimating the first two moments of the gradient using a running\n", + "average (denoted by the hats in the update rule below). The update\n", + "rule for ADAM is given by (where multiplication and division are once\n", + "again understood to be element-wise operations below)" + ] + }, + { + "cell_type": "markdown", + "id": "35795070", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\mathbf{g}_t = \\nabla_\\theta E(\\boldsymbol{\\theta}) \n", + "\\label{_auto4} \\tag{4}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1e67e6dd", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{m}_t = \\beta_1 \\mathbf{m}_{t-1} + (1-\\beta_1) \\mathbf{g}_t \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39d582bb", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbf{s}_t =\\beta_2 \\mathbf{s}_{t-1} +(1-\\beta_2)\\mathbf{g}_t^2 \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a7e2423", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{m}}_t={\\mathbf{m}_t \\over 1-\\beta_1^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "53da77c2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\mathbf{s}}_t ={\\mathbf{s}_t \\over1-\\beta_2^t} \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a4c7df45", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\boldsymbol{\\mathbf{m}}_t \\over \\sqrt{\\boldsymbol{\\mathbf{s}}_t} +\\epsilon}, \\nonumber\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f645a0aa", + "metadata": { + "editable": true + }, + "source": [ + "\n", + "
      \n", + "\n", + "$$\n", + "\\begin{equation} \n", + "\\label{_auto5} \\tag{5}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "505619bb", + "metadata": { + "editable": true + }, + "source": [ + "where $\\beta_1$ and $\\beta_2$ set the memory lifetime of the first and\n", + "second moment and are typically taken to be $0.9$ and $0.99$\n", + "respectively, and $\\eta$ and $\\epsilon$ are identical to RMSprop.\n", + "\n", + "Like in RMSprop, the effective step size of a parameter depends on the\n", + "magnitude of its gradient squared. To understand this better, let us\n", + "rewrite this expression in terms of the variance\n", + "$\\boldsymbol{\\sigma}_t^2 = \\boldsymbol{\\mathbf{s}}_t -\n", + "(\\boldsymbol{\\mathbf{m}}_t)^2$. Consider a single parameter $\\theta_t$. The\n", + "update rule for this parameter is given by" + ] + }, + { + "cell_type": "markdown", + "id": "6cb4c9a9", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\Delta \\theta_{t+1}= -\\eta_t { \\boldsymbol{m}_t \\over \\sqrt{\\sigma_t^2 + m_t^2 }+\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "39627950", + "metadata": { + "editable": true + }, + "source": [ + "## Algorithms and codes for Adagrad, RMSprop and Adam\n", + "\n", + "The algorithms we have implemented are well described in the text by [Goodfellow, Bengio and Courville, chapter 8](https://www.deeplearningbook.org/contents/optimization.html).\n", + "\n", + "The codes which implement these algorithms are discussed after our presentation of automatic differentiation." + ] + }, + { + "cell_type": "markdown", + "id": "f240057d", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "428ee09d", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "9ce3cc3e", + "metadata": { + "editable": true + }, + "source": [ + "## ADAM algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

      Figure 1:

      \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "193b3778", + "metadata": { + "editable": true + }, + "source": [ + "## Practical tips\n", + "\n", + "* **Randomize the data when making mini-batches**. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.\n", + "\n", + "* **Transform your inputs**. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.\n", + "\n", + "* **Monitor the out-of-sample performance.** Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This *early stopping* significantly improves performance in many settings.\n", + "\n", + "* **Adaptive optimization methods don't always have good generalization.** Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications." + ] + }, + { + "cell_type": "markdown", + "id": "8d7187f5", + "metadata": { + "editable": true + }, + "source": [ + "## Sneaking in automatic differentiation using Autograd\n", + "\n", + "In the examples here we take the liberty of sneaking in automatic\n", + "differentiation (without having discussed the mathematics). In\n", + "project 1 you will write the gradients as discussed above, that is\n", + "hard-coding the gradients. By introducing automatic differentiation\n", + "via the library **autograd**, which is now replaced by **JAX**, we have\n", + "more flexibility in setting up alternative cost functions.\n", + "\n", + "The\n", + "first example shows results with ordinary leats squares." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "68d35b76", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f39b471c", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "265059ef", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients for OLS\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x#+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 30\n", + "\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(theta)\n", + " theta -= eta*gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "# Now improve with momentum gradient descent\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "for iter in range(Niterations):\n", + " # calculate gradient\n", + " gradients = training_gradient(theta)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"theta from own gd wth momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "18183fb0", + "metadata": { + "editable": true + }, + "source": [ + "## Including Stochastic Gradient Descent with Autograd\n", + "\n", + "In this code we include the stochastic gradient descent approach\n", + "discussed above. Note here that we specify which argument we are\n", + "taking the derivative with respect to when using **autograd**." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4d9d93b3", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 1000\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "xnew = np.array([[0],[2]])\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(xnew, ypredict2, \"b-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,2.0,0, 15.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Random numbers ')\n", + "plt.show()\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "for epoch in range(n_epochs):\n", + "# Can you figure out a better way of setting up the contributions to each batch?\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " theta = theta - eta*gradients\n", + "print(\"theta from own sdg\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "1652a471", + "metadata": { + "editable": true + }, + "source": [ + "## Same code but now with momentum gradient descent" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6092712c", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using SGD\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "EigValues, EigVectors = np.linalg.eig(H)\n", + "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", + "\n", + "theta = np.random.randn(2,1)\n", + "eta = 1.0/np.max(EigValues)\n", + "Niterations = 100\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = (1.0/n)*training_gradient(y, X, theta)\n", + " theta -= eta*gradients\n", + "print(\"theta from own gd\")\n", + "print(theta)\n", + "\n", + "\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "t0, t1 = 5, 50\n", + "def learning_schedule(t):\n", + " return t0/(t+t1)\n", + "\n", + "theta = np.random.randn(2,1)\n", + "\n", + "change = 0.0\n", + "delta_momentum = 0.3\n", + "\n", + "for epoch in range(n_epochs):\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " eta = learning_schedule(epoch*m+i)\n", + " # calculate update\n", + " new_change = eta*gradients+delta_momentum*change\n", + " # take a step\n", + " theta -= new_change\n", + " # save the change\n", + " change = new_change\n", + "print(\"theta from own sdg with momentum\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "4d2f230d", + "metadata": { + "editable": true + }, + "source": [ + "## But none of these can compete with Newton's method\n", + "\n", + "Note that we here have introduced automatic differentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b46cedcc", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Newton's method\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "from autograd import grad\n", + "\n", + "def CostOLS(beta):\n", + " return (1.0/n)*np.sum((y-X @ beta)**2)\n", + "\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+5*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(beta_linreg)\n", + "# Hessian matrix\n", + "H = (2.0/n)* XT_X\n", + "# Note that here the Hessian does not depend on the parameters beta\n", + "invH = np.linalg.pinv(H)\n", + "beta = np.random.randn(3,1)\n", + "Niterations = 5\n", + "# define the gradient\n", + "training_gradient = grad(CostOLS)\n", + "\n", + "for iter in range(Niterations):\n", + " gradients = training_gradient(beta)\n", + " beta -= invH @ gradients\n", + " print(iter,gradients[0],gradients[1])\n", + "print(\"beta from own Newton code\")\n", + "print(beta)" + ] + }, + { + "cell_type": "markdown", + "id": "f048d5c7", + "metadata": { + "editable": true + }, + "source": [ + "## Similar (second order function now) problem but now with AdaGrad" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d8bdd180", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " Giter += gradients*gradients\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + " theta -= update\n", + "print(\"theta from own AdaGrad\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "145ce394", + "metadata": { + "editable": true + }, + "source": [ + "Running this code we note an almost perfect agreement with the results from matrix inversion." + ] + }, + { + "cell_type": "markdown", + "id": "ec0ee110", + "metadata": { + "editable": true + }, + "source": [ + "## RMSprop for adaptive learning rate with Stochastic Gradient Descent" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b6e8e3e4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameter rho\n", + "rho = 0.99\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-8\n", + "for epoch in range(n_epochs):\n", + " Giter = 0.0\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + "\t# Accumulated gradient\n", + "\t# Scaling with rho the new and the previous results\n", + " Giter = (rho*Giter+(1-rho)*gradients*gradients)\n", + "\t# Taking the diagonal only and inverting\n", + " update = gradients*eta/(delta+np.sqrt(Giter))\n", + "\t# Hadamard product\n", + " theta -= update\n", + "print(\"theta from own RMSprop\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "79d8c0ae", + "metadata": { + "editable": true + }, + "source": [ + "## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "c16ec5ac", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980\n", + "beta1 = 0.9\n", + "beta2 = 0.999\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-7\n", + "iter = 0\n", + "for epoch in range(n_epochs):\n", + " first_moment = 0.0\n", + " second_moment = 0.0\n", + " iter += 1\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " # Computing moments first\n", + " first_moment = beta1*first_moment + (1-beta1)*gradients\n", + " second_moment = beta2*second_moment+(1-beta2)*gradients*gradients\n", + " first_term = first_moment/(1.0-beta1**iter)\n", + " second_term = second_moment/(1.0-beta2**iter)\n", + "\t# Scaling with rho the new and the previous results\n", + " update = eta*first_term/(np.sqrt(second_term)+delta)\n", + " theta -= update\n", + "print(\"theta from own ADAM\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "7c581f8b", + "metadata": { + "editable": true + }, + "source": [ + "## Material for the lab sessions\n", + "\n", + "1. Exercise set for week 37 and reminder on scaling (from lab sessions of week 35)\n", + "\n", + "2. Work on project 1\n", + "\n", + "\n", + " * For more discussions of Ridge regression and calculation of averages, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended." + ] + }, + { + "cell_type": "markdown", + "id": "d8a82e01", + "metadata": { + "editable": true + }, + "source": [ + "## Reminder on different scaling methods\n", + "\n", + "Before fitting a regression model, it is good practice to normalize or\n", + "standardize the features. This ensures all features are on a\n", + "comparable scale, which is especially important when using\n", + "regularization. In the exercises this week we will perform standardization, scaling each\n", + "feature to have mean 0 and standard deviation 1.\n", + "\n", + "Here we compute the mean and standard deviation of each column (feature) in our design/feature matrix $\\boldsymbol{X}$.\n", + "Then we subtract the mean and divide by the standard deviation for each feature.\n", + "\n", + "In the example here we\n", + "we will also center the target $\\boldsymbol{y}$ to mean $0$. Centering $\\boldsymbol{y}$\n", + "(and each feature) means the model does not require a separate intercept\n", + "term, the data is shifted such that the intercept is effectively 0\n", + ". (In practice, one could include an intercept in the model and not\n", + "penalize it, but here we simplify by centering.)\n", + "Choose $n=100$ data points and set up $\\boldsymbol{x}, $\\boldsymbol{y}$ and the design matrix $\\boldsymbol{X}$." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "3fa174a9", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Standardize features (zero mean, unit variance for each feature)\n", + "X_mean = X.mean(axis=0)\n", + "X_std = X.std(axis=0)\n", + "X_std[X_std == 0] = 1 # safeguard to avoid division by zero for constant features\n", + "X_norm = (X - X_mean) / X_std\n", + "\n", + "# Center the target to zero mean (optional, to simplify intercept handling)\n", + "y_mean = ?\n", + "y_centered = ?" + ] + }, + { + "cell_type": "markdown", + "id": "b02de7a6", + "metadata": { + "editable": true + }, + "source": [ + "Do we need to center the values of $y$?\n", + "\n", + "After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n", + "and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This can make the optimization landscape\n", + "nicer and ensures the regularization penalty $\\lambda \\sum_j\n", + "\\theta_j^2$ in Ridge regression treats each coefficient fairly (since features are on the\n", + "same scale)." + ] + }, + { + "cell_type": "markdown", + "id": "cd3f85ef", + "metadata": { + "editable": true + }, + "source": [ + "## Functionality in Scikit-Learn\n", + "\n", + "**Scikit-Learn** has several functions which allow us to rescale the\n", + "data, normally resulting in much better results in terms of various\n", + "accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n", + "ensures that for each feature/predictor we study the mean value is\n", + "zero and the variance is one (every column in the design/feature\n", + "matrix). This scaling has the drawback that it does not ensure that\n", + "we have a particular maximum or minimum in our data set. Another\n", + "function included in **Scikit-Learn** is the **MinMaxScaler** which\n", + "ensures that all features are exactly between $0$ and $1$. The" + ] + }, + { + "cell_type": "markdown", + "id": "1e0198a0", + "metadata": { + "editable": true + }, + "source": [ + "## More preprocessing\n", + "\n", + "The **Normalizer** scales each data\n", + "point such that the feature vector has a euclidean length of one. In other words, it\n", + "projects a data point on the circle (or sphere in the case of higher dimensions) with a\n", + "radius of 1. This means every data point is scaled by a different number (by the\n", + "inverse of it’s length).\n", + "This normalization is often used when only the direction (or angle) of the data matters,\n", + "not the length of the feature vector.\n", + "\n", + "The **RobustScaler** works similarly to the StandardScaler in that it\n", + "ensures statistical properties for each feature that guarantee that\n", + "they are on the same scale. However, the RobustScaler uses the median\n", + "and quartiles, instead of mean and variance. This makes the\n", + "RobustScaler ignore data points that are very different from the rest\n", + "(like measurement errors). These odd data points are also called\n", + "outliers, and might often lead to trouble for other scaling\n", + "techniques." + ] + }, + { + "cell_type": "markdown", + "id": "04da94ca", + "metadata": { + "editable": true + }, + "source": [ + "## Frequently used scaling functions\n", + "\n", + "Many features are often scaled using standardization to improve performance. In **Scikit-Learn** this is given by the **StandardScaler** function as discussed above. It is easy however to write your own. \n", + "Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:" + ] + }, + { + "cell_type": "markdown", + "id": "aa1ad257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6587016d", + "metadata": { + "editable": true + }, + "source": [ + "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard deviation, respectively, of the feature $x_j$.\n", + "This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2e0350b4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "8179b037", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "3ebed257", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\theta_0, \\theta_1, ... , \\theta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij}\\theta_j\\right)^2,.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "51b239a4", + "metadata": { + "editable": true + }, + "source": [ + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", + "\n", + "What we have done is to single out the $\\theta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\theta_0$, we want the derivative to obey" + ] + }, + { + "cell_type": "markdown", + "id": "295790f3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_j} = 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "b68338d6", + "metadata": { + "editable": true + }, + "source": [ + "for all $j$. For $\\theta_0$ we have" + ] + }, + { + "cell_type": "markdown", + "id": "c015cb3b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\frac{\\partial C}{\\partial \\theta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij} \\theta_j\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5e4917fa", + "metadata": { + "editable": true + }, + "source": [ + "Multiplying away the constant $2/n$, we obtain" + ] + }, + { + "cell_type": "markdown", + "id": "150fedf5", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\sum_{i=0}^{n-1} \\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6d782b1", + "metadata": { + "editable": true + }, + "source": [ + "Let us specialize first to the case where we have only two parameters $\\theta_0$ and $\\theta_1$.\n", + "Our result for $\\theta_0$ simplifies then to" + ] + }, + { + "cell_type": "markdown", + "id": "a4065dd3", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "n\\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\theta_1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "06324d79", + "metadata": { + "editable": true + }, + "source": [ + "We obtain then" + ] + }, + { + "cell_type": "markdown", + "id": "9e3e6b35", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\theta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "cd9c3754", + "metadata": { + "editable": true + }, + "source": [ + "If we define" + ] + }, + { + "cell_type": "markdown", + "id": "8361829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "8f5f46fd", + "metadata": { + "editable": true + }, + "source": [ + "and the mean value of the outputs as" + ] + }, + { + "cell_type": "markdown", + "id": "cf210a6e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d3af09fc", + "metadata": { + "editable": true + }, + "source": [ + "we have" + ] + }, + { + "cell_type": "markdown", + "id": "ec91f8c1", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\mu_y - \\theta_1\\mu_{\\boldsymbol{x}_1}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0c9f35e6", + "metadata": { + "editable": true + }, + "source": [ + "In the general case with more parameters than $\\theta_0$ and $\\theta_1$, we have" + ] + }, + { + "cell_type": "markdown", + "id": "de80fa4b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\theta_j.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "34c99b80", + "metadata": { + "editable": true + }, + "source": [ + "We can rewrite the latter equation as" + ] + }, + { + "cell_type": "markdown", + "id": "758ade4d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\theta_j,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "ff0f4725", + "metadata": { + "editable": true + }, + "source": [ + "where we have defined" + ] + }, + { + "cell_type": "markdown", + "id": "997b829b", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "621e7d85", + "metadata": { + "editable": true + }, + "source": [ + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", + "\n", + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" + ] + }, + { + "cell_type": "markdown", + "id": "5530e459", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\theta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "fd9bace8", + "metadata": { + "editable": true + }, + "source": [ + "If we minimize with respect to $\\boldsymbol{\\theta}$ we have then" + ] + }, + { + "cell_type": "markdown", + "id": "d949be75", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ad52617", + "metadata": { + "editable": true + }, + "source": [ + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\theta}^T\\boldsymbol{\\theta}$ to the cost function and get then" + ] + }, + { + "cell_type": "markdown", + "id": "41cabc72", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "1fd416a6", + "metadata": { + "editable": true + }, + "source": [ + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f13cacb4", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_theta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_theta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_theta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "theta = fit_theta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True theta: {true_theta}\")\n", + "print(f\"Fitted theta: {theta}\")\n", + "print(f\"Sklearn fitted theta: {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ theta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "theta = fit_theta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ theta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted theta (without intercept): {theta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted theta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ theta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ theta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ff4b28ed", + "metadata": { + "editable": true + }, + "source": [ + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", + "\n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\theta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", + "\n", + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" + ] + }, + { + "cell_type": "markdown", + "id": "03b4dc3c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\theta_j^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "6cff2148", + "metadata": { + "editable": true + }, + "source": [ + "but when we take out the intercept, this equation becomes" + ] + }, + { + "cell_type": "markdown", + "id": "51d823c4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\theta_j^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "5ab270f9", + "metadata": { + "editable": true + }, + "source": [ + "For Lasso regression we have" + ] + }, + { + "cell_type": "markdown", + "id": "2b2e1492", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\theta_j\\vert.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "063be947", + "metadata": { + "editable": true + }, + "source": [ + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", + "\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "5e841a10", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeTheta\n", + " ypredictOwnRidge = X_test @ OwnRidgeTheta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta)\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "40e54083", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "edd7dd42", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", + "\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", + "\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", + "\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", + "\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeTheta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeTheta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeTheta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Theta values for own Ridge implementation\")\n", + " print(OwnRidgeTheta) #Intercept is given by mean of target variable\n", + " print(\"Theta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f9cc046b", + "metadata": { + "editable": true + }, + "source": [ + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\theta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +}
  • Exercises week 36