From 8ef566570590967bd7eafbbb23e5b03e83ff23cd Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Mon, 11 Sep 2023 21:07:47 +0200 Subject: [PATCH] updating typos --- .../_build/.doctrees/environment.pickle | Bin 307353 -> 307416 bytes .../_build/.doctrees/exercisesweek37.doctree | Bin 20148 -> 20735 bytes .../_build/.doctrees/week35.doctree | Bin 917823 -> 918232 bytes .../_build/.doctrees/week36.doctree | Bin 226725 -> 284049 bytes .../html/_sources/exercisesweek37.ipynb | 65 +- .../_build/html/_sources/week36.ipynb | 2362 +++++++++++------ .../_build/html/exercisesweek37.html | 23 +- doc/LectureNotes/_build/html/searchindex.js | 2 +- doc/LectureNotes/_build/html/week35.html | 56 +- doc/LectureNotes/_build/html/week36.html | 661 ++++- .../jupyter_execute/exercisesweek37.ipynb | 65 +- .../_build/jupyter_execute/week35.ipynb | 56 +- .../_build/jupyter_execute/week36.ipynb | 2362 +++++++++++------ .../_build/jupyter_execute/week36.py | 662 ++++- doc/LectureNotes/exercisesweek37.ipynb | 65 +- doc/LectureNotes/week36.ipynb | 2362 +++++++++++------ doc/src/week37/exercisesweek37.do.txt | 24 +- 17 files changed, 6112 insertions(+), 2653 deletions(-) diff --git a/doc/LectureNotes/_build/.doctrees/environment.pickle b/doc/LectureNotes/_build/.doctrees/environment.pickle index 0ffddc3a57297c8e4428c5939e22f334a41cd290..a59ff7c2da4e3688e9b17c9379d17a3f81f5cef9 100644 GIT binary patch literal 307416 zcmd?S37jNHbtkNm=03aU9-XNvX{JY->bW#R8c9IUrDINJMiQW-on4jPl{M8>r8=gk zCA8+Sg~l!;3?&&1i;ZnC7<1oXW58>SjlDjx!N8i&7~6}@XMgzft`}eXfAKOiA~UkO zGh({N{#p9{)YF|2`SQj4zj*QD#fyknE_n9x%jR81|8-V3>m|Q>e%Py?^BcA5q*rZq zI|KXMex-OwdGyBawpVqZ><)FVaO?H%@y^m>t%Bj zAX2DR$Nh`G-&YiAPYIx&!Z4UVjq#y(ixom#neJmDdKOneshx@{ANN&6au{VhR zK&dwAjn<(Yqka`)DRiIitZO&DQI%N%7(}_23eY`T`~m}s=Y#azZ#LUrGeEa{q1{<8 z^rcyAHws?Gubv%kw#Uc)^Yqn))%GN*^3$D-b+_tOnxk&9NXph~qP6wWLbFL^o9%kN z;WeA1Q{2^>-AB-b$?0YbeQVP7p@43vw|5fE7lR6SBdSP4RlFq~{xTIdOdQQ$M?HRx+kch-}@ zN4=^$2K^+?s!h@Nsfo+bxoXwXaSvo*LW;FY#cfQYhoKh}=KlCJ5No!_nxNYzn*q7C z+~(Pe-)y0!1A;b~9&JKKlVF9YqF4Bp7DPOsR2zL+XgI2`_s)9-L1xsgRBBVBFmttr z&=pxidhAWs(Zm6WzSeHl+bvi-*bzWIzgenHp)bnnEq~HOrS6eyJ1c@7G+Ka(_ehSL z!Y5qc8x8FhaGVeI0gsy`sQ@WO2|CLwE^H#CH{n+yp~dy$xOmd#UZ!-}QGWh3@ur|p z=BnG-Z#{18ARQW>h5>4K7Bsz9Ta0uG7}o^mzEzum5eoWsGzd^P2%jwyfK{Qc%R|wS zy&;grI6{L3%x-5@@MoxpuyDZ>wRPRjdVV^Lr^8|h;J>3m4n2zOL$|cPvoRo`%gih7 zE!|pLeY~`$w6?Uaw7#^Vw6V0iw7InB>C!D1O1G76FYPPcQM$8qSLubN7nNRI8YvyV zP`Vox?!o`}mhLMZd%ASIbfR>B>4DPAFO(j-PXYgZ7aO&Du{Kr3)0R6iTEYko zs|)pXivGD_Sjh_isg&nuDG&5p5@3B-De#jSYK+pC#pl)6qKD`$fsro4j!*hfPH6w- zWWE-U(irK(^1cy&Jo<3{i9Oq&yrFx#--G6NtLPN3FFaO>DA#EV5k@!2M1zoSSjpa?9mGGsH{|4!9!9w90?mnxCx`y zS$3iUOW_p{`h^x+we1w_(fr{CtW{?KV^OQo7M@;_J)J9tCyOw3)G6}RDHgK9@0Q-& z?yTTJZgjjx7EyR&i<`x4fM+;hHZXwh&^sjidv`{O0 z`LRl^0J$tZ3IhuJOeJl_WWbSu8FyhcH>C8pVYlw%E2mNE80th0;{x`EXpmTdUf8I0 zOYiI~K3FTXh3_otmEH@}jK_=4)Cypp$ozu~_N7oKi)$4VcDqb|Pm@yj@y4x8C=b2P{!`vL-MunZ3$muXEGkMr{&Jn;@?V?>McRa~e(*+#2TrVVXiWs5xu9ZdE8N?RtUo z*z4pEdIev6iBzJw-8qE0(j;l@F%P~~ktK?*ER=QDZ{DU5$2SMyRFQiOkdy+iv+*Ft zY~h8t6~8r&1{cvQ(cW1Zc z#TD+;D}DD?FfA2$h2?yj-skssdMdRIdPM@9nIjDQMX}zX$%Q zh?(K-MKWg98GwbsytjD|0y76F*wpRpgiEp;uZ#vouLwxQ{(c4f(xYy}bGyxEXBGRV z_^NP}#OxKHUy5i{2_rU0c1ySS2rrc2*|?pB=iCY$5B3fNy9bBxq#)4kT!9cK*)f{G zls^vn;CHy=Yzfi0Fg}5)D(1?VMlYgkj7_BW~28PH84*>tU!IXpj9in z)9RBY;>qavWUG5Td=+!7@K<>4wStR?0OFd$_g=(*BP0`5*~oq^X0{xE6!=wL3NWsJ zg7AHE_=&LW$tU0vKG}`lbtXK-C(gHmiT-%|@yDk}5ge*EA#*f{CjE0 zstMm#N@61guf`GEB)%|VZ6LmiC<{=1I(m12>P*(iD@HJn0v)pKnpVS~gj&~HG)N-y zHjW7h0xF};I>kdTL`0U0Nf1Y^(Eji_O?xB2SC5<7H1;5=$qXHZs?{URjT zSxEsD((NWiv`Qi<^Jp>zG3i%*%xxyTJv*SKh?Gy`pB)u{YzM3Vp5fbf&~zk^R-8qE zvAH9dkx-}QpfZ9J;fK1NAGr7r)TfJx`$29L{?|X@MN(Q7es^>XadL``M@1{n!fBHc zM=52*7$O0!2b3bVOA}zs?_ni8ifum9jp!`JoN?5{xq z6rpaZ!iy21sQZPp)7=ZA-pb(bQGcRJfl4@2M0aicF;sN|CXNdcY>C9PCYbtSx{KdN zi(VZ{bYxywUV2OEt>N!%tAl_eb#|%Ook6m}quyhXZD{o+%gaO|e@N*}OIwI0!{|}u zQ8KiQo+7H}H4uT7G?pUh*%*a9us|fAE^9*Ct;gEru_`oHE00HAo^N!ncYhRaj=F@Ysh@GXP28UJ3yDq=%GVy~-R?8A)DJLopUKAm9@xiEMnRqdX zn`1~Kszi{v4BH3|^{U;dkOMW~3UM1m0lLb2K@3NZV|PVwtrDL?h^*T3$9-BOS=6qc zts;CTDKk49I_{Pas!d!hUT3QT#Xh9ns(LcdT~7M;b*V8?8|+NMICK3V8vPmqnxDfC9$h>l87L ze*dx3>!RPEDZM`YJ^Tbkn+hRcL!C!1eryW|H6=v91Irrpoo?r$IioLw{gV1iE}nX@ zY#?bqaPc%liO`CJB>b{DBOI<>NZMzTw2hMjNC9_1FT|o}fS%%(2>rwefF<9+@ro2{ zohZX(r8U~F!_D#Fa|+))boDvX-Gm9lQZbD>iC5u)ie(5aD+(7tt+7TBXa53g5JjI> z1)q)xa51^Cbq~R8*abvxJ8Kz$;cyrW*16Y^R?=D_mXI5xwJ{3iaIiR{$9-3!B|n8F zesob;G%D6OJC}d{W3oLQ=2tCID1)GbBd*(`X242ah(5|?q~D|L`YR3OuL|qOWzp;ZyGNvndkP|y8n8wxJIc8&jFK7tW8I&P}G>U zN(~;_n&&2{V2x5B*X0Ijp)H#FIPlsr6;*2TT8)^@bzk}nT&i$d=U{Mi>mEYuhtAiB zundHeYMb-KQV0Jvu&UaqI{a`LT3>;qIke-U;hS!L(W5s!@#N69S3LCaBM(3P$c~Ba z&en$?zG2JOZFe|3b~ra|aV9;?XvMn6R0*?63YgO3NkDArlS`MP-T}D9W0*uaQzc|6 zxL6{oPdmIK>ma+KP=V*}6^ES%{Yu3d^PD;&8lXX|DOC)n_{D8GQ*Km~XhP506;Y?^ zOf=jvQJYjmtbqiLaO%DXOF-)+D)^eJ?MqT^Q`4vZ{~#KnR<07lP%<`BJo zfYh_v+Ai2hUw7_u#@$NO+wPF|5e2SwlvEFeSb;+sD;<}}rzDs| zq-^oZq-FH=P{|HJ$3SWG=Ist?)xlcfAgyeQM+B-s^&IMm`Az+f=x>3}9b66>9;`W2 zSg@kx4hIc(=sReg{1D0lZRXJT=6epEI+TyXMVns~QHC6eXW|lUTz`EU%WCC&m~T;6 zZ9~Lc0U@fE7qo!ZfuO#Wq6VEOYY?DJNi@)8!MPysk2^yl5}|&xjtAIOv78q4f~}7Z zZFO!=g%gjrZW|iDY1`Iqg6xwH7V*8Hzsy1}*3$^lq^@k!sB^0!`DQE7L=qijK4@&-Cb+3GAiFBl^Bv$!d^reqaO@6X_Ok; z`fzpYHdWF>$3TUs4vRmASPfZ{Bs>C%tG(hCj>%NF%xNj``^n~S3a5>pq=d1&_|anhRF(7W`X_P*YPCxzm1 zbtfjJK(=1B*nAL6!J_#DifSIVIa|oK5ran2$Ng<8To0wvK3iaiZ&bMCt9E@$gjQ|f|ij8fDxSj2nB zG2#faOrV`d?T=YIXtvY3_?UIZgO!SZ)HhIVdu^1E33S#txgLa7OFpxchz zOm5%XjI;EY|4H;VB{ zr#5h0xm<}RhYm5b5M&7>ZqX3Vod{wDIij?XPz)jy5WnhqBSR}{^3`H)Kxr@{3r2X9 z)moU!TrN=jC;m~Gl@eTAN+QNbWr$BW%(skE$N|sg90BoBv2et*ug)D|YHgud9WFP? z^E}rWu6nH<)%xU)pzezi|8DNx0ig1ela+jdm_%hMzi>554Yh@At_(jIL|Z5lc=^La zh@d^Zt*cB7!)iqz!v+>=xZXol?-8Y5L=M9aCD=-kLMC>TyB#Ykzl&l>@1`FiSF6Is zmq2fzF~37`M@0Gt!XjGk;hc-8Y+Yk zczPJP+YnY0Q{ZqVsD`-Uro;CiJ?b1EId;go^U$%8dyl?&fp|H$2V_2|(TSA0q+m#>wIrE{dG<@b?v-cKSR87$>xKamC;1l;)v z_7OnW#N;`Mm|w*OFms&W$tkwTnr%;3gIN12S>-(DtO#3+k8I+{ty;a}o%0a2;)F3+ z9>n5_i?v)KIcXPV0ZlQP{xw0IP&gCchr6G$HxlB$^(w4ECx3eLk2O0YEa z&{3#Tb^G1OAv*iW6K(RE#U22!DMxXo`V;Lcq5?>Uqh29v+~^?mNk&|R_Jq5I(Fd{8 z=$j8Cgwd)!Os+63AtEc6Left>3=82+QMd<*;!iw`4nhuElLGEfK0(7j{q>~$(8EZi zM%bQ`X`hH_lT?u18!ji_MY_H{UJ3(3p`GZVq>=>LUb_m$R}9H3#wQF;k*C`LMYwg9+}q)sIQhT*kx&-}w4eckWcw$tI;)dlPRfo|*7zDpMF1E|WavVaQ#QG3cbWU7>jaS#5 zfa`>odawo2VgReP$mqgC(J{YCnZ*cz*4oW(*nrjHXLJiX5EdzBNr-8ESdeM~BgUmz zI*Y?KcGj$QA%QBIha3~X0<+f+WV10WEHr9JF^(4HL)?izwiU2s$zUNMy6C36^8aP? z%|9RqtrenH;8dcPTsADAem5^wk&T!!HbqI=uw5mP%(94JyFh#faxHIMEYRof`51z!voo;&k)$!aTQH4(N?{m$7jiWO7FZe-B% zlEi={fr_-9DU}EX9AHM{I+drHpwc;UqMW~F*#a+AAo^Hen*j66aE)> zg7RHt7el!zY;xXHH(tI#P*AG@cppc=lSNGzhu?+ixogaz^~M}7$Nk!W97 zuw$);)nh5}0jL#gYAl6;Qj)}Tp`?Q}1jTE0mT-@N&wy~gP}QhX`r3+^rV1i7)`nAA zWd`&ZoEqvfF-D>}&p=Qtva>|um3(!1P$s-0^p0u`8#3htZ>?IKGRrGXmCi@s8H{Zi zLZTA7!HNSE!l$4ESr18gy;fIj8d6`NO+@mAD`PLv`V^T=4}UBmZj_hImUz0$_8n8?;xE6c?=j}a8Hh|UtRnFy;w zeng6Pqs7QhdYB$#nU&^du`Ca<%uW*-?0R<_X90c%&R+CxudSS=6H%27f<%#KR>5G5 z-P5!KPY$$9MXkb#in4}Iq=*n7#4D?8RK8QJAvcJ&!jbE!u)9Xo;ni+oClOhCVY$8mwe&#rGJk-k74Va>HJkl@8AW12gyl_u z%qxeo#!A7WaIk|c=7=~8AoNH%wN2VLvlqJGNOg$Wo~84oo>SaaS`Id&WmWz z8a>bS1X?y zN3_D7@|v`(U(Do{?zAb~OG((lxy4Ich~4 z+4n0rq$skH;58nZN2~7?WDK70e1cZx7b28Jo8XUk7QqJ;nITXV0o!o&VA08-DB7PG zo}W=#LMhyx#c&EZiTW96$c<+Tt=;nV+vKnKT7nXugrB)Rs8^MnA=u0$G(DQYPx7g9_9=SsMK5hHO3rV(c zm;fG^nn&i9)*iX7-B}w+4;&Yl6jN|IqbF9F?0Akld&~OEjBf%D%(iNTA zc42f@giBw*n0#)U58~JZ$`v0Clf~Pe#fVQ(ni3-_JJH=EUwzR#CzhW+DSPG4u6X#^ zkw4feAMWCZuUd8et@7b+e)vP*dEF=F!#(`)sn4(aQq)<4imyHU?cwNIQ15l`dG9Yq z&u-y&Uw`|j{wjL5H+c5{JzI*N-OA5?{Ck)ETJ-D%{Ok=cKDHxzc3bf5SH`BIXSegS zH{SRw*QQ%F(fT)j_nYsC*<11S+0Vb`wxmr8cz$l>)R&V*0qf75=-iPk3R>{o%h!#C z=2Cp&Iq%u`M+Tpt{C;VtVoAlbpSbV#&nf0Fc=ql;iVZ6K^lLABQZdu~?CZDxoNAEy z;Ww5$!9W$X=uPi@_df?u0@=Rhsj0)|`uxi-qc^_#@+Gf-xa@|1J(T;AE#=ARuh%}> z#-TaU|}XMO;%K(qXAJfd-A2|4;WM*)+5`f~vX zu#SDVu^LtNhCa-GqK4&uz4akq%91LOf5F5INckQHylUTPP z=a-(6zp>+dUcN!ranh%Qd4$u7I+8BVFrb~0al(6^1j*=JGBFJ%!a?0VRJl49_^NOv z=`;hcK{X^XVfDcXZ+D_HjBcBs#3bSuSSIBa3xsUStMKo!^1AW>{ylQl^F#>mNfW~Q zCb_?wCWM`9SO}Y*Cqkgr6fC&*2#pWUQ$pj^`)IiZvGZ~x2phL3Z2Zi1 za@P=~+u%SLUv00pEO)~#ss+dxa z%^;M!-|CiMgBtBIT!5iu`P0|0@I}ggaE%K1!y@+rPM5)Ni*cTm{DV@_u|hBEEQ$6Y z;Ef|!xb4m&oKjD9qZYQu#Cl5k)%Y~uog?@}n&wk@g63TyuD6Jrc97M8jkSkyIGuC2 z*1!!OMdvuKr@%Jf<6a9ZQfK2$&H7UL8GNGr)A)h5mtT*6Nm_5fzxaHY|9Ou8c@zKh zll;$H#Xm=oV^Mw^el7nLexNqK^oLGu!&o6j(xhMXU%=i39Mq$F+Bt#9S9$X$LLLN*>797Y%tL>T8o&a zWmle}BP#i9dTgPo^)weVRb;-uZp}AtJBjrhT0WSC@72l~`^We=__*NHDlngaZO!L` z1fLY)pM}pgM{5(}MuDD!O|8M4e&3o?Tt5_(D23r?;dEP%!~Ahpj-uhNkEC$AX(8&Z z2JdmJmMpZjuZt6{qPY4jt#YDE1Zdk7E)kfyF|$|aHf*$R!;(ZBC@elp8w!Vza;9E< zu+I^9F}TiY7#$q!ME4`m29INIbz1zvCjuS9PCM+jrYpL{sS?|rW604xYju?vvQSQt;~-ln!94VXaC= zKDyj%=;VQsZE=0Kq`|Gd)w;FI60M~m`Yf$o75GHqSQT%FR>~Y6vgU9_f%>g}@!WnBREJ{d$Lfo@R;f4dz zRU~2{zQ2mg>Ac1{pUyvvu9S-z-E6OLTME`~S)OPMMXG0M%iy8#o)MhMdLjRrO#iC#E~E#}Tiky416rr2Gb z@O&Ej;Z?{c?lh+i2N*VUn@|=c0AF_yBlKrWV}u3RhZX5H|902XS60T3x`W6 z0LPo8V#ZK;&b)l@B6*?phgoXxU+ww%^HZf5ARFO$xput_z{gAog2g4-{_<<0j7* zOSm{v+~SoN*B;#PSs7(=Tt)sxnOwTE z@W#lgDGSepoC2}vUZ%maEXFw+io_aonFfNgu;yeI%fhyixe&Xw%G5czGm$Qe-C1QJ zciiU^Yy}icN4)&AY^+_WczhUVv}lxAND=EqOX#@HQJ=3>O=pofSF25;-K@z#$wua>bE(xSvR@ zp*@C=GaEl>%tk=sd_>|EmLxh#H(_R`1*M+I4B0}n=|MV3A1Cf7gIzC*<1|A(X7F9c zP0iRxI-7D+;4$wvaW*EccX@eS_1B=oi&X$G*8zNoF-sbNKf_|bbX{fZ2!iZ%_#=+X z!dcQd_M=fltH~3gjgpvtxRvRqpKy>fzr|Snrm%v~=;_UQV(V_dHDv2J~qYm@+x5iD& zir~7ax4C)ZEiZVB^|T`?eeG|I8PeeTpSC2DaCrtf6wE~@=D)Crx$B$LDob|e-e?bT zI?9cweL9{}oJ-1gdIY&PtVuY6o&t+;3!d>EZv6GQCkB}$!ev{CgMQ#H$9*ZB5>fEa z`mMaM#obu4OuR_-sgiFA@9JjpEt7nHWmm2euFvT`T(F#vSXTU|`Fg*;V9c7PS5FBNmt=MrjArw( zfRGrh_z{cyeaw&;i{^_1ufjvrIHPpo!*YJ}e)&%N zaTk8v_cK4ZA?OSNr(({##0f7BiyJBBx!OFF9+Xv^*5nXw$K@SIdk_2KY%)xvqGVT!aZ;$BF7jz05erFA&hd7E{2EM(abbHS z4xl9s2K#>+izX}Bx8}yVZJ1)^F@Y3`y!qOAo{kFi@@QKghbc9^e3jcq7fn#xV)~Hc z*#Ctw$C^d?mMw{-KA3cpfD1WNv0bwuA-R}kq_Uk2Za+=erqP*sxZ?>bNb`K$KX?Wg zIEgEoDBgYsw?U-1JZ7)54qmyWU*WZUPFw&-Es0#AykE^Dvr4oCs#(m_^=z7xv_K!q zHW;(1Dc)LJ5`BmKIvVo!BR>)u5xBKdOxHww$2;%R%r0(nX06S@zR0+_S&fS$Mv^hX zZ$99c<2fO}iWym`V|k}BXBw8b+mh%zmh10P01UDT=`=;v_2~4f#K;wm8Xm~wpOQP?1-Eqo8Mgmz|NWZZ)gy|fj6966LaEIPBsSTxXZZFMl&sk9j%c!4 z^Q!5<_iRb@T?g)nbYS;BB+;HFlRZuIr`l9CTq`X)I9s)XYfMlUi;l;X;i4$9R$Q~R zUvsKEq*m<4oSmAPr)K8S%y3Dyn2(y-a}i*MuIn&U*)0RZv(ac3?BB^);m+GQJ zb7rQ@){GCznciSbCybk&71Nx=^p-sG*{Jh2Xy>!Cgr~x~Qiru@%$x?RYfGZr0H)qGfaALu);xvjxMj2&?Cr`Fx!De8_xcy8&AoaPy5h z)PS33OG4Vse`W0FI!Sx&8FaSOwei-Zd%j11ORvJ{Z8vUM)}FRbEIu@uv-CcemXWy4 zm>~_UEw&_*U|Ecla;W9LZb3qF@%5Qi#@bSk9K_{!l^R`5h=Z}{mK3pwUaZlB#F0h3 z?UhdFB4bf^CxqdM8}vD#$J+RaF>+b8aX;*S*yg+oyI%|z*Qw2UZZrPjW18|)I4mMh zZ6+HNwOO$@sri9qnD_NTO=;O)jdX{f!;mPFrCyGD4+o+I+~1$g3E zIPA)7wQ)J4IvwfEQkN0@4dbR}MethonX%*qt6331#(0h4n3A>p1!K-M9Dmi8MBi~- zc+{Q70RwTs!iS8^|7*;AR%90CD*W{r1Stuoe{IZy2Ekw1lIS}GgLJefrOJt;J?l7_ zFpfq_IRmq1dB3JEt8?QDPkF&p45O6HsR3hdG>lf*lIS}|n-7$5u3RE4Ak1EHy>K#L zGs_E%;%&x_&f2|lIoz`juH#_>VE`r9F<4WQsP-DOr@^}0mPFrSy*fCOjV2DWM15wi z%1}OH+|aC0UXu$DBX@ts$j{|RQZlPvZp@el;)Av%`VQhsEXK4eSc88&Wp0-&WT0Mc z+?cFDtwR1&jGI{hO37;aL1R`lXkKAUqVLeGM=E>6Kkqn`?Mll>zA?w`;^SMdN(|M{ z8#gX1RO@p{MhAEK1amROQnHPH&X^+&vA5fjNFrttpOGORp08VwkX+0X;<26Bue)~= zQ+aVzIL?NIYcz@DD-yGc9(7o2zG5uk>{^3Ok<43O%ht*DU+1% z8*`{({J(8U^c~~%BgJwX(U}r1t8UPNIJmc3n>Y5V#0YL)(XZ*shMX(7%O|;0I%Ud? z<0RMoXd5GToz@!TmD-eu z(j39HC5f<_`Rk16$BYG%713R}u{JI^N?XyR?Tk^2LE;rnG<5`D+_ro(jTTB|Kmvf@2_W-rv_MaKEEmHqNeZ^r)ks1=EI z`kB1dlx-FNVvCeKJm>qiB>E2a)(1;I&YPpdn9m7kD9QzLn%GKODLVJ!cquP&SXt(` z80H)M2J==NUk3j3lK&`UD9`-oeH@phY=B*PBxP&FW@84m{(PM+iM~U;k?i%ULr3qI z(2l#4ekHMiv3E5F@jl}QX0^$igiUUEm9_*g;7}Q6Q}#gIZOoDe+d*3reTQw$ez%35 z8SZ6CnW?1KV6bY&&B_Ya+T0lN6>V8zz7(8R>S^#5V{SB@d|MKI$7#_)qy)1uM?TKvplWnrP)GV}#yo0Bf5nzW z-;v&X)SvXjdoOTcf^y4Pc#t#C5{sT!N7gW>&c;!XrE0@!T^lYl=2z2(f0-+7uvn9n%bL7($by8_$zQhJD-&!Idk(>9FlGybZkdoK_qdJ27L&QJBbGTNJT=se$~ zNtam{y@H#V(acnlO~u2;&Ce+G&Xv&~B<$4}*L{LXoW(;-MsC2Wv|UFpGv-Ny|^+^0UUQXz={BEs4Iv zb2SBdq8(YyT1DjWAYsutaqv(as+p@YY+o{NXja&+5y75lA63Aq=w-2k>Q<#?dVJBC zEe*n7vn7!kLhJ2yFCz>ChnQVt3>-KbwBBwxB42;ILvs%7!o}fa@*umCU&K+bTmx5+ z)0QlH4i@S-)YI`DaUmIPry~28Rz*u%!?Zk~az9S>xJns?bbSshIJZ-c|Nr-x}F?*VV z-DOKcD%dR0T8}o{Pb*FW4H!iq8BtBYhAXF>lQqBDguMt)7LQvDvsW4GlCdPRVvS>{ zxg`uM47<{je4Seoosq5@`jjLp&zMaO^D#l<(i!^e=}1!S!oXG_?1ckrlGd*p?me%!9Z~gEjMvBkcswiuqrZX8vDm%$q2RiKL|)fym077iZ7i^?N_AZhy?_`* z|CiiG9bY-CYt@mIl)X)VXMv{E1XUx4g(ZwaIvG6G6LT)ZfI5n z268R%LxJ&-AEgs7PV2;fV$6#s^gp&GAv^IOGH!Dn>{@%A7dITbA<=KZ5Lc6@E2LLp zoGu^im)E^67tWw!oW;OOiEsUzt)yil)IZyjNP=ZyE~UlgeriENa?vlZ?7AqpO~C*% zWN>7AI%2cdW=-1DH%+=O3UedWBrI zpK!IQd1UED^AzK;B+b&FGG*7l90xO6|SGg!*=uLusWlgaxp^>r*Awl z2of!VQ7PfQnZC@5_>{3cvnt|_*sP9P6B!krGW1NjSal>NWBWD6jB4ulN?Q_A{bm_8 z6yTqociFtl=s$7k9j>x_3d^^Tbjvf}yuSC8$9JPPDX~U%gCkXFiU6_5y$Dkl;*)zz z8mtfRF_uwQeYh3-`-8So!j0OtNHI9HD{?QQwkdWgCAa%s#{6n(@eW%ONwu&rDbgzO zD!c^=$;Bc}l2lW2prOmy#=0L4_u2yNcIAS%EaV!5?lxFd5W3HG{8dfO{#20Y>F3P-+;T%gqRzRdbKv;##IVt`7TbgQtLJ78qlvq-5p$S!1C#y0 zL22gCHS>jPRhi}wr1>H9j5*Sv_*YvrWncRzMsu!boSXPiG1{?Fq&U6U)|P_W4D}ed zZN^f{*iEqF!$*(Nt_<7`Sj!6@HRMvJpIeN1(vZv9lE{plb*OC(PHaKk6L~QO4BohB zO6zK=H5jhr#_h@q*IGKOCFD!o4VCC(DTx+GjJeT(x<`<>gw}3{GMeDt0dahEb05}0 zD$~{4Bt<r%#{wzNRjoEDg{-+P?ppWpJFHO88jCoqLOs|+p+XTw+0dbgM3x0aCPYMsEJ8&1 zM@Wcnhz%qp14N{FDOUdt#!b=`?-^SXNyW2puce31Ce4C`Gtkw{_XAa6$ zdzdENND~{FdRJrA{@S>KSy9`Rn*@hERgdxtm_r4&lvJy~GGBk71<$W>HUZvjl^Mc;HT{}mU6&hYZdK!ei%)Tz6?3ZqnUX%U!k94)$fdR< zG6QMtub(Cdo$hTXHOKCQmg`rx^Ms2sF%ZXDvqYLF0yJMG-?T-4tB)amx|iQ;#*FJoBe?`P8hOGsZk=V7$bZ zgzWUk8PB<%C#*d^-E8q39c%iyZJXu-amdoEFkY`PZdg{l*5#TJXRyv8XECr+eCugr zhBUCw*^=lxST~<+AY%!M7nrhnn966rvc%|3hW72oV#o^Z%{km!BHIBADsEok>6N^B zC2^k#4CRrO$jsY}+0+1ki!F&H;1)9>IhkUEZ9zhE(Wl9jxHRj)y@y9m?cBxp62o{V zYy=s9tqLpB?;DFZt0M8eTG&0}Hg@jfGp~p<#YUtg#r=*kJDS4$mMw`SR2FDUh2iBB z3lfrxuiJ+GgY~iZ&a^((d;P}RL%sQwS7#2w@!+8PsSM=$37ezS2? z)BC$7;Lae9t6+}8_^<5igJ~h&A!AlFso!WzLiV!F4BTA1ZW~9inc+O*jtGH^Ba6~u z>RpZDI%(X%tZ;40xm5tAaF_;LiW7Itm?aIi`)o<{9k#pR#395kHj5wkrf6RZa%i0g z>6FP43bf;pzapY1p`nYfS;xEJMbcsgXd8Eye&K0yZ+#dnOpti!h z589fTJ*%BN#q(NKuZgb_0mzHQ`@0W<=AHUt{6+I%=o4 zJFDb48l^T~r$~LI2=}##EpG^|rf0LxTALC1uCXApBI2lsxRT!}T}j&O)oISycZ@mH zAo@#N60!^ZIiuXm8ME#}J0EOdDF}0q6YYj`*l*w@>6zh;kL2dBGu*4!^=UTK*_E5( z7U58VMt+=HlvlcBiUO$kr+9U%j9Jz2UoJ@WoC5VUjJ7>=CC9Vr{E53ql6iv3LI&g( z<5p(vf2(pFy#_b=2_zfD#GTgRb{Vsx34GX=gzRuP+metYC&PYGBPVxz3lfqG9v>`7 zNLldsZ9yW4x4dtrcuP<1x3+0lp7M(mp2!S3imOCxn3YT*Wu10!#jIEPTTyi7abvX8 z)t_3L8eRELT6AT~_^X6+D3DMTF|yU_B3EvX$EbJ0=M(i z%-M6s{AwV-(Uyel!mnpN>E%Dzjx=I(9i2=!~5fdBHIq2B>hXz?tF>eA<{b4bD&4lIS~}>-M`%q)}3$G4*B4gq_n59E@Yw zqYeZ2E#szT1#Ue~I|pxh;*I(uc*{4>6{R;ZKW&-qo5uWTXnn($MBmXGJb*cG(?LdO zI#Ma+jMC!ubK`w$qg`h;l2gGGWtpZSlyYU@0%Kk@gf6!w(RYNl5JCbVbR*%A+*_py zGbJvge_<%a;V{Y3G3!f=W!|{)S$o`8fu(>F9Zf8A`yuz#1JM(id&OX;WD(wE%%BGF zR$CH%2Y4$W4EJOx7E?e?Yf!;$zX|^*^IHt@DdXm6g?LEucto@_u&NBU%D;kfN`m1D zV-7WpkJ^&xJH~s%Oi^sGM;bk)_BulVx7KL6SkWc(i%Q~QFtWzA$mg?N5S7E?DXWX!?fUU!A27)tQVoY&uQ%-F8j4?YJRG+pb(RZk_ zhHyGFRb<%y!nl!HVM|;;IzR2Si*Ff!T|?=cwj}zF(n#2$s4q35AX_jL`pxhZ#3qG@ zn%GZ^h*3PM5;vN2K#wtBwxM4Woc&Q(rCuuA9E_2ElqI0FIT1-xO>oLV7>kX0*ED8< zEs4Hs%=UxWhbRt2KXLT5Gv268I<7b}&mHqC*wBKLQks}mjES=(;>uz6D(k^cV?ku? z`}tfkXi2_S!F(4j$-CmPy`Ux7siU++ai3EzJA*vF_f( zVn)GsR^;HfN=~BTJoP$+-!*1Klk{)fl92u9HyNzCUh!Lx4bpxU&JUeycRF)|%(oVMc2V;gbz`kcoA~Rst=@6=AnoO}wowN#! z)HNIDW*((ks!n3=ydZ7y?n>jYYp`q(Bzjt`p1CKs>)}iG*J>^J{cb%zc~PFTWM5?5 zwyb?%X>KgyL9tV*X9?YD%!MZ5+igim#lO{-giLZ`oG;4X=##y?60;s>%y)4rUKlTi zdKA6d_;_4JW~#_KQ#Y1VR-L&@WJScdq?3V|OGSxN)^8_`nbK6KY)e9_P@ljXIMApy z3Au`o&G*Utv5sEMKs{^RpsYX*<_bxU6nIirU*BNNiU!X!wj}xv&-!C@;s^W?_}+*u zlIu-pB*gnwuSyKnr;Hny6{-!D ztc+vBSnQDfd&M1BleCi~j-kPO4C|j7iy$kk+Za}*4dL!UvEYU?4sf^%H$mx{DeI(P zH)c@-`m44iG6QYxbgn;i-fI+mS`Vji-^oTT zht!b$x|@nbuW`|XF^GlXt(I3PRsF}>$;7pbqQv@f#8^yO_2UkyA1Qt=HIsw+Q8V*O zGh+&pk`#Q8G25Dg9I_?RcLg~r6~syU!;ag;!f%RZ;P%@hHUdoWp=@Hb^v+{}FFvl% zEooMvy0IX$s?dE>g_2%)9`idX2Ss)AECxPPa3yE)2Om7iN0&peyLHs z4kS0|^caa8P9=~cj_c7}5M=Fn)>vp+wP!?X53dg?$ph)*5@!P`ZH_5T%3S;n#>{I< z^NcNtOQ|%;U0oNgG}Z)JX+C8vGz+CkE-q7qJH?rqS!q%t1RpnMUQ?Qn*pleG(%@P) zcJ%i-BO*T+Nl8e&Z+OT=(nAHa)@Hc>)VR4>;igO6m|r*}lunF%B5?)U zi5`62m^BT|uiBDGf@u+rl>uFzC|Qt@TyRX#f`sIP7d$OUNG{%EyD$_iV}Ib%E@Q7b zTAL85dObJI&RdA9u%4mC>>IC~n+ZBKF?Ah#F+LUOr#v!av+-9o6~9i9xP-ItS*6-Z zt-)eGXx!j*bEAnlxd^u~ZSFf_{B=#-FSI2gd(jt>(awpn0Bhz2aKDd zsn>gLN%UR4);;JqaV$MnP@Ds_u?lxHXsdZW>M+n>F>YE`px5W7Lf-NP!CS%~Uw9;C zYV~Dfel)ax-=1@{U(uH8$&arl-h3gh^ zg<^bo>zKUAy7g{jD6;C-&Co5;3b@oV>AMoOBDz?UTA?Uiipssym`P3Le%6*m-@!(= zZ`Q_JPJvbs{BeM)nT2FIL-*f}o0JtgG7I$0yuy=$P|AAfmyCJQ5c;AmiM}JW>4?{; z;`CzS`&6;6gH%Qj*}}=?@)_zeVt;Sk#H@&2nLEoo@}xD!qk>$@qU7Hh^Q0m7U0V`; zM{ek#KR%92fp88?ARmF@F)ZSE4eWVsQd$-56peSeEi=8gRldZiZoIlrGl|aDTruQ7 zui`r?_|HfCTvhX#GS6LS%%BGGYFiS0hj@_O;mI1}66qIJq?9vkcNsS+YiC@|m%H)~(fB{-pmnUoC<-C0%srotH7ncuzEZmysPa z7DZNMhm`}lu;#0_g%*OjEi$^OqMnjsJZj9RhWf*{B$B9GxWUqs=8%L13CYD0+c6}F zKYi%Z#-E&n95p_^X{T z90R=gM{60bkaFBI=*8UmKW*I9i~`0zQR0YGk@DeU|COi2GnS-9Gd^L=iYE1s+LDkx z>q89OTt_no_#T3abJ%lRZDheE;L(a1o^KjAC_6lSk3c0q9&!|UzDv?pZogs7iU!Z0 z*pkQ$k9E-Ys{L)hB2s+0_iR_z`r=T{Sd&p(aLwGfxSG}KT$LLO_{8}<)TfjMI5~VS zNn6Xk+?Xj1zklPtap^8+S#iu2N8F`Zv_u(8eyedKv-X#j@MVaXK<+VNr&+Tbj5*Q7 zeZ4IS*sBvSnB6nS`8FH)i7CPN3 z;8wAFSEO0KdyQGspge3#A_=91*C-uN?!Oi!Bp0&`kJk2g-H3A_i0Bn~vU7$!hIr~s zdL7oECygbXRe#naycb6Hm^aOvNeL7^Zp@FSG*g1aCG;V0c))MAvGe$G+UkN(4=zIV zyzL2K;+SN6g(3PWW07Qq=!V=m$+^fQ7h_`_HPvLN-fi(#WBxSA-fT-k_ORy|^|^LH zb9|TKlsheQ#ua%9q)A>0pET1OjO>3iZgzHL`36Jcn_>nE-x1ErddQL^^V2N&=ZyK& zko}A;i6pWX=2u#9HVGCaBp3a%;6o#oi5jB(rAfZT;WQR+$&R$^a`sR5B%x*5Ut?AK z2V)$vs@k@k8?}J%3E&pw+jY8AMZ_u_P2t>B{cmvQe61J$f+|< zBm{95e0)%j+y0F2Fm4;J?bn=Wi-;c=+(s)O@hY|r8yuWufM$k2GU{(Y-;m(zSLorxSfWtgEkYGU-mzGO589fTX~jEu$~5aW#+++<@=9A0Njh*zOg;5F|T z_ei%uRi~77<&PNir76<~1&K@OxZR;_E^yol7xtrCo8kioqE$EHcMJH$Y%29g-_Au* zVqm{&EWoV5-kDQ2D&$2w^JpjU508820(zb`h6jWyqfP3zz&|qPTLbTO!zqmLE_FGrTBm&Fr`_Cz`a6*piSP<{n!TvfDD|bI@%qNJuU?2gQPf zl*JouSL}(4_E`DDmp0F+aH!h!CdV+f3N9_!heH6xtvshOOU7ezglMXD%&ks~KiGmD zIuDC5={Y#>8W%fNC${7L;AXu0+Tul|)ui5oFdU^S{e-dFq<13{S!$}%-n3MuuJKp3 zF7xApL{9@X_op~sH&S&@VBg_6-~jv3XzN)Jmp$s8^N`5R z7tqJ1`RICvd?9GGn5|W)(Zbb63??=|my6SLiW{7wj`wg#Tt20{?{y4+uCBRJcV@`~ndyy<;r8;dakgP_0 zRSr8f!3{0i32qcyo)YpsY0QYGAIAiVOJ^(cv{K{Zbblwb)Cl`?P>!#DV>Aw@p<(zg zgVi>cQC6_>Voe8a$p<5^IJuE#{=r#|y`fC&>t390fMmyWII7+k$uis3bXs&I=k#N3bhTpM zYcz0d%K^+zox`ce*VYkT>Dtvn!K=ly5)Iwxa!F4Y!*;$M>r^9$8z*)P!`wA zIi<{Md%zgXtj6j{4rj;+IY_k)HZ_Wyk(+@SFu@gKhz2Zdc8=Z*%2Z31l6vrB<3?yo zeU~kXzN^!=NGJA30e1E9&0a}}2Jf*dl#E4?RTZ|SsDgT0W?D|-(jzG=ik>lx8qj05 zB>r81PPi%o?s2P-26T`280a4}7Qw{;op4YB+|?ePa=+G(7_+DW{aRZReFu8;OYat4 zc^^W~ymY2@@i@X+USMcHWZdYi&~DD1pAh32qSAtCX{5tpPQ`SIexv>Sjd{~B{Y6_6 zeaCc_G{ZQtwwc&`rG5@dwiu;#V4fRa3uZg2W{ZK;3dLCCe7ug>b3r zKrOn~nW9ezYK?|h;qAELY4(Y=0y`3HJ9Zk2EZuf!Ky~)!eAzOw9#x2179@g2EsLCX zWsP!KT97Gk%&vy;O|~Rt*WSuXGS@+-)hgCFfvVvrc9Eo2U>NT=Zd6tn*I=zd;xx%z zMRDL|X$$?QjG56uI$=v9Gf39Ub{o#rrrbtR9I5BP22v)kejZ=XT8!7@#_h|B*T!6n z`4e5PUceD00e=c;DaT<>8FQq8))FKxq3yi-3^qbU!8aULOpBN4c+6atL3yjO2(p55 zjo3yfcSbU&smP>Ry*C>(reX7(EeYA7-pKIIwbk4BAT})1;1}j_P@keLN-^J})4Ljj z_c`MRW(99kZi+d~hxr!Ns|1H?mrApQpD|`i1MbtdB>E2Ak%PWFQNyKj{^O$4IpWe* z57{%OY(Y6@aBIKrrUM1~J6kcV*p_x<4tR8=1EXJ(raTE#GksO54WjR1Y zLOJAF4z%Zv<-oQ%lmlu;a)@bEikbeFaXU1P`lc<3q()gxkK_!C2MY@ll8ZjMVe2^R zRlu}|OcBNyeR%Ithjnd3u1}Mn&iWjO!N6O-$IzaVth3gbA5CWlZAtVUtpj&^HRJ#{ zundQp6z=?JcrBhS(d-GwUY`w?OEAQ=D{jMiz^$}BIu3(6T@yo3tr{O`=7b_^!kxxq z%W8EF<|e{+V+*g^ZZWr_c5^Ne?miOj&Yz=pQyQSsZpGT9Jx7w2EyWXs& zzD+99Ecj>rR=$Gc(G!Z0UWJi$jT@E~*>ySU*&-W5U?U0lsjSMtAU+ewo*@v}LH*7;~h7_DVtG z5>B~R-CHftHXZD26T@R10wiD}M#qXn7?CLp`_uM;W?NzNh?zI~j#j74JzH6L~$6(4wqqV$8jPXe~N7TI3 zwY<}5Yk4m>{;H-}4+;`JB|7)XHruSknazs;B}4Z^qBlIS~xn@)L!HkK@rtiXz$Zs9sJ)MN0zVBEy4 z;9ZHlnt;ayaLNMsinL9-ziP~rhTLauN%S4L4S_p^n{h=BO6the=~;_W`)lLoWkqdc z=o}G$w68YeFY=LAq+PN8SH>J^n0?!pL}tva-H7$#YR`&04M!FS2o}6L@>!~JZ=%zy z5<@m{-umI@M=(D~TH86){~e7!hM@8U4H z2y(gwY+(_9$H%a&FELKLjm46+Yi+@vcFK1NxW}<_%sp>JlFX~HrTHd1j5*YR+ipuj zcC~Gc{apJdYeynM^tya5y$YlEpmD>pqPH$5^(!rvYXvqDrNn#A7&D}S^%7eWNw6#? zesb=}ecghD*XXx0`lkp7B&g|7$m!3A3dsbb#mT#g=v)cK| zN?mnqQ`SL#*qAR(rCu#a^fcT(=i#>dyzUgIV<8QK7C0v9Oe`^4;SfJs?PyoA^{~;z zR7Bi%e%^tsaQ6N8p_wYhYh7N zvHY%B{?y|`f8LmH4f&t5B_X@<+igimW5W9PAJUi{Jm@w2b77(~Eqc*GlXVI>mE*u&{JnDX)y4cxAbckxHmTzv@MTJkFaex1OrXO)_1Jk ztDQM&T1*{M{MYM@8BVIhv-2*ScNzUxzS@>n-}U0U!#+hq;~Ic;-AGKa`{8D-%~;=U z+}y04+{uj-zX@1zXJo{04Azu&&x6LiC1EX(*plcwn%k5sp~zP6PY&Wyl6ricHh7N_ zt{96TE5bMBn(|U8vfx?|mTib(ost^q8*`~)J#I@PiM55xC%r}<`7KCDE_e!RK|*rD zD{dAfBo}9FBb~$rwoQI){v};R9XWx+VJ7{@#bC%&Had)h)>U!XT#I&6(vloAkt*LtFd&MmEV#eox8Mi1qKD2h~ollEZE=r43{=G3P8a{u=!tHf$_ne27)}BUM zyu%ki(JUPYvQH)!Wzws#@HcLq8)sazKAm;BCOFH7S5k$XMI$u;NSk}DGiFGGV6`m? z*+o{_l8|F0E5b#Mk+XDE+eqZ5(=ANlDRm_@Urwq)ouM8p&%MS{%Gz_Ugk@zO^C6Gn z!B47*P4Vgv8}p>;&jDK!Qh)k|94u_PjWC?&glAqkd@z2@yXb7{U5}Bg8aFXJai8-x(%r;VGG0tS!_K*3)GWEn?VZ{m{Zo=UvJE!2J?^ElIT0kH{VbDuw>^9>}t?S zvDQ<;B!ZW`6G$7q$zXrNSPWUgzB$*XcE}DKv?IKQBd<0?fjecT?xV&`YH)wZmPBT_ ztzF*hPq%6aEaQ~Tf_50AGj#gygNb$54D}h?Zy2{ZE4B`X){ti%;O0b89bi+IFaE@s zGY!x`79@Jk9Ov9ezV2kBhVfDy!4ID&Y<<*C<2HBJ+KkY@8OtF%Lh$2>U$N9A`Hj$u zA(ZAs|Fbb~8b<%emW1qd|I3zybfQ^P`sqYlcm2U*nBG)d?MZ~po7jhS_rbUtrIfSc z>>TRXlxuZvGT@00`bCO*yfde4>&P4PqAAKvf<({GJojsni%%UpE!Lys@|KTT%%_Z- zmbHH_$u%aMVkt@Oh>O!Y;t69WH2EI2B_TWFy$n&WfRF7wa^=Xukz-Dc1EXFe4o9+( zL76gcOjb}<(XrRaI*GVZ+>*s?V`1Q=jpPn!K|*rTr|Ig-lMPB3 zq!Tt;em&vvCJR|X-eoM(>Wc8FF}Io`JS0f;oDI%-Xy?VJ5m2S$zJ{7O@d}BlwFXjnqa|qS?%Tp;Q76{_ zakB_ZQ``qDn2!bAvJTqQTJ$x3C7mGH~2FV0x4s3Rb2~Szl?) zx~2qA+LDmg;Bi|La$dx$_#bOtG*rH@0BJhQ)x*H!u0p(swdo^582suL_l>+X{)Tv> zRy1jHR<)x8n4g%o2P00ss@spPfzt5J0)FPhOn#d#2WvNA+Y9_yfDmW`+H;1(lwCU zoSMz{7(Nw%B(vH6&iJdE&HAn&ap?xGwFlt8@m`9NIt(@v2b0GFrB~sezWS!QnLB85 zM&1s2mQCJ~D>;j15X;jBjaA0%XaFp?B_TW75?d0oyR#ZxRCoW5tuhAP{S%kAyAK{Z zkKO#}+_-Pf^UgYZZi)Bp>8NiXH5PAr-=+p8`nJ}i?=}9Grcj3kiJm=r?iVJOM3Rh2 zu%kTZ&U@ClaTx{KS(@`gG`pHiOU9qoWa`xN9rimS*mv*4uIjVov`mLL3>4j#D_j`we7#Z=4o&Y&?E5e73 z#gtVM?vRSG8>h@uGxOBUd~K?lr)G-bI%;M<)=W(!K48qJrV;P8CDC_{*coZW9vFgp z1-l%D9?-1>bd9Rmz1qNDP*N2ti5qM4*BSdiHWo-$>~~7+_vD+@AVwU!S*bz1!8K@* zDA6td9~v{K0secoB$9wz_*T+A<1yWWgyiCYZ9@Y0?34e_?%Cd$zLsAbxn~16&&`y8 z8aSJKw!-*ZnnEoVBrc(QhU+ug(IpQ-TyMuq(_Z8;#kLd8UY$Gs4r5WIcYICk9Y=Qj zUFhX*Wm+(d8#M!mUGM1IjhWGK*=9>ZcCjrC?OeNf7d1l1u)Qcs4Et$gF=d5)ZpZqQ z#(ZiTamHI*ydiLWtUdzx-yNB6qqs z@LR@=X~_MCEeY8Pzrf)4@&>FI_Gfp{V#a><>Wt>!8@D(snil@t-x)KbA^2Suf3FS6 za~}rX>~4dsaX**1PFx%mcFHeKc#gbdJl^H9yujG3-99%{9L?-?Hs@UQsgP;0@uERY zEBo8@{xxXKj)u-kTN1K=EfXZ>JhLJrh>OLLB&A*~7VBY|-e4eaGj4X) zPL|7!2?H6joq8Wy0E1jW+-uB_hU9Kr5=kU2JRa#Zac{REA-R~vUfb>#$Q=r&--y(X z6h%T7+XcA^Hf|R(y}|laG8TSzeF}%#{O-If*v`*x89ZZtG<_PgC6PqZLK&n!vFBz% zLUQr*wqsb}gn#~d8|!mMFin_HWkcVm7!XkpBsl+vqZRB2+p$PRIEZNoNA~o zFy=%lv#veA|s%oVBN2FOFYl z7V_XY503Kz$9X!@uBlIVwi$DyLAXVbxP(JZ9%mi3d^(?jj#j5NlyEQ)c8ZE!Z*+X0 zz7%ZuE<<+ASV&nR8>Tan!nW{w5ZWR-p4t|Q5Q(;EUGP3*)-?R?wk09E;6YY_xgO01 zLgy?TL1NBXM=xf$9y4xGR=5Un)rmL^G9PiIAyPH|x`xPETM|h`EPQ-9P2hfQK|*pd zih!~IQfyp0>ZIUBO#k71QW>kd6@F+hJ|+`Oy+ZN$D+;x9~Wl>BKhean~$4W@6} zl1Rd2VfUoz+uggeFOj5<$s>zf8NCZ{G|A&vk-nD@?^VZ;r%bduR>KG=?)$5EN)`ILXob1c&dxP zS|06R!0C1L*OHcZzBN{>;i}oEx?X2FZ&dWg{R-?Z-Y#D|jO`*0{n(5jUT5*7d%lSK zsKOUhUqx8eYkC{5aP2$!l86l_3qfWQ$ z^%~9`e}HcQ3>@EP0;&m6HtQPyp`p4pCg&IQkn`gt(^t$Q)BHgX`(5aEturu`IF#{F zwdqab;AH0*E<>pmIoP|uR>QQhI*}0TtN(L}^)?~arPxUxZ29XU*k?(wKQW77Hy-q{ zv4SM)B1?H1MyA-t`9EaHV^;gW{^ye6?Lvl2LYuNP{wLd*sFaON^e0)FG=uR0lJ|$` z$4BVL$LPl=@M9)hvF@Pf)uSDnp)x0A{}~YZPhb{Sg{yK+N`1keHF7)_D!(TxkYNrh z@b8HVY+slH|DLEoCMTrO%l!Nv!So08BraT$2aK5pW??%Al~pG+Ebav zUKZMd>sGeYF+V{hIBIhO({F>O8K&u8(7)rL3?CAkV9OZMsK2r9`@W;8#zU>uwMCt(VDBXV{TSDNEC@RSF!cE{0V$Yn}uFu{8eohdZi$7 z2?I0nCDvF5PrQg{`!6zXV)|@f6KrzzHQHf*dD;&1JB``UBz?Or3E9zZWw7QtER(-? zl0161+Cq*2-H$D1V!;jHrLc3}{>*O*TY^+#<C1gS(jdiVZ316wyZF&&*21N zzYsz=9$H@ou9V9+o-*b~gX=YdL{D=#=V^|7InKB{Nt?t45`|w)9#$@;V~(XMb-Vm?Vn zFJ@Kwy0J{Ms>)yv;m!KASPBj}Qq&~ns+F%Av!W@97PQ?N?86(wiPj@AJkLE_R4 zl-rKp?;vnmgLg;w7}ctL3#N|?_9*KKGdElZFC4x&DhBT{RQDLmCaay?maDXJCaTf~ zu~i^w0wvDiUUzPS3SZhVd&rnc4Z!`jBxGlMku3>1%(5EI!7ytn-+R*+=l6a2{nGEfdheZg&iS9apL@>vDJ(?z&P%>aZa4PXWN1pnP3_CpD_FWr zU7YGF<#Xlyz+ioKcWV84>o>&8S>f?uD&!t*t7u(-NF#}<5YLFijWaLT{+3h@Mg)nr z6sl#6m(wX%IqO+{U~D5JQvPG1KuXPxHn)Zg`clW z_T_x1HeV8Me`AW9sl)oJ?QShw%?sB6YH!G8H5aZp1wnsf<)W$l76bMbZN;bp)|I9# zY99Y$A8(xhAcn7u))%#zlF<6RDT%O#{R{&>(UV`7?xzIW*fhF1wXce8VVUYMEn*>5 z)4}I8;DM6r+PCN6YN{pCLr(3iX39t0Hdvn+DK>y-v=yal02ignu^N30glraKI`9ld z0H>M|)2S4(yBjf`u)Dio2sws9{F^qDl0p2;l!SGIxL6oO-|ECR!LYg$dy2=mh7+K} zrm*n*i5c!m4Qca2Hif=+I|{`HWFwbw=U3p zml5BrtqRq`-_3|;eZ-3-i8{;|YBMIme1Rzm>o6}BFy{dC{tRY2#LU`|Th8W-DXKB! z?fG~QGmtlH3#$qwHa<};p2HFbcolPQ%&XWoBe&f*XmcVVcdaQ2>&R^u$hkmnYaW}3 zC@yKHjO;{pWL9v|$HAlk=HJ2&DMB#9&s{hh#!)>HLX7yG+DcMI{9;BN*;b-5I)RMB z20|Ixm!Dv=$B4*H? z;!X7_vDi;jGe!3}oX6tkF`<;%ULMp|qpH1J$@bFkWD`noP((r{MC`=Wof zGM~TJW?C|zx0sT!Za!SI4^V6GZ`8vmuj*pwp1e74yoVX&C$xoC1=(u`2Ew5?rVokN zHklydr7y3|q_Z zwH2&tEwr5O^M>T<0dGCzh^SAg zM6v#b#h_uDn?qtq+M+YMssrnaM^Wp*k0>xs%*8wuJAkq+pjIMDzMAb zHIHWj9P{kNd6q!TYO^DOc#A2CB!~trZrEt^8jAsmkc*d?&KG*?Yae@d*VnoZ(UwHo z-{$XB#d$Trhtxb{S9&Z)xFSNfU+1VQeiMFd~6Ye)R z+uNsb(BB6db0n7Ta`FC{ww#JOZO%#4-jCuiGGBQ6Ql-NDVeOAgN`B0gL|Ba8Z%QI8 z%nbDj73SBS2p336_wi@9FkiZ(=!%p*)V09Ed0(b4FLsz68gwh@a8N50>~KYIa(gs% zTfBgOCn(^*(bhq0VRji%*qB=sA*Fg#LO{5@DmTjtyv{SJvmEs>B*a z9R*cXoZELM24SrpU_fuu7D^S+c?cZmb48=6p*K4yQ!F1S^gngJ0 zEgOSqo+lSkmXk2Gcz|KLQ(Gu?m=F%z=Zb?UCtzx^|Ko+)>_{*@-;{)Pn3hL79@`O9 zu9`dF@T0KTBjkA&7| zO-Wcs>k90>@urNaeyWoDUp?&lglg&z;%|~uz5p1DYo$1@Crphk;FsF!Qni39XlD=0 zOX&%Gd4ujxC~pwFQ;178BgBpMe_Curcv_ov$yol?l!SF-={SIaVWpIU&A@{?Q9CDZ zGL#E1ur)%NorY5%xS{0(!A%UBsjjEj&{bfbHY*Y|b4*EChh|Yv1Ks*cwdxkR-_!54 zq45IR{xOE@5^VuhOWxviCeC5dUdAinX)R zRt_N-Jl%0ZTs&<$>Eg{@KmP1acg+`b*Ql|?gv)c_OGR<0q7$rze+{Ow?$9=s)Pt;#|qXT?jy0MI5S16-!jSxSV{KtnF`A8GaEu!Xi+|SswuHq~rX-ZHFrLAm+Nu$02EVfU zPun;#eP5|1X--SS14J}plHR9le^k=@6hWeGlQQ8;pEGuqQ?*8^gpEq^RwR0ZRqAL< zr)tqN(Q-IG7m`-jXn$AI>MBzbp*62CB@uT17~2!-{BfIQKqBOVJ1qtzLR}m;tsJ~A z=qJYNf=>Jj)AU0@4CiP^sqnj?-&xfLx3y`LVB_^WE85?Z%(-YvB7`t6NKANjvH{WX z*d+ynVfQEn0pq+Z=>GG>`GLk^R^ zTv({13^~YI!jqbfKrUJcs*qe#P4+5n1|{h4F(r|N-k<{TO7ADfD!pwj%DB1jA?KXb zh6^^Qk{-9T7|bYir}iKd24%+KVdfa!wZc?HmS=nUgtjhQ+Y2dVyi)%$?QcoYe^`)c z+clW@dUGKqp7w*T_vfqF#Y~P#@$%a4Ay)DCw8d6bF-Dt|PMWx*@+P=zlR5gk+B`_A z|F$WKu+)8v(VOVe<~-kLE!7U7mQOG`lQvCECs`^0&G~8HcWq2)PS$2Xg65C^W1(5J z2TRTr=2gsmZNpBe1_u9oGJWv^)c!FBsY_c&szqoqG88h0Vhx7&Y7vJ;aOkNyoWNhH z&4q;2a#Ir43(-P=Q$hot<>`9_KQEWvVq%S@-9wDh9&O=NQ93;hU_$PkfIA7JUD`ZI z80|DAVI8Ac+c+a3Z`X(yp~O8#WJp^SRYW>~2=)DFUj?>~$c3e({b>n^f+-2>Kuq%| z^y3(WKePHrwFOXB|8$!> zsJf22tK*ocyufHZrY*E8TFVeAh@*ei#6BdSYwQ^In3hp|zcx1#ijSC*u#RFdQ5l_! zNKRjVN1J_BJjR&5d{XDaMP6e47i#Nf+AcB6lL@ znc#@RG%`n0b@pGT>I}o6{@#>?bwt`s%QYjvGX^WHVK4(tiJ#t0QvNdS&q~E%sVNET z%0I&|4z90_D%NwXQ;tF1^?GuV@^3r%1yTxbGbA8^)09&c}a z6l8%!BoSW`VyYpk2%5eVijYe4E3_GxjOb;iB&-{eWI>L1jpSM~Tj(v5fk#Dg&eU>(xk~u6f zC1Kqhdd$rMyKgGxdcw(e0_d@EY}ZzqYVEMa+&E|#G9;F2j2>-fB@?;al!SE?>GT`u z6w!6$#b^3@`%<)?cMN?Z4 zRVZe~LZ`_^<8JLQODNoBO2Rq{J9>7l+q*}sW7kkH&{4vaCGS=ot~4;EI-DBJ9CcHJ zu-06vi$a(j8$@H8t5C!SBmj0Z!vhh#_J%9q`N z?3lbAc1U(d$Sxiu(Vbj{_AL%0Yel9-Mf#97zmm1Q)0BjDYgv4lu*ZZ*vV(O+ya12! z9HaUTZ4p&bU7QvmVX7k8J(5fI*R;8iu=TD3uTt=?4T4DR-4n;E{bLN(nHSp{k1S5R zK8Hvk#~cdWigK}9tj&dl)Iw7d){$D#Q%*%EIoFl)Wo+HQl^u*Ij0Ln4M@{V&MsSO^ z;HrgfWja%KqMSQX&M8Q&;M{{E^bfmCnt2{tnXHDFXt zkr!#Wjeb$4IE>wLaX^*dVoY<|;;UlXmChgwPr$$9^H1*VV!9yUUxGEG&6EV|&88%* z!@9ulr({wCl}Zj#`weXJG5Dn zKz-1Zgmq9A!Y9Ypx)h#gAit_DvMP{9;g-Lw&5i`(6Q(4rgV?!$xITy}buZ+!6pyCG zeFo`Q+9IifG&emYcyb~nRdoC6jxHuuI z$5oADJ`?Xps=UD9U8*g#YUw%~v!6cKPSwde%rUY*It9V8gyF^7yhs>sG9_Uh!xe2Y z>=psKGAXy7jiadc3gdW-w&1EbrrO|0alA4PF)aaklQuIFkk^}%uny!Fgi{+DPPxY^ zRkF8JYZtP@S1D`@<}VQ!^SGGyK8l@ED4y6EM47AykjpQ7Bm>9EILIeXjt$_&+6q%O zfF26MH#Cd@d@Qk;u+IAulPfYL7C*AYnA>KFkrB|7ShA2iw7Hcmx?7%D5Qs)Sn$hpWP4R%p^*RbeAFd zg|jE4%W$3U^ONq=A(OQC}Bbq=hSExfQ-@1K49+ zqu9-2k_bL+)Ii0HgyK64N#`Xr1}LnVcG5|!?-;#pw5#wE`tE7)Dms#{JHjPlv~U-G zyJ?hJir-uy}0E?V@1T1{_d|-L@4=#S>z7<13vgi&|)rLEU zW~!2S?Shj|K8gOp3}P^aRn0UurHPJuaNVH_oOHdTkKHZQmA zRtGTpU7Om9K`1;@b1;^%)8_BE*G^|Xmcdt`7%=y zNj!DhK#z+s3LSTw3`m4rSc<4KyT=Qq;|C;+_5zyHUAC0RwUw%BDRXG?mSQip50Lm# z?T<@F@gY+Z*5SD9@ZfN24|jDB1U(}77#_)__Epi;BXVU5ojt6}AADW|X*@aHPJgD9 zFAgVM4o{F4L;XW-RjERKDHaTeTlD@QDkfT4R0o|LvIQ1ZG|()nTvvQwn^DOu{?U|# zb+cF*u?63Kp*uy}5HnRym6^#W)ATC7!!S>~)Yd}CO6+M4diFl=PK0$wuo0| za~!zvX9i}!wg9TYOvffUp6ix1zgPRKlIE{6C1G9jUC*Hb9=gKbM3I;s;?~joB~u)5 zu}B<8M*THLuCA>BRpeHu2Ur5XO=Jl~qvy?$_!3B!<@2<;l8`N%lCX~KB?sJNRYao4 zkd$Je_UG&39A1f8H;0IWsSIHDVfR!-sq1B@(-We^_}{0kCRO||MV0EwDe{JSa*`7& z%ZVHXPR2J5PtNu(s;|{%R5FQIo070@61z^oB)Tzi45P`p^-Mk)^>6~|vblUjTZO9T zvg<_51yih5a>`>BNY%*)wYipT=TTD<)@|oct`n?LPLvp1$@jFCrD`Qt_>GE? zb|UJP?`rcZ*~Yg`N&Nq48T8r75{E?$YKZ2*Y#mEk6i?538s`dqSoc>iwns5*#Y;8qbakJF;_5In5X#IjHp`W{mf|L+0)n6@fZfu^p|db##}zxD?u z_#QDOVI98J2Pw7%?^2*J!3p1AuH()+@DDpkB>pgba>LGN2E{`~O8}$d>^La}j87 zNCtO@Hd_+7TTMw=1lRLUo^%K+5^>=8C&r{GI4^tvUbj)-;wd*`(f;Q3aj##-;Ntf!~?8)anuP3G5*{2rA_$CpXf&gP|Q za17Nr>z#lZsol<->LRT8`%Ovw$qnkSC&-}QWvYQz2DN_vO3$D?_Vf&jU$S9P)U9t3 zH~%fB!iN_0CQ}lBatr$Q1X<8irW$BvK@Y4v$Fm@hJv|HJmuy+kM!8Y@mZ|We1%1Pm z#Gl-PIoV-7If#8U7iJb?CDt$zhujTE|e{3hpF(P1#LAY@h7*S7fz4`z0_0#tt{xN z^{YJ#^4Qa}Ab!b~1zjXt(DO}&4=rfel!QeK;!T?GMu8_quwJpsm?PPO zbwTk7Kl*Z-{o070@0Mlq_PIFOlOv9fUrf+Ktpb8T*chYc{IFK~|E$y#L zn*W9=3G15gJ~&kL3Lotu+)hCw5xn4#Ki!4RV5vjM!q#GUq28;E;2*UWp^BiBuKAJ# zyACPfK``k>+uE?n8u71bpQMEHe`<3kq5NA@64p_kzT>!C&E~PSI9|XazcW6|x7b=5 zm;q;Zp9e|#XKQ~}QvR8yB&;icy|4V#`N+WLvPX1dD0LJu&?=3Z9~I3lAI&4lkBFTd zN6CZ$!?{yiDXPV7eE_F(KFueSn>Xf^9Zx7P&ImSgM}(4K-=@u;1p5|K64qhAB!GSW zW~|vBp~VA)<;OOEtQS!oB-RzM>{+Z-#lk`~hDCH@xp_jA*boZZYEm_XOF~0fkInfc zCS@O}I<&k*Vv41QK+JK1L=0&2DOp6$l!SGQI4`h>4X6mHwPHJ{3fBnq03Z(Wcpbxi)hW;3K9al7QX4X&33lIH=O4hecA}3%#_W^;*83 z#h$K&8{ikZPDjCKkQxet1@EAmnez z%k^ixbWZ?1HjuU2DpRcuwx&_di(VkjsI-bFKr&-`WsHna>gW^sOev_Gy1zLS&CA3G0X~ z-En*fQFVD9`e>%u;H<1PwndL3F-bYz!;IEZZDCc>IvbtvfLn~e!L8_c2i$s!5Bohe zZC)hohD=FV#Ljc`N{dE!*+_>;tpf2 z9*8#kzVp}vrg{z5xK23!LlXLGo#p{`~=sE)&RSVuQjs=U}M5thg^eeX!uQyd) zxI#8+O2VR5^9tFTu!o7Be-V`v1yeXQ*r?WWNO5`49YWU2KE#kb|AM$}>%7P&{$*_? zsN#HXr0)pdR8I|qjp#|?KnkhWdP18osn+_ODGBTNc5g?Y`Y4T$_w6~zqlk=% zi;?NeW1eU@l|%Qin(u4KPM>@_0|>jwQ=8A{5)tAw-({Q^Y_&BG z?8bJxKo-&8$cM5p#w-isO6BCjJXf1B3Fg_RB&@@n-kUj|FEvUC4tT^Z<_9T*+YXHG zFmfixhOWJPb|k=6dV?XoSX*q>qPrZ`v)~(XYka;POyhcdBh_$=#qmwr>_|XvFePCf z$ay^o)_N?pt%CWSoh?1?TRy=+-J~s+Dp2zgcoTfVT>*11?Vazh*JeP1=CCOV>(H#( zOM!v= z;cN)vMqwhYk7S?_FhT$imv9940&S)wX#d=lgmq}=@68}SIyNq)_9HNu*KLURYFj_T zfW1vyFjc@7pn_!1q8%E!aeF|U0|}iso071O&Z2D<#Av663;?x9sQqJ%(igM^R7Gh~ zIva90Hc+3{WG-W12re82M@KX2Rp>=RTx8Z9mBBvTw5GfSUS?<)M-D}{^*Add=l+UrBZ}HfoElTJ)A-Svk?@PEG zG$mmjmvup5@|q>~9?9oW)M&nB4;H2>sW2dS95hF~9n_EGy~}vsrmYNBJe_!pC;IhO z5=ES0;k;IftdGjw$Xm5JlaM}UO2RtQvkwHI3e@6AN_QEnztR>*6{|Vv5E)T1%5eOp z_QxeS-e^j~J{(-KsEoCOBYBtM_>8tV>Tq!J2!_Hk9G}$wxCFw8A!;T<8$XJIBctaq|Jkb(GN^XSjT8?#HrEOK&H)@P)^=N6_IZdj?ORQITxMv0U&3XoDGBSiY>MEL8A3$%Om+~dce2=9*hibpN<$6y z>=3Iim|KsPqcNUiy#85RL{+?&@Gf`e5v%;*vHcq3Q`&4uVEvOR z3F}~W?&M6vB2c{mE8Y;bxX<8B-(jokJ{OT9IQuask{KL%)H_A{^Aaehnv$>xis!o9 z5lrQQ=MAW~OXswMP9g#rabBR! znS}FPQxev3KBpDVA;f;Y7cai5FEO&0Y74Jgc+Y8tY%t{$RN8W{^lFF{i!m5Jfd*~+dagPyt9Z5||y9xx?g9it_O08L*W zIn({11r!G@=T`dr<2a4=AY=6SuwY%o(#4Pu}H*QZ1UJH~?y-bLC%sus#K(wMXexpat2$LCV2 zU^i;>A>p;&l!SG>W*lskN`zE$20VI$@wic22vt00rV#{+xM(@^eVz7qCH+6wl!SHt zpVw2apiQWz{OVA|^*hKBdiS`QY8m^BYiRw3Du{YnsU~)htR46DRk4>D$=kKnp^D_X zbcW>#2L}|puNH|o?9|W_2GU5S?KW-RB(!feC1D-yWz=t>NZew;OX2Uc)k=*zqs2TT z7-OqzypmLSo)LYkw#cf8E>BZW89Za-IOZ0x2;xXQOCbK0HailCe`!j>I*5z6B`_l# z`>J_2F@zZ7IR@?X+9Ilgwj`azh;U;bC9poD&4vWlCrwFM1k3aEI7dvg1hp$-T1oKH zUX*ybQ+EjISsw7=y+d3`|QRzc!f zjNE_jzN2n6Q$@;bjO#P`5}JvAF?AXQN5@s3)f}%7wBBR|y;)m9ieNOm)0I#ThvcAO z5;TDt$bqYTO$3Gamw`j%{z98E3As0zk~kEA|2hVLqVoXk@CyrNdXmBrbJm7RHHX+% z)nNp2r8EIGqy&UJ3|1Tw{=8PB7~pa6Cx|AS#iz7Yr)m~EF!RbW{*e!gwjFxmvpFC& zCx`GwcZBGUCptYHi0*+M+RNkG9831{QBxAu?PWt?FGvy9#$Gs1LKsIsZXHGx*cSdp zTP>=#K=GVCTUg)D7C78OL@WvUA8PX_A^&|-5=rF47JvL!&EsNUP}y)xV?ZM0;+Ux* z#04?~WuDzm!_LDZGtkD(shyM$WuIT=AsYtfP9yPK#1di<2!x|dw2AIg*c2!7WcVMxI$Z6Ma{NI60}k799*jX zc}dw9kFD$p&&RZgR=}x8hbU5&5|TIagmGBa@ zZ_;K$Qvda)B*MDxuqlbK#$_Wop&ECS;9|nf>&=0W>2 zrF?OCbvLYR$i>=iJk3aVd0yiZ#JswS{n%rgZNh^!y2 zp_h?;uQpc_vhOh^VIA3pge(R7URT7jWGaaG9bLC*wtI+C`v+~|R8c#f_mop8a^gym5#Q%Fhr>?Rz+G)mu5QXrarTsw(z9pt4 zti!kZUw}GK@W+)}zgq1n%XgB&>sb8a0$xWwI4GeI|T%;z*GKjwMEcZevsub?0o%)_GcyK zKVwS5y7E_3`&@DniIvy>aw*{*BsyUEdMzbV2a2jWUNSV^$-K}baEv*8=xQ}uPlZS@#5C@DMl%$EuJb$ zi_#_VC!9$TvYGhvhBtB}mCc64qhToDhl)R5jmaNJq3)p$cg%f@@e4zC`lCTcna=&+tYQ{a9L9SP9M2x$7#S5Iu3k=k|wS`s%DjoF4eXhj>JnD3elGvLg zh(#CT_zrD;Bpe?!C1D-M^uA-*3gmBu4ZRqm%X4ucmEK?^zp5>^Dv~SGm1EcrM6n=f z1FY&2%9phnlAwITl!SFCSNG(^D!t#|%oK-f@U^71RcvX_muZT=hNdyO-%{f}hV@t4 zDo}-WO*$iF2>P2WgXnnrG9ZO|G)`eyPJgM*mIUt8rX;L`+kJ2dInwL?GRyW%Jwuzn z5QKn%C6o^#k#P#X8JT#2)q9m8UbM&7;KE7Qd`ZM23-L=^!~%F}A*K#V93hDZQHb&a zZO$Z==bDnRj@|rG$wKqfZ3|B95xsD~#wZ z+JdVhx)KS2iF1cIcc|43Id?+NrE+?cHb)Yg*PD{Cj^-r?+*~6|c%tu^DmO}fm=(ZQ zhnzc5b>Zob(|(-fDy8mJWGg}HXx=>*2l<33G4?OkR+B3Bm%y)%wetsY>AexWUtDn#>mt8knvsnYwo_V*=R zerig>Ixfq0R^iHsVE)BnniJ2ZYFU^osz2X&6Eouq&ogTC_u3kMEKj4s@_441@*K*Q z;5l&WYcT`Vsm+cAVuvXS>mZ)(&*Y)gLLJi}?L)%yGY5 z$xYRD+8>wTc&;f4>u_95lQ90kU@w+XYboC3kW1~TRxw4M37wm1n6OO@_K%~Dj{|oC zbQt2>wN<1F@#Zv*03o9o6wop;A~cDO9Kkwq=ps!+1fC8MGD>ChHf=T~W4P6ngmq)s zCHGr=Uom)%)sIz>;{n^LXh$@%Sa^yf|GW?C|xzceLb-E@{5s=yBNmGURy|2aFJ`T0<6HLBe-+~la63gD(RomWw5VH9OA1-G9)xm-D*k7QnJk?^j9h*I|U6Z7D0_mNI@Imj%4<8~N z?&7k`26MhP!;-x)U>59bQ~`Yz9$V%7P@|=HU7KeZ&TF+rQ-yOOCgcQf7~4d7 zlS=e8+AK%_9WW(f9iU`#!kAZ9@+L#_d~IP=A!(PnFkCAh*8aAH#Zgld*0DHmj}&Qy z{9(E|jbYXuqVVVC9M8v5=P}+Z(SM!se2cb1RPj774PFsDCAu|Uc=+V0?I3{=DV+u4 zj;35YzDb)o3Gn+(NmvK`>_ZqAZp9{Kgx(xPSY&EIX^qJrIL7fE_i2Xn)7s*y!kH3Y zk(uhW3Xg2&$$etV4h{4Y9k?fy5ZQni7*Q@Li*I8p`z| zB?B4%aR%!zv;|fLYnkXw`W*9&3B1sv97_nkL7N*1!Pl9Ru#VsodGxoxTInmgCHA>^ z?Y$45B>vqt+EsYTWCVd6>kWqQacyZ;;X9+n&<@^m%&r97N45Eofcuas3G2Wu+?hY_ zPks9KEUo^wdx&xRp|)_UIGvvE7u<=t$tZUcM&H-wLBi-CO-WeCXu;M@EuW>(IfK+Y zME5Fn@VVRLZB?6R7@=wVZT0XMru&FD+A}&xy*u#c)Q;xlF2rPQ79@a9F(qLgprzD) z0g`U)jM$0Ihlnx`yn2cQl`6JNBog0__b@}&tu3r-p<2pq*U_AlK`cj)Stk=PE0wL) z+N?;>onuPEI&^b(WGm%LX&C!KZf#Ml=!;yzU?DF7JVE42fPj zLqZuA`dsYiLDHVOn~V3;)m~xTCjFVMZu`nKw!8|T*~Sn%ehBy|IeVXrNn4L|3C)wW zIg-%)Zn3AxL{<1O9Zq+kV zc%E?`)fQP5*X0;R`W1`l#YS;#*MEAIHailC_n4Bf4&vHq8TEHVivh}ZtVj-&*X=}# z5%PIxg{bb!4DVyw%20)O9pDXWoS<9^pfR5TX`FtRrfqfoer?_)tRFEYVIAuwkpk=W zlf98lqEa5~LB{Yq+Cr*gct*5{a*^~m0kx=LpVa0X}pISuNBttI=cm4oWXjC zx%CP}==Zfun-{4}Ej1-!9lK3qW5==au#`0BZc9@ljP;e;%2BO5FB}JJ(Gt?^`M5hN zcWCn{*}zs)64q^C{Q>tFRt5bv_eiUV60r_`p0bU^rx;8KF!n`lrKn=R0WM^oRILUq z2#J}E^o`op+j(vNB;5N=Nm$2yK0IVZ$Tbl`ujR^C>qi*7d$k2q#clyP!-$Y-A~Igv z^{rQEb0DGfGE)-P(U}|W4b0cYw!g$azg7=0GVjwCN)?&8kv)6)xs#Vf+ZJx1dnge-**OgrhM)pv{r-eya0?5h-W zwc!#De}3(4@vEB;jTY|0*KQi+kB;|nbeq}Wa{li3Om+AuI4xX)Kh2%%uBjnW-MZ3X z0YMnt^14HACX3zd1MBvRqq_&Yktl5Lx?Cl@ZVztZ9b%GX0j;>>v(mE z^jUtFEXC-BNzqd~5KgAWTDkgbjO99Q1*jK9kpl%1i0&5yTczlqi2qnAJ3)q58Q5-Z zrX*ljo06~&>{6@|@ct7YT*el9X%y?p!SXl{Gl&PZg;fRd>@W#v5=iFOQ+zmB-mlGz zgxy|K64tSsFJhQrDI~IiD1lh(M;NDL+JdR#w16X;Bsr7NscUl}q4PXb64ud4)O~xX z0f{toidE(quQ7^m)K-8hid~$Q&@XAp7I3U8^FD2+Bw$}_N+JoYPPfkEV&7EsIA4kW zZefW?`D;KT>cM7+(yCKFC2M(jB zoZ7HyGsTsx=gChSoABaAnSAW!?JU%p!>&C_UMboVH#=C)BXVcFN-Au}=UKVm(1y6R z>5_(;GvJLdSgDqg%zAWixm4ABP5WDtMSew)XbVMGp&b^cP;~AuQaD1|+DyY2fD-Q; zx46%W{da9)6&2f@n;s&bgg*@NeUuX6(Xe>&KtEqFTrx1DG7_Pn9gv% zNXYW+&v3rnREHBc!?`Io!?|_Vsa{u$Rie|6RW;BPnBlzLRA1o?=WV7WESfFPaGpb( zv_#*{-%_%RoW^OgiMr3iUj)fV;@tt&m$=q>R9kpeP*-84+RM}zY$)-$r%kM4eUiAB z8u!1|=1HoG9yTRm9n-TILYDwMfYpVJ7mq=JnaEc=&a;f*x3xu8MQ|ym%E;VYf~y+x z+Sax1TiUEhzq}>1yKcKR+>BU9w(j( z!$s2nrTt|Ig?F2hu#SR-1@6Ds7FHE0{RQs7)8<9O?yIIGtYg=D^*YvJZ~X|DsNZV~ zrizp50{3sUIgrr#l_?49=p=f4W2{ui5L17RQC#|5Tiy3Wk56ZT`wVTSBw!brlCTc! zJ}=&<7xtURU`UFBK+07teYFX_+npSyMO+=?ZQq1bXEWKVtxna#zR!<4>gj{VdZ9kr zgzHU?2FG+Ayh59I$(k-TC1Kr~<|K!LadVJwGoJn00;%HJ>A7`=V>`KHn$`Zggvc$X zBn({XN#zATM z1mp4^ZLw5wnV%kZ>zL6JeEHjaWoX`|&42{W+f7MWhvu|HaPgp-NICj-{1$`pb!|aZ z!I&kYQqo*w>-g9ITl>or3STlMVI74ILV+6+8HeOeM&j4n!l)uKTOh#`gsrjockORW zSp342gmo+yACB$LM1BoFFh#t|8sj-eXvtw)J^Ll;_+HCeJqR};S6`=Vvmt>s-;{)P zux4;$E*F~U4Tj@#Z6Q<(%*-^m@2v~VCEDMW^nZ~l3G4b7brYw8-6o|^NZw=|`m}{n z#epj>&iUFJi(c(-OIX}!O2Rr8tNlC{zGp_QQatK!dqLz@IB|M0?&B4i#(Rw5UD_&8 zMR1Kj77hI~SO(#A>ALIOuHVYC!Z#5-h9iTNm@$pS}s$4n7YgU=mP^CDGklrGbS3Z&WQ0GO4 z@+;a(P=#_0_VZR~l;)3y90zZ?fN)+qjO`ki$SGB-aGEP) z!qyW=o%AOWtsijiYQNH)yjf8OgP# zB&-|BoSs^(Q5q5fAdrM9nIl)e%@E$HEs!dNooRTA`W(UK860`)3`avR)c(4J$n#A} zSVv^(j^jh*^i7a>=>o}t)(@^5FO%av%xFEREvzb9XCp^Kz^#`mklbhkZYAvgTALRM zySJEFcI? z4K_g)-T=RJ4$-vrx|FwpiP^TXfoa1y)+f3Kk;qM?8@V?5oT=F1UZzi*lCWrzyqBpv zi2g$hVdV3UG;TD_l zdnkz$lSHa_ey7cuRPX%Sl!SGZon7Uei{5?CEz@FjF13r!DEtE=%WWrw8$j62A>~It zm&od&`7Yy}y3W?{t{c0b0$IeCjC?2yW6ZKR5JfJ`XK6Dg!MwzjgmswHdo#!LrA7%6 zJdU`<{9vV$OYK0I56pc~XQ6BFo*fBrmEK@Tw`+^7T6C8q5P{&^DF%Ey7{YmcJEdZa zK3YPEAy*TQkFM?Wx<}KoKD@)09Yk+BMFLtAf`U#-_FkRAEGR zIhqLLp;y3>cK;;>cI|idsWK6$*}xZTO3tbI*_G%%sTBqv_CC@ z@r)@6>tLjVxSkxumEs$WjnA@nZzLf*^Lkr-{AFPI5Me3;82#2dI%m^d-u9JbM`Me&*s4Wud5|F$_%(=c1t2feW=8^YvndJd zK+fG>p>vi0I{yo1r|0j?o)+Cr(qG%uYat{jBf54rM`8g~6M+TWLOx!IJ2 zbzIg3g~@A{M2DWel4-tVZ093KydBh!}FVW^q zLi$cq64sHP9fB%Qix)rXE@Sm>ZE;kwniB~`-$(rvxmW)V?T<@vJZMV7J{(-KsEoCO zBYBtM_^P%z>Tq!Jpn}o{jxTF}T!P~XQxZuybQVoKF2VrF9Q4+JM99S#O=G!o0OUq< zcD<7AEjLQNn2xT(=}06kc?W|2?p{!MCmog+-bIJb!n^6vQFw?B(+lsR!|cNUqQjiR z!*rNZcrP7h6ds|&yu#n|z4y^!cH#Yec$5wc3Ll`uDTT-AFuCwSI!r2jhz?TwbQC^Dhf@k4=X;OSVM^f>{NpF-a8=<`bl6xJqQj&@h7P+$gy-Gs8_f=q zv*uF$$XI$9|E7pLIjHU7n^ zxOmLJn1qY(`WKUNamocuYxfjftnx3W;$n|~G3}+Jjph_McV&tL9v7#8i~Z~H<|%@U z2A`e+hVJLnQ*rtc?^N*ibH12_-{+nuUYv~63;A>kPOtM$>FGvuS{9x-@Wwme`{1eg z;A;>%diSZ~gCFEm`r!ZZPQ_PFLS`2{a>{Pfz*T%ftQ5Ud@z9(2f>`;ccPf7Wy?;S3 zpF3SpA}06qDKXjL)2TSUpHHXZ^y7Rw9j8C!(-}COK0{ELiPN=wdKyj-^64y`9_P~z zoW6-qXWtbOeZ438B#8db1)w!ai2lF%ltlj(pOS!!7m5c+rpx)1IM4DaaX!MQ#Q8&f zN}PY4Pl@y2@F{VA=IMeMao)qH#CboT66Y`HQ*a(4wZUU|GT8kg-JdMjoxVt%61!{p zl-S+xoeI6*!575)gM3Q7f00j#_h0ZS@xEZOAV<7k=$#50H~JTZ-5ve~nf`tE1ARsVu)^Y#7(+2-f@f+YE=cPd_=eYPMwMbv3q_=2$R^G@lZ zP~ZzafoDMA&(RYzgup-JQxdo{C4MD=H+ZMwJ2}1}8SnB=#qS^B3li%|?^OK$NB@Gv zN-q;cC+{Y)4)Fy^GR&vM?_csMnd7JUl#K4jdu`*Zm9P={Nob@zj+TbV$0rd`bdU`II<)J)aV%AM{QI$$#`Oh~%j&1eK{m z%nSGe0yUa5vJ@c$-s5Gzi98L=r`O=|(}?*{xQjjEX5s!Th{7x(+#`HS!hMTRNw`1o zDRI4QrFfRO-sYVO{s#RE;_nT7LGpaqI~5Q8j4w#I4yt@+QT2o8H~1Iy^0od2>FYNC z0;bt$PWPMR^E~c5!2PeO(drP~FPh1xg6L)b1raUt1<7@@pOUfuj8CbC?CcQ4sD|9cr&L4Uz^7D0-p;2~L%xkqQ7IOl;L|xc z{Wm`C#OeImg4kTLlR(ve*Jn0VU7`nO3st|GPf4L4;8T+AJA6vAojgZ8OR}ZCQ=#!g zd?89DpOT<&^iIViU*Zdr`f2Y}{64Q!kenu}pzVA?`snvg#Y3;>3sTNU_!P>C$m92C z=0KhWb3t&9kY_WWl03b9O7h&{oeIhi`WIv?-{K3B^bdSWk}jJk2$FC+`ILkk^iBnp zSNIo1CdI~=W_aUnEqTxe{P^ZH_@M7`jeqQx6+^I)1TMUpV!l$hw0BF^yhu_=TZ9e82$Mg z{rNlk^G*8mE&B5${rNxi=ezXhpXkqT@yD$-=k(``ZmqXo>8-kbjeIdjRfJ&ll{nH+ z>0ZHcctxy-S0qDtMZ$zvBvp7t;)PcvXLv;dhgT$dctt7*uSgf+6)7dWBJG4%q^9tS z^c7x_;=(J^V0cBU46jJ1;T0)2ydo`!SETOniu4{{5dh&8ArW2?B;gfd6J8NW!BwL< zYoJ)^16IPqYNM|lZFaKhxw+`C9&XJUxk@PmXQ~XG9)W7}* zryLZyLq}gT>Ln_o1WH#oPpjqoFmfE|q5ZX^;q_LxSUHC4$-*-WCo^+ywwOVPDe@t_ zx;YgoJVi2}tA#7*)y>)Y+5tEE%-zlDy~zGq%vIg;=uM-|S-sesSKb!>(&%eWt@IV( z2Tfz?!VAFV?vuDaDi63dw=P7Pik6KoqmN9>=Gk|ep735oz3lx+k2YudS1|g1`lZo4 zH9IsUUU<%pC>$@s=L%QiEUu8k9{hB5VK4n{P8Y)OtL__u#2wfmhwM6acR#dHs^OQ@ z{9k&i1Gp~i6R++Ue<_9O)rCL9`Q3#B_>0u6hr|IXqh1HiM+BZs&Tz-0qD)QBq~L9u zNx^|;QgEP|6dY(K1qYf*!GUH{aG;qK9B3v52bxL2fo4*0pqUgLXeI>*nn}TdW>RpV znG_soCIttYNx^}t2OMZ71qYf*!GUH{aG;qK9B3v52bxL2fo4*0pqUgLXeI>*nn}Td z^4Hu%wdyGaf~2ePT&M(KO)XV&4b)Ni%{PO>XQ9BzBeMoGwcaobB;BDYrV*i+5uuh5 zp_LJ#lo6qm5uuV1p^*`xkP)Ge5uuI|p^Xutj1i%W5uu6^p@|Wph!LTO5ut{=g%(DH z5=Mj$?iMN-5gNE#DBx}(|A-KOL`Xj(gdY*Ik2HMonx_m6M>J4)wNONG_ta`lSih&B z@b_9>)ZWdRaEcwE(QBnFwB{*tN-@*tB9bfA)0{ToBG)ciR^dTVA6(v?RU1M!-s5YF z6*!H$s9AasRgo?A1j!nt@!*OqwDbs0l3hLlehU+vc46o!`0tc!kaG+WO2dX7-pjrY4swHrsS^@{EC2*ix0tc!k zaG+WO2dX7-pjrY4swHrsS^@{EC2*ix0tc!daG>e|Emm|=6u(l9fCJSCI8cp%1JwvP zP>t|PESFH_|C}p6_y+oSKPreR@EfH{zd6&hS@^EQ+~({=SB9FMYmmBbdp=v|=IoWD zXi9QL(a#wz{1R$x&e~o%RxVaDIeMeUk2U93s(IlY)+@;OHmZ5j<8-RWd%ZTaITH;= zFT8;2?r|!E&6%a)TAhACFAyAvD};1Lrt7uT^o}+=_U6ze;2O?N^o-x2G^e5ELt0z6 zIyyMNIU5m|OO-O5z>f67NQOu6ZgxVHzM|WU4`I`$D9VHTnsaldOntByZrzn*D1@Rs zk2E`@zt)B_S$A|K{%8)Op(Pz@&f`aOl?D;dm(jet_*zF}2qL<@M7@wXjt6IZr@d&{ zYe*9YOF0>;>wuHJ;Hx%VuH(VNe1OxO*{BwKYp}c$9vpmbbM~N{K`9MAFP*(y(QX}Q8*VFJ+|eNNAlS3KDNwEOes)M^7x`SjD4&fWjEvhw7k%_$Imu#y{XPRcau6_{JC zkf7?$nBRUu7(R^ZRwg_Zc*yul7z=XrclT@~Z&_CLJf zwH{ylZn@&!XRZG3v&z@`9$!>>#`t>iH{W}Y$Jg3(Po7-=%z4i$U+X=-s5X!Bb=Q0T z;#D4BXI}KOgUhGwe^&Y0;PFMxON_6V9{lDR9$%*{J*)fUYquo$LU)K7qu5LL-22SM z-byWzc4QCGDzN;_r6mxUU~7+d$ATRhie>-1-)K0R|L#`m$Z0$HdjjlcB$p2xkH zCOv-h#@4HTP+8)!)zSUq)Q?s_o{+^VTbn$#sH+iU>(y(Y z`69D*#napGduYX>XH{DldTdeGC(hQx)omVI|M~Rv&lXoC*#h#^QHs6vnuX7Q(tGJY z9=>P#hrZw9CF)kiU;4W(_j|ZZzHjq|U+LV^8W-0}eH>j1^K?Nyz8HtEec$0t9*1Xc z*z&gPPWxVK4sV=b4uh$Ck1^_l#u$6usXbXw-@iEPML#}VyuCGJnTcf#U5RQokDe3d z+TC!0#~pQzFpkOKYiC5zvVpB!WyUt9e?Rl&w0dq={Nfe5ANNX6bW`m{q5LG zufO!R2fUYlU;g&buG*4($xhM6V3ekD{FamE1>$TyG5INvt^b}g`AYbO@d7?d<-i81xY+PNS0m|FBNcf9C#OK(XS8c?C34PJ`& z4WGPW%0PFtZ#etbUytsr-5K-^2Q$%bA&Ahxh`xLkS{drCJ$XZQ+aC+N>FSDqFMV$C zpFUbF?7`Jwrmz=>ct_HPO1yfz(OfL`h%wDaQ*=2Rk>DnedU}Vdc?`Ge!_Y(*!qOD6 zQs?=c*p$AyVggoVSLbnPZ!q{?5dD>s`IYOHD$VIM&xqck!Dck|4?O65o|+`vmNG+3)Nsz~4%x!*~nMC4D$APi$6jN*ny2j9`?Tl;?j D8rD)4 literal 307353 zcmd>n2bdhkaV|lC$TB0*BrAu%8t2L_3xPIJ3+x4YQeU2eePNWqe2 zn*oS^;cI{S66qx@`~r?T{iDB`p;k6YLv>gb3d zDUV*)-Tpn@^W6b|zT0SYkNZoD^+LPpdBt+G>p!sL!Isx-?U*c8yjshv?KthZ1yG&X zaZLQ{9rfI1jsC`4JC1sVcBknbueZI{j?ppt(vG2kT&wFZY&8pl>K)cpL8MTxjh837 z{)(V^;ol1na6#0bCH1pjvso^BEz!KiRkt?LaVI>qab;tAg5=Q}ZMCPX9(>45bbgW)K6FyrjD~#3~?Q*@wk%tx)JfjX_v|^ zhe!+!4R!sMRku^4dYyJVe0OaP5)&UO*V zqwV@A&}nr`+x(SHuTgK6+x6x&f8@dNBZc~8qfwd$C#|80avRi#%Jt0fleBga`{#z} zSIAWP&{)%jC?Neww*|%_vWa?qqUw#hHMct5#urD+MdEHb1ln{*YwlTyx9i`1vOeWC zy`nQV?cC)$rKUH2_qLGVKnghUCWU5POs8uP)Umz0n4g zW3*g@SPI=|{0*I!H>xr#0D~ykQvte1n_plc@q7(EFSlA9uN9!%z0mPD3VmtSJI#Vu zE!WPBwmReE<#Y7ag|*Hksq!=a=7wAIs;yDCSR`d@x6s|(ezra(JAg~ zt?uJ!!sK+TjlMPMmZ5-dsJG=dY4gp)jYhQ$`5biHt}{%+Y(WRzTG0u*trPZJ2;c3r zOLa(f{jqwp?bV$7D)_gQe?WBjE@@6=qmL8TY$-nn}h9*H(p>l5{^znK0RYSf^B ztG#oDYNtqoY(uc%>v5Qs^*7%yGrW^QR|x=}~BfJ8uKDNyxD4Phk+Ey8fJQ*EOs6h?v1nAfDQJ>zdAfscAMcMSST zoYh*Q?^6?(p>x%1qvIaPz=Ra*)vDW^L=QtRCd|v@(?G1%8Eb)VhinGq)^=NGs^wN2 zEnO{WgXz&0WHbp@h$?zTx!Q(^7m#YBFAEJv^^M*+uOP^by47lZY7}O!-W0kbOGuBs z$p)IZ8ltav+Ko;d)(&Qm^8vU7i2h$Tj|opa+c>VB$TJKuTdDVqYg!CrLHArZ2qc|>}bh(!)U3Qe8KSR7J=##nX`unZN zjUA*zL(?!o9si1!*Y1dsE&=1Zz}&a%6EH$Szm5h0>IUJnMFOxI)OC3%8nQP8vKU8b zuz=b1R|S8DdI$>_JW*TM^*8do@Yw8UMSsOx}&tObZ6G9G_k6cq4MTM7@+|pR7P%8S1&UuVdY5bYe zM5*L&P$f3n@~ZFy$fA5qwY zAy~;O|EZMcXDJW#TM}S>S1It58ETBum&NDR*P@5$ErF3P!j4atp`6hE%gKB#9;Gp| z49ojE{PEJq@=xsNf@u;FyTERn7I2;KZMz{&1<}W+Zgr)F` z2g`*vTDAQY?9qauCajge8e>tr*%6*zkv*OHLz6|AI_ea8>J$sv;CD-J?D#8qkQ*JZ zlSLGs*rFDH2c~j_e{*g?gBD^eg!c$ShszT~;?WgtGPe-wY5G?@T3V!z>K>vnwwJk+mPER<0~VmbPRPOhw%#bhG>wu0==+V@0Q-~FFsf= zbcF9L>XqID(~QT9PS*>k;U)9i1N?CDVL)&{%&t%bvgO4N*#G|S_jF4ig~cb|nbanD zu!N*Xc7vZ_92g#hn!;i9oB|w%me+mef&ajN7cM;0?v9l{2uEFf>7)MYR@-e;+u@Xn z!8N409OI2!olqWn{{BfphyKYk`EA4uL^4Rn92fadB ze2G+|HRv3|TxpUt_Lv9Xs>l*WR~E`T>o;##h~t~9;Z%`(43Lxp&)<9yW47=@+-kW! zjRqIdE79IrKX8Xa2On4!Vz{%%^8~n2{RFzva}{3C-*C|L8j-^pilir#8{kPaAx^I~ z<#}iJBy?hV+QY;vsy3i`{7T|s;*|u<{7T|s;uTc|qOYhAJ^%WL(Squ<1H%XS-0Gl1 zjuUG`)XG2*@jecMD;v#v1=^|#-}M)tE;U#*7P!+hsOZ5t2H|GjtvVz{>xyrb#60#Zg zWUf$=6{5Zv_GzwBOQ{iurkPSBj?LMW8gX!*lr<8aCXUf}%X;yC&CI?^7AMHW(Ro_d zi}!UV^8>OtK_-sjYh=B6|7S9oJ*l@tb6EvWN3`emtsxsmf>*(SUC0 z{rGLtYrAlt{N=*Y3;ij7QDrPI<{q6fe<{aN$c^ci-s7*LDQLw#w+H^Ih?(K-MKWgP zuZD%eytj290y76F*wpoR!X?>_S4M-Pmj@(bf4`i4=~1`ox!qRFU&X#DzA79gF?+@5 zmm(Tf!iWu$-O?RB!V4vMHm<+$tXqZS!QNqD_uvqo6a>2de1tH`j?w(3@=?eKzr!78 zONhpW@d->-F;~VUkLqHcQ~6|2cM&3iG(KZ88@<1%iFpcQ1?saa+V!G4tv*>Io{Wx9 zw!6o}S24#5e}&gxFSv*ZAg(EV??wDKLNZa6&Ft4=X3OzMfnUv~0OQ7|2;cKVPlaXY zpMpzxz8k&kOn8V-oNEUY{qfF|Pfm{_I8wsr(F66&-Z zR7P+j{7~0_*~NdLFlT&0oDq3*{PMeH4N+~185D91_ zpcJuPngC;d4=dqOZ1eGML}w}HjH4bV$Mp$CSHx5F-}uwDLIYq%Ao2DnB199ae!yCm zCtZZ6Ip~SGcVk*2xx9uI3xxAIjDP$%S%}sA4BaOT)}8N)g5{HLje@-`5$cvIyciLR zM!9fix_d#?TN(U4TArv;pc2j$(On;Z3{{LB1qon7j+zlLn^sP_b98(MwI@-k7#A5uEg(l(;WFnSbulngDSr-*NMclLPR+zv?~l}o^bd~pl7>*pr?vCDCB|e1^S*=|jFVh;yqE78h4dF9Mnc3mc zakp|%ZQ^3_dIzg0{5%{yxorhnA?t3Bq|8qzP~B6W28=hoD8dxIz}#Ws2;DV^K80na zA1wJ%D_GKD2tQdhkQ>5UK^K`H67jidRDy`DRAd%3#lbN6L-W1?b%@?!reKFgVK30h zm9Dg2h+`7|jVKzODi^6s%6GY~Vt_$7o`O>48lofkM2Ioh5KBF@ss*OPlGTDZmKzJr zvRD{QV8P{Z#bD+TNUk>%Dqe8%UZDT|CWD zBDCTl3BP#G2#0GIlJ@B&ZR4Z>QotS13$f@Kpr^PcLO(GAV97UdyduTgAj&XVX^(aq zaC1EPoWeH`U44#pw_w7sR7|5z;#GK{Vi^L;fXU{(1&rC>+Lub?$Yfm9$ofCFJI4eT+gm94wCLao<&F$xmU4A6--yjfyo+ z|MFk>h-?ps`Bh64${^_Ai0ihg8L(0pqK|SJ>GvqR{z?P+tHSzm88uOs(_&IYhlN33 zhXqEslae23;uPnBOntb-5LuJJIuLQajOhv<t{g3zUZ)h-Yt?Bn%M5AIhyca#wvC&op)H#FIPlsr6;*2TT8)^@bsu>aE>$?Kb1=BM4G*F9L+2U;SO&sKwcUAY zse}KTSXFJ-9DXma+KP=)926^ERM%hjqg<~a>SG(dw^Q)(DY@r&Dbrrf9|(S)9Ls-jNKnP|FWqBg0B zSOWbh$2CV->lu#rca?86rFO-;lRk!NL(z)17P$tXb#ZZ2S`0@?Loml zbAM83Ce;{3Lm*u5T*JBB8F#BKZ_pv_BMMyWD5)L_u>yxORyr<`Pf0L`NZI0*Nz3T# zp^_bdj)Bs)ZG#SJ)xmn3}om>tX9;`c4Sg@kx4hIc( z=sReg{1D0lZRXJTw)+m9I+TyXMceL)C_|3KGjRzvuDdRcWwr8s%(tkkwjtuJfDl#N z3tB+yKu}*wQG?EtH3(3qBpPV4;9QU|k2?b)5}|&xjtAJ(u$&h3f}37CaFcUODx7$H z)AoU(o44PzU64KRU=iO7`pYczVm*xzP3p>ajXJj)l5e#GO(bEGUOf7^M2}m3xZXiF zP@rV8aH8H;01z4&E0h9P%J_?f)xGsLE2HvFQi%asCG16nG5Wz!i$jWubg(P@#8I~zY9UW?;H-1^3myCbt27M6 z`i(Ngp!cX!sUY!=`x*CW$?hrPx?n_w!(E0jCLEYI5(*hi_iVjfbSTLy-|W=mZ*O4dpp)Ol+RvIBcoj-bcmv%|y2aF`AWf1y+AUq0kW5`lBfS-w$>M>@5E|^!mO0w+ENlRJ}N_e!ePF3ltK=8F6Rh{kBWsOo_+auhN-oMVr{6>BG2<|bExLE zchnk_JA%3|Lj1e6cL#vVPfk|z1!5AFrToIxC^gg;vbi$+U=VGgNZ{p<4IqN{*!He6 zF$}8}%NRDWP{Z{erFxGm^&)Z@b|}GCf)p~bo80YKQRQbShV)MQ5puOETzm=i1{(8Q z6n8|VZ*^Ei%RQWP5tS_u%Q_TyyAVBJ8$PGb6lA%|kk;0yR8COaF4h&GsfeeCfxAs% zH8BMaSAyz@3vN05;L)Sb@!?~KoVyMk8@~VOi-r##JauSfwx+W&i zLB#w@E`XWi{7z1>Mb>O*vKGYJSIH{pFlR;BT6|;+KW^6>Rqw2as1+xS!SWy$Ph70$ z3du>kC<|zc$s~vKF0a)&hQE$q6GapUKyV#_Lh%+d^r6M0SRE9(!P3 zc>|g+j2_wg8z`X)Sw)x`gf$#2GQo6;=>SIt@=B^^UeI=Kh$uLZK`6n}(4$A8O0~gz zkwbLm@uxcEHH$p}UQ3SRO7*8YHADrF3`e~}*0{kz=#z}N2<-`X3!@KWrO`JZLkOc? ze~esVT0%rtE`_9@dJGoAo1$}0Vv=#;2&p$=OKmB!He&{hIQX_0n$+S;J zv`H$+?hTg{?MOTs`=YlZ&9Zh_KB_m&{Bl5X12q5*c z+=QxA04w-zM2FTL!dJIQ@=cF&_u8W+&xM_jURZ2?E!E>U zY;+0w`#RcBQ1!&98pRDOKd26)YhVz{wQ;dsCYIw6@+8)Wu%dHfJ~m!mdjhT#TI#_T zM2i8e)*_<|3q{AuEy^rL0JPp|b;Aa%4L_q>(A8m)VwQxMHiiYM7BFI5ip5_XuCcRb zZ3qd}&^+Xrl&dg%oj^95!@@$djuhi)Q9i_-=wn*}OO^~40-}p{%&}Y19^nxNUPcA~#ccSJ}E5&d1akK_2`PkUBh`fX< zVy*rX3cUM$hh1 zmajG!#ESeVR@y^D5!8{=+Ei#XIOwY{%7dErc}ZfmB!P;w zohg+F1sq^T&=yPTX6n(7NtL$=5&$X`KByg4~qg&%-{MSe$-SquIbcY?~@WEVrZ zDQt4yQ#W3|N`8kI8vSK~?Yv@u>ybx8N`j6He*<~V3P+0kqiUQcf_@$Hv{BL$ z&BuI>cMq-F(P=ezV8`T+QW?9au%H^gqDU;F%kQj--U$ojTaNq?<|EO*uwchp9jnJu z;H#lltf{dS21-d1&xMk%p&=+<%U{Ah0zL!6`9f8rO6hAWVwx(5&{!KzWtADw6L4y% z%fuLo<~*x|Vv(IC60hW|%Y!oE6`^<3>e!GeCwS}C;*?olX{vNC`p%lzmLVi6p&P6? zKp}hzI*|2{gg0t+#ik+k1=>U;U)UOZf!3$UWFmhEUx{Y@m7jwq%V~AxgZML=0#`nS zzhu=9$Nu_A?5~gFFNMs<;(vZT{^uv~XXTUPM=+DzAY;41)L&?xq)3~u!m)uiU`4-U zLss-VHfcq_za%n|*uoV)RrmU_Z5|6P^BUTki6~T^vhQe1(N@gkxts68Cp-;bQ$q^SSt29 z7&y$D{!rgnY>memNMt}@Hr9opl74m$m z{1r9Ar}%7wRi&ACC{)3AqM}0Zc1aK|P1Hj>mdcHG@NS^)*a0dT-WfLZw(buD?PyB@ znS<2g`gR*Ph>vZw3tNQe!yQT;^UaN!%|W< zxqmt4YDo0O zXHa!%$;FNg5h8~%_cw~U0Zs;iU4v5LQ)E?(kiGXH zo>kY~CLiwR*T3(tU-NPKa1TFx`WIGxK`|GCtS)O*dl-t+G0*{%HU58v^L zzlxsi4W9kObEW9nZT##<{_(P3ik`iIpS|`)$96=|ZV#S)c5Etob_YLu-3_0;Cfy_o zR$upbfA-Tc`y-w{_Y2>9d(ut?JimVB)bAyW0@h!D!oM?F6tv*=FWE2_S~l^8*L%;s zFB&BH$?ukSDno>L_M;En@i}D#37)<4_kGpi^V2WA_`EW3@UuU;<87*~=Z9Zj?nFb0 zc=Cp~zw_UMCqYBrIGtY@JPG)JLT5K_5ArE;tK$8JedJWr6kjfz!}@Mkmv?-7I| zzkq5g1E5-SC)G@lj0ng}m>Q9E$cwwM0Q!OYp*?5wt?TaU@yzaW5JU|e}89?>YXgr)&FWdR`p`tu47 zZyo#P8*q+=m=?f;+u2df9mW5<{_i}sjeM?c`9#f2;WtmjgT_I#s*?@nKRW?*M;@=}zK2L=3t~4QR zT+TvR^*j;6yVHd5TPs)yYo8}Vcu$%T)+}ZrEPb8`;k{`>n7ERKkb9m8;eBaBIKP#J zfMXP;GyLCpba;Q75PoRU^Q$|2AWaC9OIZjjo+p~{b7?~Oy+8;npC>~2V44tymarx) zf1U{8Luo?T`aE-%KAa|m)q%6L=6RwCA4wI$M(z$*Jx_%2(KI2r^H~TBo+m>1Seg(9 zpJzh&c$yIY@8!=gP56WmLOh&sE|9_ePOI|Cl=8n^z%p8h@=v7+@lC-5Y}xaqJAFD$ z2;X_0Ia{C6HRZ_HSyAReQ9hd{qu&eMkk!u~2r8FVzT*pG#@;ng&t*BvP zzQ-cTnt9413G8skUnth7#cHouVG*m@lw{i)qO%vU&~}BW!6%@I^_QkQ6&yfEzZZ3uL^~Al#*z7M$6tgq z?5S?l!p@jjPpLeEPxH+>f=?uI`gnroJxg5QDDL4w&I7jH9>&pi&f$6!_j?qbmVEnWa6uN9xtH)SQFmadB{C8|NK6hmgWA_Lw=m4?1=dbi8dg1h}0a z7EIc-b5)|9wE8hiJFh-~Y%%Oxq*e)-RJ8aU_ee8msmt7d-kST>3GQjxVixXiIfaWu zn$$J0#anQWUA&}ct$of3tYXq3J_I5i7axbm=uK|f7p+^iCebolil5O>fu4d*t-+jLw$RoR;c}vwL@5kE3#Z$A9QBWrbQBGD%Saigdl;hLYVaPnYL#`X z7AIOoarIeR8-OA>9Mu=p%(C>%mUntCn5K1bZt;5s8P zIym-;Zcm`?AIIF06Cme#Hce4=oyinl{6We$&6bGRbGAw{@n;c)nblH@!n;3z=; zRH;d=1_PG2~Ov66FBApCk8lo;Gnp2u}0ScY#(%R zhvlBQQqBcImcwU_HE`lXQSC=?%7%BET1G!yM4# zwl7>{Ia2WnJc0!h4mLH1zLBQMGB`_ z3b*Z}6SpFZAhx7pn;7q%BQv5)g>F4of-LRmn!~tC^y338ooyE>9a@l>MLKt}bY2ih zr;Q6|$Y93B!`(s=Me3CK;z)6ecV1+RY4u^27Hj_Z3*vC5lrwKHjPqu@5)gFbSBn^2 z2WR2cae4~fr6AWp#Nq?51yCrSqv#Z@u824@1%4|cW=nz6iimMi=&>TAh7{bZh`1aD z&nhDBMFFRZh!s)TrXnIP6n3bHhyhK>D`GZHQ{Rf1wbJyjB4%zhd8&wc4^2=iVwOSE zfr{|y$@vyp85MF|MgB#FT)K+z#>lCu2+xF^0@ll=8qETWYj#{)_!Qe%pb35u16@J}iNl}CG&RNP(#K&&s~_E+Qp zOXQGr1BYmNYH}87qa>yuZgsj)h_WZlZ21}w2(1>VUU^R-)Dc- zRd~=%aHqPzfI^bp2byc<1*RYn2{|;SZEB<7#EEur1rvqI+9!ACS_1Dp@#Hr1IPfyS zn|mity#(I*I3gUdCEsSgJ~Uo$Df5c-ZLVV#=`}RYxy@X6qB-H#%1?@rd8a9c!?=d0 z)?jR0RZ0lm>W7ELhe86QvIt zBDt7*E6GOkIxV;~=-^NxEW}5LGOOL#&rqM+{90qNWmTr@a(WLJEaxMZ6~AeL-mmHQ zFSDtWrdPfoaY<%}!DzMs3kZq9iXXAKKg0})v1oxf4l6ujjWbFYK89JrOLxb|pzxuJ zaA>GMV<{w`zg`)}`<4Cl;{bj<@YYvs3OYl;shIOFal&iG;zmk&t~Sr42W8c!bvcCF zakWP|zdIi>qv}tZ-~9ng;V2`+du>TboA53{V$R2qy@$)<dGq!0JRS1q<=M103P=hDnB zZgOU=&A=WqZf;g!9Wj!O34ZedzZ}mA`BluwLLJNf#++$b-eXIm?^tfUPXREPT| z31=5_Xgv0vI}L{pq{mZf*#5h*P_kmXJHj@Nb|8qDZO!-t^@yS%&VpY-?1gpcOAGh(t+LkkVJciO!hR*pXyV!aILiH;B3_@ zE;&J2EIJ-jhKr)aT9I4YuQ}D7QY&_2&Q8tDQ#12uX1JtU%ty`axd^aaR}aUSX-z$@ zwk6Sb^>8Bf*t5?$P8a1=XIeX$y*dMZpK*h;c7I0#y(eFj3`a9*)!?tt(L7|#nTFpo`~SMH)ibMmZAsErTGnciSbCykq(71Nx=^wvD`*{Jh2Y3H-Dgr~x~ zQiru-%$x@6ge{4_!zz0n@}p! zY@EaGgL2(^tOpRF_HvYzIl zx)zo9AX|WS^4E=FOV>#{{5>^?^h5Ympe!QEE0)cT53ThO|3(bMBCMWg z=JRpp^C9z*?FMXVz%4cAPy=p}EeUBi7YGt_zBao4G&=sViCPqrw556i+I~Boz6wZqTx;m!x1;=b3l)^v0#i`R&Cr5yC1eW@51gE zgT-}fbDrCbfB2ZD{1gt0$Wxoi#zbva>`cmv`^$`Z*A(}0LBe9pwjCa>I#9>{p8^ur zu~v;8e$?Cou5(LSvYxpr!}tBhBFGBg)wu$5D%PSEP7_PgE03gRrN7FUF%6qn*piUl z`Q;36ukLIM;wF)CARwgm0f8OjkcqtpJ!>&^KWp5)tk7-Fi9CXM5`o}P?N4b#!8?q3 z(olPwEs4IPcD3-9y&&_O`ky!!4jX6N9bDb0PDlJ%>N0}AW!%)P2wuZJGnSlSHLC*1 z7_TuLQ?izS-Iy~C$6vE0(RUmd9(AX2z(5?Z@F64fe~g*Wip-*1mA@W?ASJ={ABs7&#Y&3D0CF(PCRfe))+|aC0UY!dNBX@ts$j{|RQZlPvX3Urd;^Vd? z`VQhsEXH)ISc88uWp0-&WT2igZcJ97Rw4f>#!ak$rDQdIzcDKsG_SHH(RXMzB9*;a zKIb@-ooc&`d}EH=#mBc^l^Ck`7&k5}R2y?hMhAEK1amROQnHPH)|ev=v3J;#NFrtt zpOGORp08VwkX+0X;<23*Zn%FEQ+aVzIL?NIYcz@DD-yGc9(7o2zG5uk>{^3Ok<43O zKMlBK% zoHL?DN>@yoseIg+RSo`+*plcw{5MkO!2TvKH)uNtXnMeVSO=XNFsBtzCk6DZFEP|# zHEw)XsBh%0je`4pL2%D$)nJ~NZ#8^Vc3XVKm_H5QFWHjlJH9s`rbE}-9g&h1@8L6h zp(Zaf&I?!eYi@B1_Qyx9NUYP(Zd=?tBxDF9ZL1$$ykFlxP0)WgM5JY=B*PBxP&F zfH8v_#5dTI=sU!l$zGp2bo4<9?YKKxt|m4x_O8Yto-}S?RuH!co80uO9SL5*p)$;- z?14CD%#sG%1GXgk4%@o@ZW}!_+{=Q(Qw<3YCQDdX`bWA?M+u{c)@zOP`AvK8S+jXBUT=-QI#I|ggS z6%BN!)`4odLHUVsG*ZeLm(LnEDJw2(b2z!WSO!l>KMr_O5L&JGl1~})q9OEgTM~Uo zX!|hEv~A>zv|AH_%vJ;WO69X2g{hCn@4Ad5XYd{)`scalkTuMyvvJfd zX|M(?S=Fye#cepRFKU}OY%97!Ejpx{Cy~^)RBc$RYr{fgel=~FZ%ZPn4Hj#Xa#@qN z4q1?pT>PT#UYTH%*t_U%ic7ai%;s8THFR0|RJ392fH8#Wc8esCx?$`W(>9E~$oQ+8 z?%gd&^c4EQobO!O8tqLwbdGP*q|2;}UcpVwXlAO&ree&v`5A@oZ;kdKVXwZp?h{Pn zEFNMqasyVS?K&DY=1GI(FomMi(ZSaPmPb*dNJelZsP`J#c7Rn zutSa%cv4b9-f7H=2G84VN%S3_t0>45?Z|4?t0IR735(8&gNNc!&0LjX`yJzkW`*r) z5$uWfQ3afeUKTs3ZdF>Q$8Q?5r9t=wTN0TewBBC#V!|+Rh}lKPz=5Md8=bZz^7RKD znsZ z&YM^>E^6M?CwJ}o2gOmf;TD-gohlZ4aW@fJZBHcbYoklHS6Hv^HdzG=S8A~E7);NZmTf(ryuq!ReH@GFy8R@E_Pf4OWW6Y+8c}bACbcX&q zI+7H-FtC+IWM8=C8+c7|DKlL{J(q2RiK zL|)fym077iXDqO+N_A}xy?_`*|CiiG9bY-CYt@mIl)cXwGp6a&Ck2U~J%8>)VXICf zE1XUx4g(ZwaM9OF8G)}GH#92(t8;DeLxJ&-AEgs7PV2;fX3UEw^e@|zke&FC8MnC( zcCA0oiyIEzkXUZQ5Z98YE2LLpoEER?m)E@^7tWw!oW;OOiEmwD%#a4w<+db}U|E<; zX>qxqT9A-j^vf%|HVST2Fn|mh9NC_Z*sQf#lkPB%YE8BY>NrCC#9 zKp!=hTUJ0{kW-GLCQL(1!Sg9MXrFMksd;4SMe`Knu_VpXKVr;~$$=JSN%&4Y*Pur4^>Zgnv&s}!eyvyjnv^&{tHDK>4kMBlB<^@hgSKQ!86PgiT}sLAey=gVnp(We zmPAr5EKG{DO1uhhK|*q|&o;(QT;PD-cVF7vi>nXRYiH}#PMafEGa0wa7os$azcEHD zT|ud_sc9C!o0ewrSH@q})a)CAL{C3w?&p@95)yUJwwwdUk0*wWp0(H>EL}S{a~MtB z{$|W!I!#RW0|%v*KikR|YBgn=zdFqiS!B$S2E_tf60)z&6C~!`Io-mCiqVdZBE{*& zwzd@1W~j%w?J#a)#%_WYA3l1Fc4gppzEg*wD>M#ZZx2d3lf*m+U-zA6Wlu> zj*o8b!#YTHx>ld0sAuA^npmXG_-%&mtg)c7!nOke_Gom(bp_lWEQW;5>3R0fYTO_{ zPft>98MCTk+OQ=dJKrR$#asvKUfctGtf>@mc#?>)j!>#^quwf$zaf0+lUT2)7jeJ% z;d&E?-#P<_aZgVf3#2$#wbr5(vex#vYu(51utL4rSj1Ts>fs&=6^bazh7Ls}vMeAn zAtE|t5hB7rLPB&yY#<>SAR@&}vHEW?Zjz>W&)JelDxQUVEj@HLX%-|T7k%;?Hl3cT zt81ptiLnZ{XCytnUbR^De%)BeSyc}Q0mu2v*8~2dx!vltWU*f}=19|=U$!ODcg!{) zt2f2&nS*lG9;OL5(!>U)-qjeje=u%fR@AoSCcz<3)uX%u=1_qxCDrQhjakxQ`x{#l zeTVIu!<{DFDV&&~!Zz)kg6G#flK^kF$_(Mgb^V%RU7H(cZdK!ei%)Tz6?3ZqnUX%U z&X_R`$knzaG6QMtuaA&}PWQHxnq&7t+bvfIdBR1R7>fG^-;t-P;aY)sZT53g`tk+V znLCZ;nzh@HJWS=YUs+=GCPVuUV=-if_LdxOEs^bj z1r;~1@bpUFypp)j1cvfRN@V73#%yYU|CB9>B;Xb^Avu|1gKa@Va?z*Bl(;nO!2O4Z zPwm{r_Y%W+CTs*5f2|5D(jOX&H>)D?y;|5k;x>2g;xn&^GsQ-vB*p!nF*};V{H`sD zBvck?ONHU(6AKcOi?7)(;sons@0n?RtoQnjwTF7kDX-9JQqpG9|J>)CIEz#4nuQX% zs%OjIR7^oDe;h5BtzADilM8Bl>T=mv)0WFt8GlvNy5)kzCG_Z44?8V!Tqo`z7tz+3 zNPWGS`};2Arl$9IO~C#d99O{{h4Ek6*Vm+lc!!Kx(WHKhEeYAn1{k=xcHK4)V>81! z#2pa=7e^MQ!_>PP!}XAH1GB=lCFj-vl)_;eY$;CMh%rkVY$t6=^c}Xl;lv@tE;frF z_oirH3UX+jhv}5bVG6Y4kiR0LC!wK>uUW^t;6>761$fF>R9O|^h2-c3ZNOL%Cz5Zdj8R^Zqeo7B!gPWJ@9mvxT!I{WTuHEl5Z%mWYP* z%(1ZctJd7>)!;peHWoxyL>v_nSMnRBD@l93 zHq9CPmN91@ zeVWbqyK+<9A{;8v%#Txx@=BLXQ2-VH6t8ZBF{>K>YXym(Q=p!P(YA-K9OkgzRv;Y)QzGlVQK8k(0Z< z1qsOoj}I0kq%3&+wjdG2Ti!cUyrrl1Tidj)r^>|%PhS@mDoXe^QX>XnEEni z!p_J;2jdv_sKda0)3|9_f!m1F&cR!rc%!}u-tw)pMd?i}NLyz63uAsXw7zajqVH&} zIeH)M~s`F72*NK;}Ox$z^XFX zD*p<`DG7!T8gr;&e9D$Y-!a}BW{P5iJ<{kYwbvN{xb)^zf8JO$S@FL$$}5#^qpJ?NX`<^>6eVmL4roNv)FC(J+^#2$xz%)G%9cdm zbz$o<&ufd(_CWnCg*|cU8k_NRaW846ij3-8jT@O2)hlz8;8Kjgg@{Ww*XLC5QnE~b z!k8%yytmkr=sS4pkCtm^DAp+sXG?W%)2lFGzhd05tblF6Z3cofUSdpfZBtHc`MfbZ z8dRUNCDC`NvW9T{nJO}De{0;xtgt1nA6<}k+Qm1GzpkP57q%q&j?!@0p{Or4qaa%_ z6qZ}zDTpl!54EtL77?R(R3&aS=YSq#zGhRuCOG?}u1dXBwmBFh%Tbnq(&j`YNj1SK z2Vtx<=3Ud6Wws>xt}%lLu@6xkh<@Veh%?@-PdctRGS3|=SFxc5C#AG7s~8h!NyL@I z>{Zr-+l&Q~weRP1#h@kmdKL3sv?T9}!}fxfV5g4K62*N^x%6<4F`F9XJ8eny9r7Lf z-D$7&wO788f`8bd1H`c|l`%x`=f&GMWqp@nf7DnQ*S6Q zpcn9(+_=qu^8tf0+lIT0^w})ec2EXvWvf_VRjEvNS*@&7na zY(I6ph_&vIjGLHIu>Oi1{8q_HG@Peihw%HxY-p1HqAdy8e}0?6n(Gz6jo2VvuEF`C zbL~b#$3Cv>ey>Ul)4v-xE-Or%a5NorN88;a7hK6Xxgzaki+?p{NCWIUwj?qGW}Ob9 zTBgYq%jBn3V5D-J=Vl(ISt>s+x(3VDf<#Y?)id|Rc0GK_{(8L)zu#@d zCojr#mh2(pwq@-DOLJop4~m^iJxgf6F&CPI@3AEz75^?<5;Dn&alR;jqfhp7D`q{; zSlPv?cwxL4>QVISI2Psi;%09vH3okKj!Pj4Ah&A8!|v!cQCoGppI!?W=i zo%jJi1im+7i{yIK842-z)vFRi_4CG!%L>(|=-3_PzcF`-t>IT3Nm)w$v@t^(V4tuh z(RaYM3&1GZO00}y!&vN){rkloSCh1pBaWfLdkpJ0j75+Y*6j?d(uQz%pjdFj83#CA zg`1%C%#?M~uNkwb0sW`8Br*eS?R0KDbk1uQ%CsI%;l7j2ddfj&y(%$=OSkmP`P_uN z5(4f-+zyN1%$-u76=kO{n5Ek*S#ym0XxmpGIN~)1md@o4Mj-PIgHZ) z&(LiNwUh%@E{+sqI%q7yto?FdglT%3t_ag(Qix_ET2IZ)N6l1WPfOw5Zp^g?{EfCG zWLM3x?#%Us{!XbO`*k-JiC*KP2V)Qm!&_~yP^y)m=p+-@E{YQC$9EZvDXV_mDfJ`8 z&!uK^Fh6Q$UTJ1bK~j=}j~lbCDaa995`9;Yqf$Yfq(AJqEiC+|Xa;V-En*|U1Ru&K zMoaG;7Wm@h`rMLc6*_Ay$gC>#fK;KR7oNxbPRc=1oji+y&tOx;5YwoXUE(d{R%ja4 zuqDxVjoL3Yir0bU2Av)wk;AD3Qp9mRnhS!gJ#RJ^T2}2DmfFMXLrU^M`nbf|KuViq zN|Q1de}ggen$kRHOX5;0O>$S)MJtUpK~|cdHx`~=b+}|*6ZdSPI5;x`-&IqLwBcDi| zK|xn(Qe-1$0aBs|Uo&P+1M^R9NhHCvh{no*E>Dy!NJuU?CTKxIa={Cp79=DWAFy2* z3YM|oe`%Mo*Bz}-h*Z6vn`Rd+yzDaW2e)jUn+ZBKF?Ah#B|a7Cr#v!a!1$}0ir*ke zT*6uStWxz;Yp|FfFm7`>V=`M9w_v+Brq!Cwn5^nsfuB@$^%i!ay0aq6K&l=M&$a7vuKE3y-8st-fT;kA~JC+LB13W#Pn1KbRdR3lfrxZMF>w{NN8dJw=Zh129JXAJD-tp!u=f*pxhNpVRSEqT$8;rlIsoYvY!ooYY zUM$>vA5L3rHmWWJEEXGF+}7PLxrx)ddspM`{{rK-r+0r%;{KLgNpdL6HaJA|1sp~* z^2KT1^R31#X>jbag~K+7Z8P~~iHn|`0&bp&kJi1bF&Zy6ZhCe!DDIWu5M~)1s%WI) z@sKf78XhCIB&01o$&k+VY-Y`{Gv>9iL)&p1Ski{YiX)a%&QN`~ag(w_wH8NTGf(-5 zrx->li)l|8bE9E&&Xz^2aDT|W- zY|N8}+_!B>^c}f@gXQsYTndD9U;_CF43A+E$7^EGYm3sVXs2kr%iTEBYd6W47}aa8 z>eEcZzbRJ?`OmBPP740>(LPtze5TBEuQX;*gLtzoiM~U;hTP%FI^q)P7geN`Gi(Qp zo0PRPuH{SNh^H|5R7EIdUGPQ5yl4pBZA+r>2<7D=Tv*nvH`?XN@{@eE2;!7<(V=%v z#wg=G(ePbHwqz`dtjG>22XbM}S8WR|1an(tbWue;CB@h?=2Js`%$7tFbqhCGdeR({ zuplA1SYpD_Na zrfF{xBrL*CwsDTty!c0J8Lp6W+%o9J-1$Fe+|-N$#ywHuh*OdB;bQ-lr^GXsq(w76 zW6X*s^-tQ8kUi^T4BT8tGgk9G1Xbs-=e9e@f=j@o6*D}4VcekX@bEnX)%*2fP+9AXLp5VfMs3;EbK~M_R?ELKHx}@T z^LeOGDGP9N_*{~vvzGZ9*!tuiBb%DAyvk-Ik6 z3b|E!3!QEiaI4t8`DvE#gfVLxlt*nzB%!qM8l~gO{nvto5xoLW zc21MW5Ko;+ufzKDy~YyGsy`bM-U}mp%$w%Tqy&myY0QtNH0K40OK5Jdf2iE*VCV6Z zwABTn9$bXzd4maH;+SN6g(3QBW07Qq==$7Q$+^fQ7h_`_HPvLN-fi(#WBxSAe!`Z7 z>|t+V)aTk{=lCweDR)}rj4SdINRzx0K53>m7}@`6-0bYg@(qT>H^mGTz9XEK^^he; z7NlA5UoqxSL-zBwB$CKlm|tnZ*(6wykX-c3f)5N=C+djumnQiVhY>8^k{#(Z| z*^-dmbG;xj=SMkR*8^XU>IMp=2Jwn8yzd`7NH6v?^rCt2W?HvwBj8+Wtw%tm~%}}p0*{C z)DsK0OnP=a;#-iAT=dJcyZR8F7TTB=I!V*Q`w#AsZbE$An58bO+oz2so>kqh$su`x zcojMYUi0PR9_bdS>Xfps{0U>eG-djzAaMyDw>y>11&%x6!hY21Q+&WcwCX1OZULW| zO{E^`>t7Tl2KG;l1(+4syK>4#g}i8I9_{4);c@R=K+m(r@PJTdv`O6-_$S7EYry}J zEeY9wf1h<{u4n$(E6%6~yL$%3gr9RjgdpQRc=qay^@3~r<MTGg*n;VOjtImeH7If?7yXm&qH7hojj_%l~! zb=qqTSyt0=RgO3kR)ILBD3XYF(F#CpDxZ>bvfG#|O_6p85lV?GDn)`EoO zf^$$TNJv?{(RRh2xaf>kK6q*Kj0%TpEpKuRQ>)<8f_*pyP~6Hhf>|;ilOse^tz&L& zTKvHl?9e$_gh|iAdDpnusWvf)_k)}99_WY{kyev>55lccs?v`ct4(@0B9W!0D&3iu zs`Mt~uWDW9M+J$V25Rn4alCf8=A6L3!?R^?ipGs3XmNjJ*4k{vK4mP3jNocme(gWRNPx-PdqRinX!FH6|x94OJ&vmh>e)H~}T zk()1|k4?$ron|fwGOz>2(#+c5_J>(e)apE1osX_($QOb}i`iP08ZBIH#9(6cb2)i~ zCZyTR8;rTvbYYt<3E5Y#wIw0_FV@J5^1o)W+SV3h%R^_l$6Wq|Eo!~~XYd9kL0w&a76SDf5PGymYM#@jL*#VDlYT)fkmDNT>wE=cq=3UfaXy8C{dZh&ORvpA~0ERtmg)pS~PBxmG_ z4!T+~?=_n^w&eh3r_SNjMN{)FwW(MqY_+4E zVKMB}tjHIP8Q0MNRa+9$iu@8Q&|FWkhEmido|Yq4c7}_t=)3X$Z2B@Q!nciukyR0f zQWPPcry~NjZqXH{AdY;>SqI-T=2S!e?`%o*9r;0Eiv6Fk520Pe!M_2_gK#BU<$@Ds z-NkV?dzDe&d|kihgE-tGXbH_1M2HPX$uq?8BhZetHQJ>|Qd2`V7_+HCzSfpR5^{_2 zUb+lC;8>85T+A{*u-!~}}(ZKZAUmLVaIax zYETx}%Q>aYYJ0yim|2b0ksQvD5ps}f8*FM6HzT(KF<^o##1IWw*z6p=8I-A(EG6~e zuyG?ar9NOwqVMXoJ<^H&QGi`Ne6v>)qQQHt3N>R9WL1UjDXO5JmYJ55xb#TMisBh# z7B!$twj};jfKIq70q$|DkOp**_ZaBc8H?azfKE6l0q$y#PPt#}hmBd(fc_y{5`71H z+avdiuDlN+XI?tfx_BJnEH5y$A2V)rR%o~7&P|B%3{h#pv^3J;FsEX=M8DDg!^XU6 znEsqCiN0gHN}6GuSldc$zSBw>tuGrlCo5X3ar^ZI;(>H*MB+HarALG$;oE9Rwy^%P*Uu) zUL$8Ty7Xk=rBz71NCOptMYc-@HG3xT7ex2wjH+_i!9xCXh8XUb7k2wu^v^3S{5XN zMJ&uuQYC7 zR=hUn+RUHma*YCxC<*veKub9e^Sm)f8fZ@n5|_|+UUeE9A)??Ljw+_b%XB`HbC?hFEvQ!s4%IG|W(hxU%#;S)=WI#z9k?S0%kD%Sm&%o& z6rIiym$rJqE^=UaxS(Xu`dFK`#lRLJH$C(!X=kGFDiaRPoKj}>_*-K!W>t@)NU)CN zAdD`{0U8p@AY4YQ5%yAeDyybfg?J3DRTaEeAbY_z+iN2$C z;9jqe9N;FF;V_fJogYoF&9fz%J>l5vGr@8RhL}#(Z8{IR)s9EUVNj=QVd$yX;v>zR zP-IQmZ!ET~R_9=DB5XId@T%<=b1P~$=K|sGBhl{sS!y?>0V?fQtW8Qj$34c}YkG5+ zEs4JC&3fwFq#~_C`AoT;ui|+0gd(I@VPq$a84LR!BA{#^HOo1w8>8fbVjs}%$ zOCmE=_EWC?E^d~97eVfWve!1=Z#VU-#fbUF?aPYTW+cED&d}%%{?ae=T9&p9b-|b; z4Ya2PiAy-;T6KS|K-+Y%vrP<-aR{_h7X5pT8=6)0s}W?1_@N+M}8S|nE z{by`R$e!}kjN4q#rGl(sIOIk0UE<&*3mKa)8aE~@HepIJaic?%l1KF0#;j=2{Dv)w zzC*K=Z{*NnQ41K7?-(~CD} zYOFqi5u@p}y+Wy0exj4yDy$c?f^5IOUlWWqIX<*O;Ye{jQdZb+G-gFplAJAxBs>-f zOC{lj9}5zaitMz2!&vO?ni$1v9xlFXo{7g`DyO2gyn-=4 z>E?)@1YiJlUj`(*N~4tp5t5QL@$f|+k7p1CS_^H&**Afw>@ zt8?QDr}>^Y2Cqo-pk86jmWIsBZAr+^HqGG9wFh<80bK4s4{3%-!eTQ(%DX-AMk|mNASh3SBoIgW7 z2JhF6o0t{6t;nkhcuWANEP$^_+obzz#yn}r{jx2Iz9Y9OaEEX+uE;@29hv-|wHUR3 zFm7H})Ha9C5%EX+Y9sz4A8AF}73+U*%#nuK-`JAKjG46?u~A&@S#_u3$l?IOg4aMk zOD*nA_`NDIWE*do8-G^wF#JuqNxFDcaFjB!;v6*T|Spyh0%N5xM5k*+mMs`m6po20-K0Z;yo`hW=I3; zVOtVOuq-Bia_-1|-GYSVVitGS+B(znr6X;iqnK!C;MmcV@eEFX_Uf!luQ8T;R$aP= zZ=y`I+WE<9Lv?IZ)hYoPG3HxC{%37T$Zq@&TN2Wku>Sp*G$scRdd>3LFj1Knz38CH z1`aS4I!P%};=%$~wK&r2!kxE8uNrqi=7uDz?pKY4pH+1a<%&Veg|3R_KBeU%8BAPv z;PMsmNq;lXEmw3nWmno)jM>*T_)CIBPun&35wB~6Mgab^n6c4nI@v57LWESWW~Z7+ z;hU{8qd5P@eoY3hWfYlP;aP%PIx-1opbKtQeA2wj%Zypm5c+?%AWIwbe;MSt_AdAC zM?RW3Fst0gP&DB|IL$$4yxhbdW38`yI4MF;ow20Bzz-NpH*5FZn;Q$-mPe*X*ft!3 zfu>>WJJ#;i&Kxx@rVc6o>kYFq{M>@e=3PerJ>ISS_!Z)xpA`RSda-R*T0KbgT`#UZ zT&759Tmz7<8;L1)KisUf8S7)l&CQCnlN%>~6R_gW$cWzcPFNGotuJvHqh8Wf`{W+3}&ZQ}0|_ta4FW zr1D>kS<&$MXBO^5=xN)(r+dCTQ29ah^3t}7&wgPqsoRE?){h`9-rExXQ;=@bHZ3kS$pnQSXSmSAMzL;{G^)L6tDiMF;ANQ z+;2-l>QBFrgN3bh5Qg)d@XRZR55|vq7yg#s^%%LfaTBv62b;@0$|GL_9+keI;-f!d z%#(&(&6Y&pksCPW;W}lx0rn`oL~K%0j)57eS7`+x4rKP%7}qx%3m_}5+mXr=wm@u< z#nvOVKs{Ns6|_KsIc3fL^~Nk}Fu%^0MBib)NB=qH*RxQY#j`(AC>#sS6=}oQOnMBCEg?(7}9*nC|N;xaeZN}ha?b>T|lL1e3 z&@WQddKQ%N*JUQHrnMz!r@I8vVy$V zSftq%q!DtH7u+Zcl5+g$yNnsp)Z=GtN%S3_dY4D}eU zFB&&7D_mQ1ZpdRk$)kc?%HgEHZOoH~+;7;D=sR*NPkD{1TksrQQG(>@gw0J9GDhDq zZcJ8;u=AU_$=92d%B>EE%k)!Y|H+sU4VZtlCDDIic;-(Xi$Ze=U=oE4%+=fbHG^0c z%-^XC32u}gvs?#et1%-QFq>>i^c|R;BQ6$5M9$(^2V2GJQ+!ZR=qg9E!1N%v`Rfes zeq(`T?RC3yICu>CY)G=qqebX=Q+xza*rI5jm=-c$p~HWVF{>Keb-f@R>ib-?r}ITWl+o3cJ_%(|uo-)l=kT7y^Gl92Nv zR>l8X^P+*uvq%H*m#c??$6bYZ5o^;YgfRHk`40>~5`RNHQ7f7>Ijh>y0Zh^P<@t(x zPWAOx>89kPw)lPbqA*42L$)NO7V*5+L>_@IOD*CoCkqmii}%@Pl@J$5b^GY0O?A8e zK(&KKcgmy|ss0bvS{-cez~UdGoMB>HOa&{Ag&}DFVhmck22z_-v)QKbsQ@II&Gyg6 zU)5~Zw*`qyH*l>#0RN5mQVcg>u#q^JJQgUu3itHQH_y%7L6bA`cF41A@s3={Su}%K zo;GM~FlI*sV680)+0jZ zb6$vMSCeVY__LZ!&)AZXowUTT%=Kc#?ojy7u6>TO`GL;nu6J6_$?0|p7lC`Vvt=Ax zK`VEOc(e@xhVwaNp=5<~cLZmC7yNy3Z*jiUz!|@yQTc{wR6f=y4ffX>v!}uStSyPY z!+vK3`|f?%Regq>mg(?@fuh@Xg$v`IYgFsPq3L}J@I_H#MfkX}n6fIuol+5YJuqJ0p$Q14GcLVwa=P1G<%fu2B`c zSDV-iN~%I7abs=%I%EH3V}WGFey7BKPrgMBV#KkVl^VnwT!RLQ65aCuu`zQR;D2CC zA_=&KZzbI`9@8yING|TTZAjpref&S!J=+`8*UAq??%Bp$=4MJj4V=wATW9<&O`%o` z5|_|D!}Xc$=#qyZuD9c*X)p4aV%v#kug)ERudyi7JH94%-;o`E7kar{ofZt^M$N!s z*E{-dV`em5cG!}TU2Kq{oog5GqDJT#wiiW-VSll)n6kn?w`2W7#(ZiTF=9)i?;2q~ zSjW`B{B_3uys<#CVm}9C{gcMbX@F1Jl1Kt>F;L5~o(BjE5|WD*whal!`cM2PkM+03 z$ND!%WBo^r8NXlXae_^m zBVI8X`$liFIru$e0c8ZkAI#NdJMz>HI*ydiLWtUdzsjvVMecNO;CGD~(~$cuTN1Jp z{yKx-%Nwv>*q_}&iy8abt23JaV%*}aXj=Gl|7^^RhTykZ{Jl0L&wUtlo4Xyd#{FF4 zI&pDO*r{@H!gJ&u)_l>aWu2%Z_BypQz6r0<3*F0R`$2){cDpkI~qFc zZAr-fwMLMb^UR9e*miMBWO%WdBQ6$0l9YO}SgeO-dV_&{p>eabcCuV$V!I$W!N%=E zrZ-rhYR1CPu211mo8O&x1>5=AEyEdOel&e5*^)>iX`u{KpV)J=AR)PUkL?&1IN_hX zv`+ZVSdOojk?L@eRu4p+s0B}v!p6zdePZfbd1vIEzrz^1bZw-T&*q)K&G=iIru~#4 zaS6TiD91@ZSukG2o%@%Ko0w6Y{)#B8NpPYK3%_XmWlgw0Z%aaUpieVCy&Py;gS;lp z<-v{IZ4Bzk^pliw#^bMzo0JugwK;J;C#`NpJZYf(rSa!AQ2yMOMBhQV>SPnUah~YV zSrI{c5FL9r(X4lBiA@DFS7oSH=I6#?)+`aO7J{=ZITfo=3a1)s%ZxeEP+M$EA~S0C zbJ5du9xfJSTb_gC036)3<*4VwC2|dfV?ASahHtlVi?jB$>%{Tv%t9U<=fQD4;5bhw z+O_oQ&JJU4GzbR;iAy-ty3`@)0ct`-(|=~jD?gH zvLQMXDQpX`2ca#Zy9%$9`gf)B6?%=KutI&{v$5hUiE`Fb(K zHD%nOtZ=QtRVU&wNWO9!W18Nh+Qwhk5P8CuL=q7TA74%rxL;e4kX+2-0 zZNi~HP1kSMtM!R;($(ovhgIYq#uCn|A{()Pl6gZWhr*lIbKhpnf~F@wB}iOC%eyAX zxGUFSwyIU_XseWRmiMn3H#V!h*XDwRIfW-p*1u%@c}>>8U`s;whR-rwb8RIzJ=pTb zJGjxCvmvYHF|3kk-JxeK2Iy~%o0k=!&Dgg}{Dp~)l0OZmZyGb9!SollB$6;$*ga`F zxno+8kX!_&a}--={6*Lc*lv1W_jTRW^G9Eg{YP{v=^OYs(l_xxZ{dGpXV?1+;Rhm5 zwp$V2|04V%|6}r{N=n+ z(Hk#UVR!L%<@zCP7rBvs4B&_7FP?PI6>*Ze4PM&2DuaUl-a!|BmIM7W=%cW@Tz z?WaQry(#i;BR3dpo;Zyf2M=NsG2&cU9u>o)xRqK2xsLM5gu~Sd55^%O?+1+Kee5si zrwP2Wmm2V&CF{jFv#o(GPPsImCx}c`&Z^x?b#b0R}|2VaFlzzDM zqd-4A`Z0kYGj)KQ#jQ~?LQa=QjdU7~I?fz_h;IN49N$&~s!32b>l**1p}H+5=NI&l z^D9ZFubM@s`GX$zyU^`gr(q~@DC418%bUc($<8rchEgwbuy=pGj%g#j_PE7)=D(L% zZx>=+ik;-amcJf?eKQI6$7T`i=7VKytRTs{$Woq$ktud?{tp@QnAQFt|Gi{*hmhft z(5CE+|H(EcDrF-R{Yh3P&0u_lSv;i_DVQeUuVjU11K%I}E^WSGMW{ClDT+ZU$5zb7h?$q6a+GCzMnF#Qqz_!9m2 zGX3~d`teoz@pby~4g8o1#OohMd#cme%R*al-RdA6^AkjZqc$fn{Tt9U!!+Fs`cE8` z;X`5*Yy~44^%qxj8Y};fHo7x-ii=P3>&ano!H!EH8w>AUnE;5z2n!N&0`TLu>lwik z>!&X566>9DA9K*T538%f7lo%f+-$njJOoeTM#ylgdf0Q@9pn_n!g`gDMr*D&0_V<{ zTQQj5EXI)rUK3*#W(*6Pn0GeI-3o~!vF0kaK36`2PieEzobgw+S*RmOT*AOie2F!d z!4ogy+5REpCZ^B!HNhrVU!xu7m#6J8-*3!@Ch2=@Nyv_N7lSp|VVV5>ljPC6wKj4L z=zeT56ANzmE`^<=frUVKBGCnW!*?0fim@oNf;yBdhi$<|MRC_R_|J1&u+qdG9ZO6> zTNKn&uHKt4=2JtxXiFk9>h_`O{ca0KFb0sPS~!10dL4%GS>v{4g>hpJCkU4dA%x?h z^;O_XxqQPn=0}6;f*{e;9L{-~BVUd)?oQGsae+i3*~8MSFajSiZgExwg8OrrGsb~9 zQvpb`lkYWVNQ2;Awj^Ya`59Xh(k)^|xG1-%Pu6<%@y=vpT9{~8I4d!q#Mg^iRla5{ zldP(;CWr85V_GZ)2OKGCl5*9`pBl5GDaoJMl8{Q$CwR7=ax(o~u1;Ks_Es4(>ym2))Rc==78o<9fjZBYL}pN}$IA8hx*cpN zaBI#Xx_qQ2fQ)y%^eT+ppmEExcD@a{iGZ`br*NiVm2xY}c4KxltZoz}F5N)6{pf=Z z0;hF&cXW?Yy~ej-mT|!zWj)aam=v^t^BPCR;5~-wxUp=qLbW|t?chvQr43@MK+ptA zoWZ^B+yoWAv|;v$F_RjA_t}z=o$Zh<2|3KN8qL8lYe7PC!INMM5>ggd+YU%U4CK?7 zHU{$l%li^AIjZA+A#~rjkN`1g1X>A=79@l~oH{{BLW(4iFIdmc^zP1Rc4nDF(&G5Q z4o1XdjLt?9FU4Tdh{KIDcZt8Hm4n+AiMAD4`weQ= zZOQ7U^|XjjAm*2XRZ&h^7waiVUgXN`gJS&{Vo^T_&O^QaCcCINU?@e9uS@qldqB*W zhTHwNxY;_auiWNUGUc3d4WRahTvoH;ic=%#Z=_t*jo)Ix{$8vYQ@}cslts>ymwxKpUGzHzoF#u`MiJ9;8Jqglal)V+|fC ziH^Oy53HhE5B7jpxiJ2-D zu)7;Eov^#RM+rHMLHt_Gq-GG$+LCZ?5Em(f=w21uCKy(CYESXV)-Vb>Yzn8HH#WmP zts$*%WK-yFx1&&PKsK_3C1M^mOITz}!nq}+lqF0P%~=-UNMs^B&f!Ww&x=~%s^(v z!kPk!jZajIXR(9aBdB|lr{9g8WLN}r3%?YA&0$4|^v!uAJ8xFlOeMr2v*?jjUF((>w zpSL9uN6uoNEu5<3VZQ~5kc;=Y8AxKOJlH9XJnL}nQ659lilogmY%Sx~x$5uJa(ciU zlBYL#(+c==wit(bQ+{Vl!Z|?8gS0oQMSv$emM!7rNLzD#=fBu|o5>3d-3n*WB_lB+ zQq@wb!8PRoCYQw)d!^45^P?5_Wws=onx z@i|Hj2m46>ApiG`RkU}CfopAwB=<hsYU^RPOO90 z#z?w|6>#M}lK)w&9Rt4;e^s->-zpOFqHeujzGNGc#iOYVx^uqzwYUMpJjdn#wDn^% zz|vG+Uz*G^k3qMriN}U7&q=L3d6tNI(g0bcNVEl{y$6HsO8BKsnT*w(D>kn(IVvR<(8-h?-KOHe=rk{fotnYv^BSOCoF(Hn0JW^~(A@RFznxsG^{%igVkJ*dVOc z0}Nps{**D* zPi1odtA<^lP)%zZ@i)mSUjdB8wf-osqo&3d@LRFEOfBGY+S$YMQhI_w-bVK)l(!MQ zQ;AD9qr{E$e_Cur_?4J-%~<~3mV|R-nYAAS!%}|&HUkgpSnV9Y$xxnlzN-<+>?EA} zzzr=QC~hLqO!j=ehOPoj#H?u0EV3ox9GX)$*U+smmCIg%`#n8=8yYR3?H^;fHj4!` zEqP0l=_rSd_A*)lPtk$9Ow5!9+{Lye;=ozV5`_~@Jg&7M5ppr#wsHu$;OUN0aq%nL zNf&?a`tcJx-L*i?U8BZQ6E0W6mx|(0HBYeS|GhDdb*tD^S{DJ5F*c2*RY`v${+4Fh zH!Bh+aIR$0P6Qw=BMw!mj3p{w-J_6H&FZYTdx&e9cZtO|RB?SV7MBC=uox9`r>XuO zVjeWrzs;6JSe)O&=#BM6#o3L;n$6hbi9m?dr`v(3u?YlbLylwkglZd}>ZUI-hMyM; zZ;Ii{q>6rtA`1VmfcsPrxJSx%=02sk*D(F8m?sU>PuY@ij_Is@8L=>?7|q5oIKqvF#XpO`tzq#~TN1`tSkK^3Zq6Pq>TB}=d_*0M5Wf>kBv&vR;2j`t8~3sI#Y|Dj+P@8aG`1S zT=91`t)^^Agx0*;mPFY3V{Av&`QtXrf<(v#cUmk+gt~Z{ZROy1L7y0|3p(~IOj8dw zVmLQ*7iaQ8L8BNO8hMi`X>~Lwq1j$SS+@oiZR-xbkf8fl{dv*o6ONqi+Rvg{{veRVX1qP(Hra0 z=KR2CEzu63mQOG`(=HgBPO?@2>I;&A@7l1?OcpbsK{LUYgmY+4*^MP<3iGOFzP4f~ zR1Jgw-RbUV0c!skgS1vGplK0Wf((Vsp<08Xy;{U!0UY{j4kz%>5p$s-)nQA*IZ}%N zPCpv(3{T%9_<6DD6=G{F?H*#3_KSrxMQL#oz=YhT8r*3Z?G^K&VYJ(pgma8$Zsm-G zyj>$&gktv?kt1SJOc9v{M5yma`zo+?L@z8A@uxK)25d<<2VzPvp&!K{{F&8%uUG(6 z^-twFN?HWfiohGhU)3}}WJ|)i=9h2t<7idXZY4Ows0~|*jw<{HP<0*gRz)!}d4bV- zR4lY9T4y3s5J&&2sC`HQ*Vr-aGp(ceh?pA<#fNQ4I7hKDQQ15f5ud(%O3c0~9>Yvu zeqa1iP3_;aCE;A{bN0gwX$uW%Y`C5vP<}-J`fV;4lQz2QBF{z4GsO{wX=IMH>g;6k z*EK}O*^+RMNSkT7deiTW!D_JxriEb|nvx*Bo2LAg;?HWzUtvqax$;j9ibJK=i!l7; z`iX36Jkt^_8N)uw09_*%(-ff78ik~ZPc_a@Ay1A5*)B078e~`5l5h^$%t$)*C`hq; z3`uDpM8TGXb0BsfEDg|1BODef4I?};urDP?*EA~F6xqEM z3CY=&1#RHG|#Bs?r4J!j&fA^#Ny1l<4Uuozl5md5n(fDsRk+_nW;#yMj4kvCSh^+w z@3m5~7eALNt~$K&5eg!Tmh4qVa{Ps^h7fC#=)JNeDL>h?k|Y%fg4N1U5UqawA6taN z_&on@OTszGn>cd@w?x=7Smce=l}rx77&+EQJWzRp)~K(|?Ttb{8bWLUXFD^1O@5{g zUrGwOOHC;P2Pi33TidTfM|c#Uw_H??ebrPg*J5=#DP~hE^9frL&dp)7y*Xg_O{rLo zIoXZ@JvNT3#VRwc=dZ9g4%&qbiM1NzN-?vViEOha;oL;#1`TuqY1&b3#{E27+-C@j zVv$TCoQD-UztlE((m**Z{=5cCpDhXJpv-7?`iNG{(OV3~pNR!Ag<@tTbedi??ht=j zL!oXj2e@Lu4Q{1;E)45bXjbF1Vk{x}kK=u@5$K>rGGC>~kQc8C9 z7>Sfqo2N7RUe6~n6ibBxvR#Uh$wwKQ2J9%-IvWU&5A z%!UTmf7p_64%R8#D;2DoV(}^g@7hKXsXe=+ShatQp;~^Cs{!efq!)0A1ai!w!mXee zt24x0Xh@xAOTsx)D>fGs&6Av+{kbBx?%%`?Mij;Z+KHlO_6j4oLoB#yVLL0CE~c6} zPc?H+L1Go>J`|yU*j6z|8k$$wl5mdZ%FRgC~^terjRXc^iR?$;w`0k(<{-}h&j;kIc`hBIX){};*;2lys1YK z2F~lDl!Ccx3^C)k7{dp};+tZ4c1sKq%B_e6Yy@E@_EWhQ?t||av!sD~pDhXJpc;fv zj;wVVJkLOWQ!KJ6kXGTA|4z(~2I5z3NjL{_?!LilAEwm(kkeW`nilsNq~D1}G6iW~ zazOEvYU0V)XxK~nt@!gAD8I5L;T)93*H*lqS|Q2GglxCN!8%u_fUgrsWjF0|6Dt7jgfZ#MX1p!2$ypCq(tQx>m@gqy0#e z7Z|)PVxdh-*BO}o47g5}Q<)TVjO>r8yy95H@NzLP8itqHl5mdUinbW8Q31Nr39p)o zqG+!V(|8yqQ)S2{&ZYd~hi%xFNq*p`HIAg@3;wSmEecZ^ac`#ZIEA}f5E z!nR=kl2X8{Y40Q0DTU&Rok5hzx)-_pGKbS}oQ#4zdU9+4e=1g(sR3-JAbbOZ2*Ae@ zs|o8|AThZjLt^zKON_Z~mKYfUeTg*-xmC=qW+8uKOTxK@%wY?mC5?2soF0rJuHR+^ z-zOHx6v4SZg0vvvbEF~iUh&s8ME=^AgmXmZsbZNyX@%n@cXpzsu+;;M%Gbq0nW8d3 zsfrG`LI*U(T^zkG{fhYe8ZKY7C1H$<^-lfSjp(!cV4Ue#Nzv{yB)<{MV+zTfFyib0 z^P`oMUy47j;qh;Z#0lAz-;uv#A|ef(bpZ8s!9!ZOB9>d>tJaG>#ubX)tR{)z(?$(c zv`84f!;mb#n8pD4bEk|Ox9aAhmkxF0Uqj!W0k5LNxhgvn4dv_j?fM~RDS9h1evjVb z51gSskj}pzB#8VyOzK`fyqOPw&WE?DgRk~0z*2rQ{?vDLVzg7PboTeKe+gj`&I-6&7v3Bsy6iLo7+UqgJG>-m> zU<#|6S!_yU9rfV(2TO3$A=}4N7)@bsLi)W7`N|}^b49hy%PV^2UW|U%CbnP@OiA3Z zkda03Jt`-NZRdCP;*Up*jl=J-HC@xBYcmy_TGR93o1cP?U`Bo(PXy~#DnjaCrJ_`z zqEt6&2oJJ`$cvCQr22vJr}b8Cmskj`7QM=rgmY7yzqj0*F6M5cI+aY0tSxFJEuUa` z%VM!i;a!j{HTY66E55Yqpd@BMgQj3h!Z|c&Z4Ug5(%o>6^BcaRS1Y4dD0qYHuF-IDnciNJO<0)waeJ;W%blh#S zAQ5umD5B2n&3-7IARu9LFQBg7WlQ;tSgEF#GKUs#DfUwP0EwRze_S()$8AYChvU*i zeS?YJ+|}LR=n=uk@Nhb@w~VG9kt-AE>|tI0z>PJe@#JtjJ?Z{jVKC-$I7(U!_0Pqs zGKKmQEEo>9=>0)dOtiA7)cYn|U{O^A&7$gc#ZSbHYG(0Iwj`XJ#aT_Z5V$X_NzgXL zbeU6SrgQN$y@u~F%(E_WwGeU^_B1zo_5tsyCe~Ambsvmy@oKu5BMr|fwj`Y6*E~gjNPBBv&uxo5dI0tshc7LWh3S9Uz19P2N08?P5 zVv`)tb?cfxApWYR`F*w|oNK=0N*dsyE9_4cso5cJ9sOU@gx?{0doMmuOdoauREQNiW1}h=VCRP;(rOMR9{Y&H_VrloKRU#sWF{I1p@(HmDP0eNJXv_sutYvb_V-;xC z$zx)!HQV`^EeYqgbD7^z@FdqLYn0IvV=MWwSXri4a(U3GC~3!{UU^!~t7aQNuqE;T zqisyP)YZsrLNb>aZo@9IVxQu>!%Xyof!kI)Clfy`r6Z$%_o< zez6iv;XEfv7%C}LL{Kz1ZI`C>!njw=mxk|dTN2LkUD>uU4%bIBev6SkA{O5i*_G`J z>InU-ikK%2)d5=)&QVPvhJjZ`#5_)m&O`99N>j^e!2O;r3FqJn zE7H+d!jmp@IZRovm(7#KAJpI*XG_94e965>i-^OO#cU@!zO5I2qg`I3H@G~m7K?3K z9#Tx;Svdy~$PsU&NP{3vqre zoEzBvcl=vd`0l?xd&;ty_6?;azA`hcwlS{c;31!wsN<+CRK~cB9Mcn*%*$N+8&^v5N z{J|~g2V-PGKeg3BD+_x2S!eqeW#1Ru(jV&dOj<3$rJ)AePCM1zn(9&;ncGLkpT~OX3f1LDw)uqqd;yY&Foz zf?j!5hi^eXdol~+mt0xUM%{vT*$N+8&{ei1{@@mL%NSYEYi%{q%7UI+x5~F5pFNoc z@k_2O=tA9sUTG_QXhAQxCE?J5c$4Oa9loC!;%9A#|2U0oC=1O23TUgQRkqPqN(8Xr zopjh{jj8_VP|lJDkCQ$oR*mU6>4Ie78;0Oo2y~}pqrj6YSg%@T%#v)4bwLX$nwXyj_wj`Vzz!Vyq(_B;()9`18=?7u~OkqOiP8!Y<2b$)e6n|CI{C8|gIM@7| z0|N!W@X;Q^Z4@*T!3z!s(_PpMmNN%CH1d=p%9a6x9#-tZ* zYr`gM#J{F}k{Ze%;%WX+GFJkKgX-D0Jf z!nv*iXX-qfPbN2S%qgdQq5L=_*vQ=^lm`1wF?$;9J8VffhyCIP?CUmR&Gs-Y9w00~ zw)tbdi0UA_JMMWmX}CO zwe-*sbA%uf{bD{fi^$uOaBdOjHY{R2DgtV)*bXR5BQ>$q)&|X0i}qIjI%7X1R){I~ z=Z4s?PpRgY5vOLCT3XfQ`aOv@&F)=d<}|=>w6 zy`0-@H$t4?y0OtT($?iq1^aBaUlPOG+H6TH^=av1rms{k!hv|GZ@CX%xK8zX@wYT9 z{H!9;R{b6MG1vdxeT6hqBa_zwr$^opQDwcweb(&1h@~~uY#pI)i6=FzC7x32TTICR zO#FFG**_j$*==>5Pq!-_J?(C&d7)E+oYal1g=!pM*%Z}y`^Q-EGdGXT=tnc@`YG69 z#3|cj9O?s|X<{Zc^-s1X5!Q7RY)OPQE*rt9YTT#Uj@lJXTXQ)$s>l#)5D{v?L!P1@ zs#XgWKZ43q)YRO3g3a_&F*c_4!~W64kw6$IQVSABl=@So7L{#XH}Fkj_B8|FXiFkA za7U)qK{I8+JR(>AK>&_~3c!)@18_vm#r!okfwWiwrY5jT&9pTnP)$W3K{SMjj_eI$ zt~6v{XiLI5vWs{Qv{LOXlqx{BrH4?jlY@^yQ@r;-)4cvdWCE*<08Prf-oz9fHb4Xws!7qA?AzHY_)u3yp zY83MUC%y{9S%rDxFKZ~wwk6>lg+<%Ea_$I*!AHExnjlR<;=Cvx?H*!eE)ol8T6h*E z5l=VZF32d5YO#U-0x=63MC)xyIEM(Udn&wCVqFxKrr#Ne>%}6N0zs>OiicE-EqgBz ze^yieAzKnBPWkKGD1ZGBEB|J(2y*4O*pu}#@n<#Vzr>criBtahZIpk$ul#q4MUX4M zg;)FA#h=xb|5jTP&XvE4+Gk2G?oUNz6KXG#THeZ4D~RS4RLxOv1n+Ul`J7k7-NMBE`{HKm)wd||+yhSAT(JZKpG#Fm6}j21Vtom@rfH&77aFo`YW zQIOg`#VF0)>Z;p(N|^nmkNNYHJLwaCv&2kjFip25;T)zDKAE&ZtP0bD*Cr;zI`IcJ z_&RM#IEQa}(7Q%8;~k--Y(e0o(9tb2I5l~J%ii_y4C`NrRbUG1 zsP^ywxtJ{t+&9^ha1O4_-8_P?>{W*N(_%%KLOfbe^-qX7(@=iQmV|SZ)k1ZqL}}!( z^+$zOIhw2XCDI8_LB<}Fa5#>3X3bw?bblsRfGN7Xz8y%w0ai!?goE~zs41sdlE8;0 z96A12%$0`h)3zj>BfDZx%m-Szh$?^AL}Tkl6jifV7}1&AT#Z-GLITaGTXV=cd%cqL zyN>2GF-IDjlWj>jM-$=Mv$YKQtz*QE6nFjIl)$bBlRHQtf=o9!?ZHX5)E~`#awO*+ zjedu>TL$LRDO zzE@Ein-2zQqu&AEb%Hci<=9&42+uNNgJMxl5nGlFJfIY>w3To8{Gvz1tZ2YhY)Lo= zZr&csgAlRdXf4|60fy);VxdeSLS|`7O%UZutMu*_e_zAp4Ynkl<8tPXGF%zGVzn?x zbK==VB?EIs^%od#Y{R9&^NiXPVv$WzTb@LN<~&o@f!fC!}pY! z84bSg+mdh&-|RhT8}_7+p)FH(9ED4}%kWIy?rOMz4n6On1_K%@I5(RD)bQt24SVg8Ka8r^-fRIrQ z3TT)zCyTxG3AbOx_B7TUt0RXk0D()mOm1!(9~G2YSBvyT+%A3m zUaemvW?D0yUA82go6gdMCD=i(csQZ*P_OZGA0r(>Z0>Kk2N~R&SV&WFmr^c!#bt`K zWphGg#iUl!%VIV(xJtGpoWnI!m2{;FRE*K8Ja&&kd81eqQ&47ciEHqp0r7hAr!^q% zwk6>lh*<}!xk7=9j)p}1CPVRYu`s4k%ub@6EEKqi5H}hY9~FOF!{TpjNjS%1$Duy# z-86WcXO`cy-D!{YM=`^i!F<=2gmZ&g zxT97k7dCio6>|f%z<(_=huY>DhI8T$SL2#Rn2=MvVQkaPn^vO7i&@YB`k()`0L?m7 zP7e%F$wl;*cpiZGO@?HZSRbY(Y z3FSrJt@us({BnS#fM2cdEQ*=Y;5%$f!a014u4=-J2l;x;&vp+nSg#WcXA0J$WCS5J z!fnEV{+XBs4Wc`2NjQh71D@jOrD86IoLLKArbtp-; z%wJ;|KO$CuDU2O()?f*gW2-L#dYUXjDy;-aK>)4nJtSsI1NMWqB%A}g0l+qf6x={! z-Hy{f(k!e=-YQEP4Ed8{)tEwl0YTo74GYDe$vmV|Qyn6=-l zw;CoymY>o-u(JS~JHQf;+_AXb5h#CE*;wrTXY^U%Av>@cLsu z7sEZssP&75G)3*y79%>ip)r>lUU@Me8eYA&B%I^5Xh-f?Fyk4Rv9{>j?jgqMF0pW? zI4wq+2I5XlNj7t*VRXBg2Mwdw*phIL(ZVh1N-jeghWn^fh+b9Vz;)ZAeT6p9FhU;| zi)M<@qGUJmMq5Sus8a{tQk5fFz31?tm<0`>_uG3!_J%@;b2)s5I zdJ#js+SeaNYNUr5vTuupHHB;$H(5ur*!$2EFq=v@nAOVGH^r=I(EXh)3Fpww*`6sC zOZ|h`269u2Vx`|^c%BmrWD3vRBzg}Gj#?L{--*AjA@W;W63!8syA`v=97D5ZP1oW+ zV{+zIt_DW)k~Aa60hHU;aLdG>*FZVdmV|RqPQ9jtjbF-{f-1Vi)^s7$dL3-oCmE>i zVlhn%&uPgb`1GAV&<>@rQXBHA)ofeDjA)QuZcD;BWGlnDCAIO#FH9Wp1p{5IV$+B! z)-5-Fi=oVk#W#ggO&R+0N0p|og-TyQ1N8>~8mv8HrZiYHwj`XxIxBFC2E1DIkoZjo z>9t~EOhKC6aC$}ukhsyX_*3z>H7stmCE*;46-{}AsCY3X2_#nN_OOlTh*z$9*=X<2 z>=nlC17g8VaXTy7lpKg!G=?t)x4CW)bG1I_8k+ADbEKj9URx5*(cBPtd0;AF0MO;o z3+X^jQ!2xgpW}$o_PI2CVh5443o+m^rM)T&uO$t}`x|1_nBskY!%HNTE!7Z>qkSVe zP#_xOxI;9(Fn?Xlpa%a}Y)LqWe`zB=)7BE@%R+9FphQP89_~Q~@;}8wngV%h7^NxG z;LI$pmK^P%DOOIs4o@j4&=B;``mcIPJ8;9r6r4CNrySi@aC*4eQm!5E*u)75BW zLlTQ+fn;h%r3q?Ch86IJWDPg;7Q4fiiaFHKUu;XlIr?V~gLbQ)k-_tf>t?aYre$|I zMv+0qqI$8-IJWCQT_$En1My;863#(f+gwJ2P0(t9vJDH7y~WNQ$Sgvx4y_QCzRd7? zVr7`Z+X;B}a;bpE+ySIX^{_N;tLtttZyMHJwj`Wmy|k&oGMov2BonKYhkKAQyiF{m zDTb#u7f~*f!QP)173{0Ud}w&R%9ez4yyi8R%~s{F)dP&r{bHd^@tGefV+@eq?Rk&* z`x-8PWlO?2F3WUW5-JwcKH6>^D@!9i%y@lGEUYPBXSBd8rQ$9PFt@(q!+n=u7W1NE z_XS%L&at~-c}Qs#XJ*QaLhaX;*JA67#3w zKHrvvbKDoeF-B#ZR+TlCa%~O2^&^bkrDDNM%k4sPhEb8!RK-jswB-u^CNT#ZIvZ_C zI7eq*xFrxfh-k-KYOSSV9u<~42A!%iaNO7HC4ApX1t%L{EuIE1AmuTs3u z+>Du=o*brJ5y~eKtq5cH7>!$$EcuU5iR4?ZFMCmUsgSJ<_T%u9=e;y~b;H4-ARV*% z=tvJY@B2e0=hsa{a^%hO!;G`|^t>N`5Gt$5{1XMbNFu@}5z=Rq%>!B+d;&OPer zsc>L)oh@ZLck?a)T9>VKQYO;%8`kCq28-Q8`K_RK_43+I#kxIpiAawMGteKC$`^MYqeHx{u*QX zJ+T5zvFzZ?T980>zZlr2?TU@Wt~#*a7Bi&*`%POC&VgNqH3Hst5`fDk6Z;%$g=&1T zJi@~a;<#O|I?-p)ynln+2$0OJulR7V{G6@g!%Fmbwj`Wmw?Kt1X{e-C0c`yU>-pq$T z=fm67p^1xHcm7ioz|3`%BfWL>sq6g43)|f2$fw)f$@jFop}O|_522@=SbxDL3MX03 zk)Jm9-bITt`Pl2bSkTjjeOD-LqH0ULOkXjFfSu7sB)d$RBGDF#j(j^TY@wLDuRu`{=BB^du>UC1ab<%{-nEGpm`$qqZcRo5!-&^(-s`&F7ER$?WIE2M^hVxt{ z%ZZ=iywFyMV>iQjL1cz=%gmGgt`@6A(vLN*Lq;*fnX=VaI5=NzOTwYq@(kzMv`I_# z-Do!u^^JFu(>V1EsQaw^MUZnO+8r=`iEEvVSa?%VS7N0)i*;x$OVS=8Y7>;pyNc|E zO{{8tlDO9z_ZN$K(yF5CZAm!C^mK;MBLMefbs_DCVNhVk($tRdEF*ZUSX5I4mtm@m z%*~^XLFJIwwyt%5B4$Mc?q*vO&Vd_l6(Q2Z5BDHL_Fl1&rjW^&8~$3%hlbZXZAm!C z>(m|PQh#Fa?gO+_sA;K)=Fp-w=CDsPVqX!9X^PlsNJ&!aPn8OYhmOUe=wgvxH-1sf zhz8l`Y)LqWEH>TAppE?s(J#dcFa>iID@6Y$W=aG0Uu{V^2X=wl*oZA|+qoeubD94ZD}xl5mb)>(%Q>hrRV9T%!I;ESM=y zrVHGEDds>!=PztYI7cVe;~QqBI)a$_YmDL-#0oG)G1lXgEO7s=m?;g|Pur4k4(wh( z%%>mio5o1VbioLkeJ_)suv4*G4zbN)51#tn0Qx6aV}Pw$w{5r17nWTq_% z_lU%+>KG#0ZAN6HSRm69Ha8isdv!$47k^ztZ#2$ z4B8>FXr`boN~(S1Bad+oidoPA`XgHs&H-AzIXb>slPH#s@|smTF;FTDBBZxU;FXKz zJCwZ0P`*U01XCziV?S?+MrpxlCZD8`kxTZW>bj{!Y4hEIxBW@-#u8bLoGbJOfCrbAffpwu_X z9LwRR8_DBM?7q%>V4V6V+7uSLzJ9!kmWYINYnijTQmOS1sMzVq$&!urdiC3kt5M1s2$|mC)=wBl9>@)DHSDH{dC{<&XiLH&cK-f{jq z)AJF^US$(aixpwoE~LVsAT+P8oO#iW4<$)OB+=@f8^oMx_09`zNjOJ2wX>M@20Z*h zXn(bTh0bUY3R9fdU^7HR7T(Pg%EB14EGdYh z7v@)r8Pi~Xxh)CjFemq9- zi)9Mbf@C@cz7XDW*hQGniy6?M`K&Do=g_QHfqGOP$S|r#MOCU>(t}>*saL#_Hi(zX zUMv%0#|SU3Y`n%W{<~NOrZ6Je6xd5~F3ZM{x1uW3Zjyl}pp^hVF5$@P88K5DwEt{N z!a1}H_N2?b6y7hfFYGo%d*iJiVZavtk*iS@TvUC|nmaUl<2Fysfrid(TN2LEIc009 zm{n=90Z@B{+CRo9T_hIJv{;>jeG&~0hX?8cF%ue4>upIm2Wn2E9z58p9-O6aucU0| zs*Yh;t{01A3d<}c?H#sGdx7}V8W@LcNjL{1xig5e$06ltyYuk)oZ0S;IAk}Ag*Juk z%rG_}b4?qcM?mpqVs11PUt&wbIf^TSG>Aa&+Df`t?OJcZ7_`>Sb2jzmZC{CZG~Ou| z+Z4y;jf9E7uSR?u0Qq(?I~tI0wI$&k$a&jJ{=BB^uh^0ZOV}3~uCWePd}-)fNo(||^r{?4nkr&s ze)f(8S=-TxQFpMi>mBpTnH*yJM%&C$(qgdxQ>-deurF&0Q%oVbNY<1JP)(J5Agk)v zLRNLBFbE&sKvo~saOCi+Rzo>#`-`5W5rM#d({O+npEZUAB5{cyW$8ci+ZHWb1kNjQ2f3S#-$&$JBzy z#uN0_wyFy~L0@G{!l6~OC+O<1hlwO5!)*FF@BC`Btm>mts<+dc8m)H5D@$}s-7RwYULr>qGVg@v5cG!||4$TSk^c@hZz!b(&c>4B> znbM%m+mdh&ZEH8)NPXG*5w0MI#DbXuX6osCmzV<$o!f0mI7g?w3vUFW_Kz`24~qpf zMM>c4`(ZH?8c+|~l5h@`m8b9b#NwF3GNPyNx5b~K( zwh)A~Fu$`U;T%P$p1v!b;b`UQd#0EjtvD{TCE*;%R*t-p=Bw2M+;Ckb7Rt1SH1qV` zF8;oT%NAP_&T;tzc={H^$}q)qw4S~>F=ramJ+>sABdxgxMv9+ym&@LrVsT8dGH@Jx zt@z^_9Diy{!aW>u*T5(o@w*Jihs5HT!vVjNN-5m*sxJq8K>TqHj`!J;a1O_!Lw$pA z9IB-I2MV+$rN4#KT)o{x49>U2!kL1z7@3j>;p!7`mkPM^6&xQt?$N0#>n-I}!_>W=n+eOxB63?9Ay9Vb`mv21zv!WchIx+h`g^g6rg$w? z`BXB@BjS01$M(^Lz9wcv1MADSB%FgacLxI11sMqxSkZ#j;y#1(A7YVA!I_urVV+cm z0R@g;^Zi=sXyC_3r<>+!rO$aBN-Hvd*QBysrEb@jQv)2z?l9iUpD8m7i+ z=AilaW|>@g+ERtxoUlhJDP&8lu$S0sC0xx}WJ|)Kxv@FS-Kky_rQ5An%!{SC;63($$fS_(YWRMvzq2f**Nh7y>r0#0{2;=f5v0$dS zETmv=K4+W?Kyjv_bB~w<4V~B7l5mdBIt-_a*>pLZ*zC`8B@i?cVS*7%mRiqiO67j^ zL{$LcmLZ~T2r#mbiIrlC?0OoCE2&b!>=uQ>^s=c^QT2<8N~#gUMSo1ppN9KKY)LrB zy<^z%A#5OXxVK!Y6|>QjYW^By{IpmBrWkh&J8A^n4FR+Q{R1&m8n92=l5h@esui#* zE^TA<-xgiryA1K<=ee4uPPKxVpi6oF-sbxXW5c)4ry|DNGnxuAl7j*dV|5-D;C=n%;fN3 zGTT}uyj#qVhU89L63&r4Yb)if^p4TWyQc!sQC})7ojl7KEt`h#Fq##y=%#3%jb3rq zYvjx8!4^yK9?r-5>-2ij91ydl0ord%!a1Oe_7zIiYB1Gz71C*xddulp`1f`XF=THL z3ug-1;^ctOor0>b!JSsZt@~(X(Eurzcv{hI^2) zdPFRwDORVd$@2)8sRoxCUJr};(D3@OEeYp%b!<)?O%EojrNn+OomHdFEri~6>2fYj zvjqoRqG$dZWBGlt0!*=7g+-_^LIqY=}(k+#{iz>X6$rNR2#|5{j=UW>F$Wi_=zE#c9o zn7XkFc9;hlyp>`-nHI`ZlbEy!xl9q4L0nwD)3QR$hlbagwj`Vv!)XU<{r!Ygd>j1mGbh(I3LKU=r z10_VgER|z%;@6J&`pU@5jATiy4pSsMwPe~V17C&ZDUdv=3R=R3G(J?J=SD%yn}&AI zmV|S(&!m1!sa8p-0dFE#Ok~QX3Ux*cxva`@5v?Q*o@Yew7K?0(=<*~bX$8+I#WNN` zQX0=1h;Hjd|3-`jD6n z4Xh8?l5hx?@9A;2nq~<)o0zIdXz)?&)M&XgeTi}Wu99B!3_0m=)F}kl&3)OxALMCU z-L>)u`LoXj>2!Ujlt0KKL`f}VHNhk7)Z0DOETaogk<0YSB+Ym!;_G20ghA!=En9_# zt|Z^ECE?JX*_C9ITBh-54J)()+n5qp`9|WZQ$Kp;a;Y4f79I^DHi>yJa5cEPm@^Xw zQqqcSLrS$%D=ww_(U(%I-)4*X)athxwj`X}!&;iq!v3gmCK(xg6UEX|&R`N(RJ6!T zUuN7d5G%v9+3HjiUnEUqZW+>~APr|L(WKF^UN7cN!}?rX5^=00m8H)`804SBy<3n7 zxwzjpShXMI-#f8E{^xEFLU(gS^Yzhq^ZD(K$o-`lLRaDrJTG_hVHY3v@L{hyv^GEz zuyy2q4{x!t7sTK4p?q8rP?D{b(k7)YNGsI6AI!$Z_O( z(Jo0{^d>9lcCmsC!KklEmO?pFBnPF9qlKo19JtC=ROt6$+9pKqHDbmz8YVE3tYHr#^>a&Q_?@TAg9zeEX#y%Y*3xC%!cILL zD#!#L>k*6*O*V@M#i}zki|v?sj>7f!HL_IiCCMf(a?6xbHNDOQWAEl@mX-xk)jvjt9h z*CdvP{NIWB(~$qFEr~etVT(UAjCzjyH=lFEoHijwf-*J z`;$Y?5!&PMZnU(!7UthWhh_Qq(qV4?K03_G-%p3B`S;ObcK-czn3I2i4wLd9pu@EM zgLIgm{~+J{5FKXcKg@@R=&&&VH*^@Ef0zyv^B{*!drmH!kS)~TFFyEf#h)39qotv-w7taurS+Kr_I z60<(7vAa`U_buOb(k(;z)Ox&e(yo(UJ(S<;pYA&8)}eegxEPO%HwG6cY;b=1+ns7|5W`xZmN2Df_nKpz91&A z<5ObtC44#=r*GraDLDN!pH9W;&-rv3PG?V3k?R-j{e~?d!^KbDf zasC{i66eboD`Ld?m3&H^7x3=P_Ak+U% zZ~@Z~lTS&G&+{p<{tG?@>$UoP-+D^}RcuteiRaan<@{6i`|ZI6>FP1QAPe{>J|zp7x=PU@3s}piWB~{Glq}#FpOWp~>z^vxU*(I4 zA#dCs=obOKBX_5cDnkNzHpg;syNC77sSyJU(mN7@=w)6-wiJ4p&82* zCE{wWf2tlj5M0nh$ASy8&9?*>WSd{+3zFpD{8RP%!ZQ@vNvc}f!54&epMOdZg#vH% z1)c_hzeG<=QvyHBrzG&=g!+{PzQ{jS-^ud@$yoPK)$bqY3li)5{;B$X!kLQBM8(&t z;DW?@0bh_LFXvO@_b>UB%<*%4N=Ek#pTg)u87}l?m<}1PT#m0yS27g%lw`PzPf3Ok z`lpJgZv_{m>gR$B;%RMC(IM#$@F@v&j8BQvx9}-(`j~&JNd8lBK_sWHP*f)GB9fc< z0s__Q(=vq`mb1zjAp6YE0Q0L>YK>sf2r&Pf55Z`ILkkf0lZH zxL(Pp#Pv@9RPk2~E{MOk@&(EBgnz0YdX_IpxCK=C%%tiE&tDW=(91Uj7bM)Pf(w{t ztv)qqj?eYEp9Sv!n;NZIiu+~L`BV|z8e9<3DqoOXulG;YLyrU(^w1B23;OcJ8H(2U zT~sJm2NzUf?+q@fuTl*zsIT(I-~xS>{G-9eR9yUmFG!gSW-7v@%!~Mxlz9W6lFDAi zr=+F7=2J4(FYzfE>$7}HHRR%1iWt?9m+&dokZC@p8uHbAN;Twr`4p96{;PaC2dBU0 z)44cZI$IH&M|RRsbRFO)m4B)<{sO*G zrIJrc(7*6c)gxc$3zGU*{;B$X$y`NpiYn??^9AXn;Ge38-oh88oKNs6l+z?n(4UzD zc}|@Nf^(ESSMVvx(4Q^z zXB+*wp8j;vpEUi+&>xTf^wOVK(4SY*pEuE;x6q%5=+DFS=cDxJWAx`y`tuL;=iBt> zyY%Nt`tyDI^F#XcH2wJz{rNrqc$NB`o?OAJbX7}TWv{!ID`csPP>kMzBMp`AR2+v_ z#Cmu|GK5zoOn607g;ykActvuCS0r$FMUsbCq=N8@bP-;WQo<|JPIyIX3a?0C;T0(^ zydn*TSES1DigX%Yk#fT;(sFo3>JG0+@8J~z5MB`y;T1s=UJ*9o6@k>as?}%q7E0Z~ zN?BNC=#HWKTsA!~+x)AKTYXx#)SpIZhcx050M+_Q8FiRm##X4V%3yzYsW6m(1t6|Z zI>1+V53Rac8Nll%jvIGPeMTkMO<@2xcVp^xD7@a{6-q~OJyC5M$I0}pmno#n+*Q7& zJ~>;;6tQeGbd3s2a7}%7uCm{2e&)9N)Gka&7qVrqICT9`eP&m>SSoG}f2noXCzrbO zn8>8DbN*%Ea@RPnkBYrs#j7e&CZlDe%jQR>WO6+DsRZE{Q7`*H(xdg6!4-_YhkmKm zPtFVss2BFU2!-QS_+0)PoJAFq-;bZJ$seG<^{Gnuz2&_FP{6FJhkYCAs<#hXz}|W2 zD)?n{xfj>@Yt^gQslQvN5Sc?bzb*ef{Kaa;3)BGtCEg6pHwiou_<6^gMVTC*Nx|DR zlY#@yq~Jg^DLBwf3Jx@rf&-_F3Jx@rf&WM*Hw(&b@7yQ-$K=?+aX9anle zuGDf|Y2~<5%5kNW<4PsRl}3&$g&bGz72J-hPMKtc7 z41e0@+dEF4!wG)`_`fQ6adV;fb+pr;|d-z@5f^u zv3Kh;_9%luWB@7|t*C#jiTb1pybLpaus!_Pj^y+M>Y>EGbg$Qv;wPIZMwpDVT%W&h z5P8Oni3pR_3!g?cTnK^EyRUroEIx`4umcW5(2^^@Fz=3KB z9H^GSfocgHsFuKiY6%>umcW5(2^^@Fz=3KB9H@H0fvN|zSk057_?2n|9H>UXfocRC zs7AnnYJ@wml1B!9<>eoFBmKJ<6~v@mKX9rH)u;P5TdU?^ZgX~^D?`oBwZ+_x+j5yI z%o5Gn9YbhJvIW)88Or|_YOK%PRytZNl+syxqr#8X=atI2UYaKDQsld8HUM1D*b?7AUIH01L<Qbncah`WDn@BYtLoso0+`AMS#Y3=Z8^p9@jC3tksKL~%l?DEIBH&&&3wt9@OF zAX++#La55~@%r56Un>LYj5l;V`e+uSp(QaB0f^{z5%qlf z7#^JMpLU^PuTZk^{QW0Eb+fX$isGv>Sghi~`~rYepI$2$x+<`|emvNBU43>RHn*VT zfL1-zN4`ea)aM32RSLDK4jp&yQAbaC zhto9*e&=pRP)%|6^(bn@?~NOt zxh=nvuCCa&f65o0TfH;ifvfx~`m-8;Zp)v8zx5ZoVe4Fv#Vzck_T8zz2f zfBYrCfVq)ngO3X}?Gd)_db+gKXKU7)pC*31>hYK?PT9J^XN$TT5w`wp?XzECwl05W z+dcQMICw&}wb5sbx;|019w=}1+4`?%rhcZdBF+|&r;bwOrMnmX@wfe#e*M7hQ$PB{ z7B5k^D*DpjU2(6E%fx#&ZT#}wD_Y~?IjN7Em%@BqP>(Od;p;wh=mMX^)7D?{&gakg zL2C|QG{zh@rtW>ls1F)p?DZ#a&iMNNx6@zs(?f+8RnTYtXK9d(VP z+-)28*FJYYfA5?AA@ytvYoH!<^rcT;`Jn&OZ};WjwP#CQB-kDGw<9mT;gXlW-GAwK z#qa#f)mOw{a#OT17^P{Ppyi}_fhb#_pZJu|*8j|z{p|H$YM}rcB}87j=lISW_@yf* z?s@wo+uK!B&l_7zBT&+qCBoDjZ`gLV&(x$ZoU~}$fyZNp22^N-6MgB+rLr&nujhXK zk1x8Q4Js}q394HfZ$+4TQ{|jTd`V9E$<43&&t)&xB&pSxXuV%d>Cto?bzSG-CXaBs z2C!eOoU0C^N8EuJLxo&M%_(#qDQ4H^YXgJmyszaxb?QjYJ94uBQ9*2GT`L$aG~Y_}|dnY1rFvxC?%*{T?(}Db>muOwfn-=hBFRw;&o1fz_K0I-HGhSWV8)N3ewmqK*k6cn*`px{*L4HGH;Ns7x<` z3W*WTfJ2gzm#8b(Jj>9$V{jEN2xXigI!QbD*4RQMym%NTZeBhjs|xz12XIh#r1^OL zTiTW>cF}{*(NWqlL=TmRBWr@Q(k8wNTS&w$vLgiNbOlO;rza*gB^*p6G6&OCH2Yx8 z6W4Xzpk!>M-EMkBv%}&LZ?w^DzZ|9Ru)H|3&x?m&#rz7^NSk7z5owt7?&L8yk0CX@ zLBt3Jeg$w>gv?!^22#>A8~k3Tl$l}lz0F=mi@QtC8$3pE3^f*)$2K@ZYiw=}V;g2h zcJb)tadue-l2|)O#PV`;Es2UhGC9%pkPU3LoND&Mxm-264r8)Bq=Dn=1*9ob;Bk6^L_FZO4Uby3RO^Z~AXBQLMV5?KM_~N+^2%(J+t_S{Z6u#!`DXMC z?3Eh|dPx-eUK1A$*(NiOBnmtCH1{R2n?bKA$L2RFYVew;%|ccp zlfR^lnw+JOVt=TUa4@V#91JqSUgcFZ0Y+658V|gzx`yOaA%i^0?#wRq7Lk#KWcoi3 zHF^rZ&1{JW$rO|#yo;R{`3Jc?y{n)+=Ciud*LV28`3Ui>Zz~k(&tuv$XO|uzVD+`4L zL?pAid0SC4^>*bPH5MiP`_$f}qbjJZ zu2ONAQ8%?c%sC1DwX5W@Z20ZGR9k^g^G_26G8xq=Dq(8N~8LO!IC#yGjIidUSivV=4O?vNYijqP>AE-{U#A-z3X%vT+12USEpCGD>@PjbSYQ x*U3(BW)@>+pEtA5n!Svg>=HtznPp=jNJYLuER~&f*6WjQb2>eaz`ou}^e@?<7$X1x delta 3059 zcmai0ZA?>F80MT_prr-+L0fQqTmyo<2Q+&W@tgq0}R%t)Hcir!()MloZ$^X%@3R-A=pB!$Mz1Gh$(txRJymSq+Kj z#VsU05icXLMN)u{CIbANktC5SwFjBx>{saDfem$4!jZ13aX4c!Vu1%Fy z&TiUh@9($wu)Azd77k{cwXDb9KX5d9~Ow|E9zIh&9O*5y~Qhzu%`Z3@fg;k@i( zHq=eNLQpQcs^V)3t2r?2QCE@~JEg9LwZ=T8p(WG_OTZ zc?>U!_sRrkH5XAm^l2MWZD340j+CvmZ^5LH+WYd_kd;rqBRdCsky75EfM5e!EW?^ z63B3&fi!qXVSopV8MsxbMs@J8a1>n(x&@O($CQ)tX5BaD!3e8I-%;9}pkeJE1xg87 zHi_Ud>p_zdW_KeK7%e`9&|^YeM2d-_r$mZp5e$}$E5$hOIPM!QSb@P}E?0%<-!Mh! z#c;W_EfK1;OovO8qr7Deve}TDfo+aDxLS4{ncz@SV@@qU$?d*ZB1K4VGxIj!6&P&t zxo8DAbQbz>SfS^=B8uR)P8!Siht7sNg?wkm1bl%H^iovhjuT5t8^WJR3tZA$KyG{) z560-2n2Hd&;bUX9GM)ygXWrx}*ZeN*4ytj(pYs6)}>WFjM`xGEEdWeOIfyaJEP|ebaMsc(~KRQ&WP|M8djSK|W@} z?$lgIdZ=!SOxVMf5u#ZXcv|}=QktG=fZSV0>tDkb+JGK&}*sX>Icmv;h_$-7d5jl%o$rWm~#$giiqiur23)>P={kE_c_r;rt}m`ZK42ObN%b3S~fWXjrejzuK-LN?GhQ}{}p2ip=A zTxin6&BnKo@M$PJa(QxTg!l|KS`~Vg=YAPwhW$-H;R#-@a^e}ciD$=~f5q2gJgd1E z0Tb9aEtQF2jJXj{{|8K9H6aoBBoom4;EvfY`8(*$lZZ30*CH!R=E#kV&Ar=Pd-iUy zdG>9Bf0pD!b4xm$TY54l_#4ifY{MBWX5etq;;=_cGezO4_7RWNfyw6)Qsy#oFT{__ zYEdzm&EYG*y`wII3~%SJU=yHaCOFo*A`#2O)|KRbAuQo wL2*_dW99c-`MuTwE{|p@A-9BQ;~+>yKLk9LgC697H}h4OYo{}CZA}&W7jiy-BLDyZ diff --git a/doc/LectureNotes/_build/.doctrees/week35.doctree b/doc/LectureNotes/_build/.doctrees/week35.doctree index 549aec66688efcfd6565d5bca3d37f986bebee07..18e8d142da4f233167577d6048f26533e9e2cb49 100644 GIT binary patch delta 1874 zcmYLKJ&#;P5cHmHj29aTSxz8%ktG6;`g>+XKtza;z%N1s1Qv4Q024v52I(c_*5ytJ z2|!%v+~NoD8~6c)i0YZQW_1_z_RMs3Rdw&HpW0trd;a_3!ojiP8q`0SSOK-DR`W)a@LC9& z2Jhf$P0EyrLM?%2mmz*qXPLJ_Atgn1iW$&0AZ-jJ30%@g?q?&l7M*4~Eo*^B+Z*Yt zcQu+M904>4U2sfO0%%tel{WcDACdCxZzs@z!Fli6OAV0Pdd7w}ozckIb+DP62NWD_ z!IJ_P)mdtc2ORI*rWGI$Cd+t|lKxi-JTT^Cc$UzZ@gO0clKescvxjR#fa0hnOXMhm zV;?{h7-5(oXJ)f&*i7=faT;kM1&I@=0(Urt^^_)ArkDy6$e}H~H7;W^v41f3 zmNBQpTO)b1aX<>jxPgWJcyASRKGe(7Xse>hF-&O`q_7$HRfxxPOvw_7F4&BqfUw6$ zn4B~$tf)6KzI^YGJ3n0g_Uqw;_fJw_FNM|n@{men#VTi6Fkl_O5NJi0huGQZks$7Z zx)n;phWgd<(YIeedHUq(H_PX89KT5TodC7cTe#c*=}*7-1bG-OT+tHVuh2*M_~@}d z`|f-j+gS|Kky;>w#D0&l35giIkzmH_)$`Yf2X_0#y<3NC`MLb~t&0a&fBt)TcDP(F rF79sY#kOAG)*IV;b6elq)?3^9_O{;M*5kImv#ocw_3n%H-M{|>+m(_| delta 1474 zcmYLJJ*!<+5Pj$6Ma@-%qzEC^Z7saa=bjm`wG_dmlN2dJNZ~7tRhlb;2-fQB)1 zTwY(?y-Nq8h;SN2Ny9Te{OYmoUR?k5YWd>E@zKTE*;nwwHHXF?M;T^eIghT&Y4gaX zq~5o}>SH}puw>P4o{@XxjALTe+OM}v6(Tb+vr1VXQNvvJ$JJxFn|tSoV6vGSc!Gpy z4h^#gXF@445%Cz2p|i9AMUE|nl}Bxe1iZ`?Lo#JLt1%!qxFbi3ca9+gpgV`N758-% zy6JQ}6jf$s3$D>eEl@j)5M0H$Q;3Rq^ECqePiNer?U|T|1|ORa1ZHvb9OUfQIaGL5 zH(O+$9SDw+Cw_8G$)1`N^F|j5nrcGL$6)GC%1wMa6EVAk+at!E>e+NqxrWRMf{AkJ zR#?cT`$kLbG7&)xipi~Ph?`?&3}iHQnr4u0 z zn~bJI{a8r25MGPut&u^6Cu6-esE#+8Gkms18IF3>f!Od?^!i)YX!fyl!N%8r|GxZs z|Hy~(iUH#_vE9Qb_Wau|8u+jokV|M8f}h~aDKN9B@DJH2 z#N4mt&u^CB^y;Ipm)lo=ouA+P^YZ-pa_g5DHZ@5i^xIqU}P0(KMj4(wgnd$3!u R9qfJB2e1#X-Y&QQ{SS;PGrRx* diff --git a/doc/LectureNotes/_build/.doctrees/week36.doctree b/doc/LectureNotes/_build/.doctrees/week36.doctree index c117f6ca184548760dc4c521ae3f540d2581faa9..90607f78d835d56b9936037b8e87d9fdd2d009e8 100644 GIT binary patch literal 284049 zcmeFa37lNjc_+#aDi$M_#;m@G#;R_*suv*;0)dePMi>Fn0!<{;uI{?sRi)~xaxIb? zy>>j<@z`<2iHYpkzbtmhWU(h6=aEb%zr2@m9$sf&oQ3E3Ws=GANMcXo%*zbVN$knw zm-qj^v)?*Z_f}Um$VAxC-S?hz?m6FfzVmJ8UFU!Jva`-Ui~ec9p;0RoD<`tKe7;i6 zhmBM1i}KZl)^b>Bp8AnfWB=mR>Zx>lLp^uAQEk;1!c*=1dGLT801>b>T<4GT*#HnE5TA&3G2CLm=6}~ z)#YG!b>XqfLKuk81?fVwS!-;coLnk43$6L=LUnl(->agD{yCPCPlCm29Rml2=5jeW z9)?G^+>y;!v&|FDAd{(7=azFvtN88S)TzRSPoFAmXrFf=JVDDm9jEyHrx^By^9vWW zFR0}J?dGY%C9U~fJC$oT>&5w26WAi`oLkH{fQj~n3*}s+frt40{7Pu?B3?B(|^$PtBzf;g^LB}&YM--^i)G&itQM{TDht8!hW3RGKp9K5c&~5)QF(xqid4>Oal<7WK4)(WdVI9&j z-vT9o%6F|Sl&jP-cqBJpg_y0@S0;kJMR4j;uEIPNS|4Zpr3)Ji7iegMf<_Aah%XE2 zM+@VR^0F`BuP@nazRI^^MGyX$`WtQkdwlqi{o(FBLfCl<*ttbvhbN4wZft+zw1>F? z!|6ZQC@q9t#TKFDaT_J~NZI~9TDIS~8{7W&V0?UkSZjvM^I<)>J+o~hn3J;7{RHh~nrSC4(AXG}$n-gjnn?`wSOo%W~VY4KJ?i#$c5#VA)fbB~9l z9#BE@Znat^<=>(nv^PFp&Q%(u53wGE4O1=N^F}Yd@MBo!$C8M_nU)lTA7OcL@qDXX z4x4jjY{-^SQ1gPEq&IU#VA=E9P{hu?pon@Af{bR~EI?O;96J^UWvNUAP;Y2AZv+r& z2e5hPb3y_>8Rjeems;mAhBAQpB>?7sO^$ye{M!|X{*5I3pI6vXVEMHBpkUze*#6e? za<0B2RAOj#O@T%%I$vxow1h^3onrqJyC;Hu#rzT=1^wm`=zrCK6e{RGb*=kk6;Osg zu7S{(0HH7R2cbI@gkBU7>djNX0GR3?pOrA}4#K1RLYx(_(dKco*cayV*bcV`4*`_L zFbJCgXl$X7gWRbHTP8qOVQ`N+Y5;ymvyECEJ;FMlIOeO>qm$E9+3BgNJ16hDb6aLA zvt`?znQgak+qx}tr_dg;ZMLWT{-`C@*3cO+|IaqeNh5)FAL6h8C0-rhphj<7>)I6M ztbG7OBBV)t!=7$;gtsriGFtWGsls2!=(mmqoYvOS_ zAXVgvgP70d%*A3ITwAMGOHj%c6#MI>Ay>K(hxNip>3sTOX=2^sr*_}%5@@A!WY;+1 zPF$us6(kgTL;B}zzLc`AcMtl8mX4nkxG6+9Ic-V9sg7NtERK`B2^3m&-UZBtf2RU> z{~Zm;#|6mKeS`cS1@c=p$P0al{s#{5!4$TUA|zD;#|T=+YA~CKz;QwU^pniu)l9GyhVZGalN8=Fz8N+z)(8m-G$y-t>zcQa=Cgu zy8RIwK~T--=8GV@71FzKOnh3K-i{wXp3R40O(|it8)mEZrO7b?#jh(prv1QBzN~OF zq|r3ihBm#CQYA6s^-k8lS?$4H;dx5!bvqT6loC=2DV(~~Qo>HeVY*Wy(mJ#sdQz?o z9dmE1UMs+qAWcqU@B)mF2W zoewA9wlFib>%q58J#*(%9ej9sw7+7h&KDXY?ap_A&nFYZr?iTu?Hd%4`{t?X`#}Z6 zpGgQqqHo>l`GlQ_zICTWfIoKV$pQ|k^VM8EA5_R#Z2)gMAa+%kYtnSrVeOtTSLY{} zam)zoZ25i&JMxqDViVRtVf`lS#w7m=jpfd4Ex*`_zUVA8`Z``%=<9d~^!;X^=$lvQ zTffz6B6vsCxU;BO^`*rjQJ|OfoRyh>$ni6f~JSp)h6y zwPcizm{EqhOjB6tGJq$@SXt=|A?u;yZGLhnRwwMCq^C@gy&{A{0cCzN0371l2msTx2)InJ0uJ+z_W)$dE@h>%K>7PEFvu93pscW@&~PM4AgW zoPy|9{~pd)n+^T<;lrsQ!9UblLV8;E?HFeR{-L4ths3vMsB83_(Y8FkY$U&PvX+^l zEoUmg9DB95=7v7g*uoB{oNY;OCTHNMaAU(e3H&sb^Zxah;H3fgujk*ZWuzf?YyY}x zIm}TgF{$AT@GcyOJr&W&6vMr-?SHA~So#Oz^^wwd`eAvBTzmM1PkUzNuweUXN4bhW zj&@h4j?CB!5S=p3@o=5LqCv&zyztzC(fQ(6J%F>&8SZVBo5fmrg;2^t|D-k4i%SKB zIgu-S)P==pkZ#e5jFk(Dr>!nkVF;1IveCw4L?uh1h`AsY1HP)>pR;zN__iRWYXBnK z#^EX*m${QyWEy?!LOqBKnMk#G=^r$l)s$GZxUna^tkuReT%VX7F3Nea@oF+I$RDHn`s-ojDSFIq^u%h^cGOEE7zyy|eNcmug~20(3xk(8 zoukYJ2Al>9?Hk!tSO?DiSe|l$K)PM3!k-dmnyfj|N7byI4~3YX=HAanZLMEa;9Q z#;ICwxWe$*Do#=47p7$!QK*h-QCXNm+7LGli%7FlBg*dj5NMseJE?e7x;1zJwuYsa z5RqF6=KY7g#yeX0Xx`|=A^`;_y)NssAGUZTIF@!@pmeAq0wzbDnZf5?jk=l%tfA^GHYQFDiG-Wkd5tI z640F)C0X-W(A;FxKqnz9iim6Gg6`*s3A&9Pykr5v^U||q=rcVKKv`jLgEo#^o5lbBKgmlh3 zoQ*>O%xoUBe+^r}9{B&TEg<6qZb0}fC?69+ES1bNVI9GvP@s)VgG7+CS@Tr<#G#{VaF3+1FxhIvyTL9b0F#l zCZI|rbkN~us$Z;I5~u&ZVD>+V2t;=ucJ`s!RV8zyk6r%wv&fK%_TVc%syRYUiRIp9 z!mg^w*jjG%jb|}F>8eJzwznXTH>|J5xAn}!1!~-mC3S$fW-ieGX_!F2+RN+~6uLC< zEWI}htPSoBh8tOwSYPXS<@t^aVuwJV@Y5|GyCA&&JXa+hh;HB5P4t5}dkzVt4rL{khlkYWB}BpXJ+upJay^@QzVC{UHPQP2GE4-WowXK&KRi>8Kb=^r zg*lxVWxM@2MQ>}35aD=)P^sb~x70SZP!Oq6n|YtYwv~E_vY4S!3logd=H4>s$kyry z9}B?ya0EHNO~upZ4bcT^o@{r#@_Z#8Mb{ba&x@kB?^z@_03yp^FC-nDb$1To!s!MY zw=~o&RFCD5Z%*-J(j1nq;Jm}`hSW}ADJ*Yp2vwa+tFXD54j8Q2k^;VK2w$?WHk9E2 z(oZKqoTtrAV$CnV>tp^^8$9078#vr^y(r#Do!jIBQUjPDz?gJI@SC~G+K~cxFJ^N2QWf>1$Y9+O`00fe!61Z{8#0BX zMN_%?1}qxb5;$5^!6PE}BL)oX2tVC7`Yd^|hR%=an;@;#y3b=5a&|=-An=>Zng4y5(>*6ho8jB^muG6ety3-9vMt<0WVlw&vt}{a zIDOCGI+!c1rYrkfvy5-+naL2Sb>#dt9NoK7D+lTM<(IY8UadTf>`g8fxUcOhBCU`2 zXyo~}3*hxJ3JengmuCZ4*jp>_d>6amH63Ff80Lb`r4U&yu-g2mXWEHr5 zz8LvrA~5bYU{q6P<#vNvt42}VdgxhVZv;@Vs`W8KfX|~I%`%MQz;T)%bdJwmG8pX< z5++;5f5bhgd%jFLo5qX(I$M>d2xi(pVP5J^Z3%LN_T8?T*${@sop%d&iasfK&7Q0v z)qt`mE0jGcyzH7+6X!7MKTeQ4lxPCTBEt&bUWIX19aDq`W7!8|-7A00W+XHs8hBHjR5ebJo0 z$WRakk^GeI!LXDzVudMAQC>&%Nwi5cAyHfW&Ooqya;*^5l+OIL@q&TvoAqZkcNv1W zw!*8?B-bj lP4A zO4v4MxSQj5-MlqAnY<vrGV6H>+PyDAyY4>coTU%?V^r|i1>r-HA$zie+VP(Mdkr)-CEoES z`-O#`OS|u`9&p6(p4P;TFp08qByGdy9^z#uVdJ2iut8e!tIFC&&;3m5yKLSX_sq(< z!CCo*o6b|mQiEK>%Cq-T46KSc)$y8!LpuXbI|aE2K+P?oG*qAhsD&@gA4iK~B4vmI z@-w556zWSsG{9d*5w{J*#2wJlt|=?|!Z{0*gmJxUGVdcA9qXNJC7*&7q=Bndyn&_g zsNs#Qbp`hxRz-?_y?WdS49h2Vb1Wc@oM|U$3~qi$L=w5eEzX$Xk35U=u9*}Q-7}F- zCK7%xKA_1=O__z?l?+RlG|czf-q)UG`;3X|JC+-0H@krbBsyz)!x^vd^;mw2i!BbY zCgwc$jPH`kn)bkt50eb8?aHmzqIxunUE;cM!VAL$!FiN;XRTKJ?O9lFTDNH9?g50D zHIada@yx#m?In%m{H7dL5rI?7*b7nEWj@62feMNt zDx2z!#L8j7R(Py0tfBpUO!0UB;LXGbh|XnwArB99^`>&sHx_ zI7Xint4FApn{sY>!IC%^-@J>{z2N1d>Ynu7Jygh(YZZcZ_filYtntd~m9)7&ca`oU zjE$7;#E)J=-ERExQ(FFzDsZWcN0SLKX3g_#&V6j!< z;9Olm78n22TIMcD+B0g6u$4!g9|4%t7jyD_*CjlVM2cIfY%DPdEN<ms+_cx+EA^_@Ww9Wee5x zFMW^zKT`UDgg@7aGs8<<6Lv4h9>8S^?J2VKly^>_GS00tC=4VFD?jKxjWTPFIV#|Y-pRAJ~ksu zoy>-TzREe+ED*mL_b@k+!w}L<&A7=q-G1WwLsNHenO@Ee?;nh~d`b`O&Y9lzDgt;w zO)kKaaK#z&Yv?vxPO_oe$YZBwZ$Paz=AwY7KC_G@^+VH7)5u7vuZX92EZ;+fz@^H! z2V3strnfJr$EaEA`a{!qZ<&!3nWjhRwPoAgTkoFPcK6iuvS2nmy6f&K>)kZHV+eP= z;%>P{$8*u8rk=ZBp5ZUg>TlmZy}W~3E`zP{ljnnHS$<~N+}6|8`P)TGbQ297`l16qYDofq1ppvOsU%xw`E$; zddDzl`k`!H%N$`ZZSsZ-u-jbZKR|&Y@-0U z`dj-v8pZ97)sa#lHr4Z}4hpLj?-3&^mClT49Zl6oqJ#p+6#zUMiUW-p_ zjji-`0h~XV$WkXC7iAIsiUVF=`8BOvDab;(`hJX~>hTLQD*Zs`qaU&!&3+CwL_AvW zEo{&bXKm!#ra;&9gcREQ$F_5c;mw;j2M2OTfeENVtO^2SLUWdQyPr`b#2z)FzI0O@ zjL%5lRki||a+jyOensjaUZ(M}u`GXwj1;gnU}X4aMJ=A?D$K|YW*$1@3y-LtEEyH5 zC@)(8ZGcErRwNH5=0_CMSb*l6#a4&bL~X*f4Jv`%W88wgkw2^1Pvj5w-`sz0f7zVvD_-BpO`SeQ^$#em@B4wi*w|n z6IXU>BFON6XR=dIPsk@z^cYX|1H2UN^b_ubc25UTlx_(gwKHjD0U$xEeH`-yUS`Kq z_eVi6OK6zIcXrk(l@xmua?&+frgl0+Ly!SG1h;GFZa)-hg@<3uM>`r~N-ezicOSGM0wIwN03hYgi02y5Ay zsV<5OPso{K?w#Wv>@q+PRyK4G+QwP5iyjd#kMWZv@$%fb$M_s@J93kZXLPgFUapa# zt@gcJX^fU`zKC_~H{nMVo>5hg^0KbjaK4I}yMen{igV@uGK_bvr2?zIVk@(*lZ!fX zA~&s30R@VA<8r@J*`BaRTF!CMfYQ{A$Ni?>-?dEWaV>G!$PhfNaBtVz9f_rJQ;UrC zJFb}qKQq)Fkx#l@5y66TgkEJ3p{C4Elc~PJs3+6zzpn>ihKIV@)~%@%4GG)2MMpnD z?mE}M2GSsv7&WKKR$hEYRJjCK(Njrjg41gei>OtSN>pu@4tqE&60Oj`d#gD1WgZVJ2+R$oqaEJmMcChh=5e}1GauH- z6a!N>E-y!2C^E{Zf?X)f4pE7dCmLh0w<@~E2c+Q6ro^{lCP4gvY==24K-k*_ryQzR zg>j-DaJ~8PwnU@(aA`>UguRC!BW!A_O(yKBIH3v0c^i7LWizS;&|Zi_owmTv8E8#Io=l;GLg=4gTa`Z z66+jSbi&tsuI+mpyQmxN*p&NMUHm8|w|66V!9O<41)n`;tq*^AmJ?6^7BtP~I&{bXAv8LN zy`!+e9la1Fh(A`_DdYm`)GgEzWlRzWb=zTaq#L1>YRMUU&X_VEnnI39gM@J6+bWNS zVU1;zI;F@9e*+Fy*0u9ReCAk|^LZBGhN&K>4|BP85o<;|D-;7;#F1`F&!oDmP1p1c z0@Cc>7l=7oB&wc(PCp;v16P(f92JF|m+d+lxhVfiM-$!fYRXYN=V;w(U{ zb%a>JU1>9zBn|~H?>iSve>zMsec*H=O1;kZX*DemLgOX2MVFSuUW1=&7SRf8^K-?y zd~PAlz6eJXWE&qgqJjuQq{04|JdF78NCcDe?^}cbgnS831QqyRgq-R0)!d(L6mqpt z_)%^_fz28nNka7Cbmt0pk4R84%AxJDFJgBl0_%^bg)@R3J^AjA9v;Q#>){tYT{^q7 z2b4!gdP+oa4IDaC*)0>nRQ7iA(@bTz((iYO-|wW~w;_yXDtni3?!a*)+#T?Obn&c^ zm~OOcVLe0sb=>d{+nfZY>vK`w6fxjV8gLh!j%01192asOYu}*&E%oT-eA$)RX`C-- zXzUD)iJ(u=>ab}{OLaa>{8NlxM`4~`kHG)b+uj;H0z2QSwS@1;aiYp)za9Z4)xB{c z@4LSdtW;a9gET^YuMX@(;fWmO76@xK@vR#^5Y^Oz+)_P+g2&>rL(QYg?=oV*4j2f= z#~-CY;PG*z=)Lo`osp$iz1B0qc7FW^*Kl9>UKM=OKA$0{)G_&&CSm=0q%;~Cqo=l7 zcnbcIeLtg#P)&eURbM`mYM=sZ;uQ{SbVZu2dDE?ZqqP2!N6ND33GdVi%QvMOvVA*h z2&aY-my~hyBymYyx38FAqSTsv@W}o9_dgcw3qj$yk)T>pK=0K=W1{ph!GEOm5Pn2i zIpPN4+b`#TQv?42?qWF+{#-_J)k6e=!qD16nogMtt>oqrZpK^}o~X%LY6C06cq;i^ zJs<2vDac^IxOy_+(CZ+*cmD$@*U#xjIPbAx7CA(G2Y}eJ7btvLCl3)g=xj!A1G3m6 zqD-1`aeEgceNa>#rHIf&xV#!c`msm$k4ek0P8SMNtPIppD^0mQWULu1(+piQQ+?@m zvRX@|yl$2Rf_@?*S9~59Pn+v|pDnz_}(u+|Fe>iqo4eXFZSPS#deSM6&R=<4IcCp&LIURll9 z!@2cJ75bg>Afja^47E0U7&&#XkYM=?^pUDUo<7>KsWMj zw3g>i7I#d6vMZVCRY8d>bHxnmTbA?T$rZXTRMP2;bDs!Yg3ZJ9AIFfJgm87wxtSv~ z#+Lg!Ta_jSCfN4i1pD0m9>TDchI_R&m+$-(TRgUK_nse$EJk`)W=0l-(0jE~b_N|r zue&|yNMN5I@0)#|FL!8wHeuE|+;~U5dGatZNijGvS3ELf$=N}N z(STwC`PYLJ$O}K@S-ROT+t<}e)8%k5q^32h)-Cgvwk=w*KFqCPlhgWehpKIq*d(g< z^$#L_D1f3!fmN2;O_NdN*RzcVS?S9}7$c=G#y|}Le5`E0_$wMTpW`luZ)UGXrL>(Z z0i|3)OI2Q5Xxhh&`YVpRP`GP2f6(mNzmcmWK?X4s4fo<(vAe^&q5_f8So?mVndjh~ zS!q%LJ`UeeEj|SsnjF>j^U6_CN9{@BOwQ#Is7KhL2s*7^tKR5b&MnX9bE9w%QPKgH z$L=9y+rU?2>-ty@{7F@zrTZ?czi&YBSc>t$LjmbuCEQz4Za#%H+Q#)1U16(c0sY$QUnyAEr=pAdLGjjW?nW!^;Z$K(57g}9x5ves5 z=dow}AtK=)hlvFEG7~bDwb(v7#e(8O$*YP8_1`Y#U7m6Ye;~>`7jnT`4*%(yXkwb! zx;7c3SJ+IUJ+Ipt(PBs14}41Y7lx-~Yo%f^RrxkaB}Pec#*>Fi{V~$6Q;MQq zC#%ua!jgCf{LstPmf#fl(jPgBLB}cZZ-mK=%5GC2F7Nw7b1`a>QXNESqcU(QZ5*+? z?`Rb{P#3vzqkSL`hq0hZ_X}{8#%Pm*%u^`vLn+=m<`;Ya5=CuN?jEJ1Wl55uHW9`A z7Ap(oRv!6%Efg8bVb`xTkY$FnI!YnrrBHBZi*20^jxICi`TGZ6xFuYMFc{c;XGHw@ zM3bk@r49nDSw&uWzTj?HQ~lyjF<<{3T{=GyS$HD3|MGbkDm#J#3Tqq}DT1;fVV=bT z;q0bb6@GK4xirjacDeLREv1*rV%k_L%iMZ1j)p~h-9P0)Qv8-@(SzCY%rU5aF_e6@ zg)0<0&-4OIA0R}Jl-}PDzB$+W>xL%H9NfeCFQj%wN_$UqD(hqGVTtN@d!06>tM9H! zYee!U`lG<&fSc$mw`oyh&}xy^EjZ(_O$xRroId4AQ*4*Y!K6qoQCq!&(kBHe{#@Ur zc+P&$oLy4<+;_bOIIZ=+*Tq^i+L>_w?cjua#WNlvun2cPq69rcmMY62mJP%c_Ph!g zsql*Gn+zKTlrf+RJ#0{e7Z>&;zHuEfR=H-iE)*_kGOxy`H2g|m5^VE@zS(BePxXe_ zriWxiG4ng`CkRj`YZ+Uer!5yslBRmb?If+SC;eZ2uV~b_B3BT+C6U2+nhnNmDrQBI z495J6am@`rkOl~|z{`WPz|5CD2sPS7`+5o=&s58eOcMrU*MMwUjk#HXW^_p6X2yct z8KGg!u}RFv>D?$-a!eU!rQz5_X#@|b7zMS5B=4ZXS^6iz5&swiQDsox`8OJD|AV_& zQsYNW+Kq^v%!p$DQEP9E&#uXQfP!oi-1aHBH$%Vg-Ese@# zQv{kceu*3;x>3QnxpY;tMr&~qCNa)?QW*qviGmyrR}sNPCg#Go8S!}p&NwY?O=#M= zShgkE_)jq0j)*YvS)x2`HYbFi^~J>dV6)M&i2Xjvq-KM%jH& zSfZ65%Sz8$sWA% z%?Ld>gjFiiX$Qw(+7+Vyt8YbmCSrl-|C3&o`ZBY?bv;(0LEG-t3bdj|R7?_2E7o)j zkWIJtX4c!G#AhISq&>%YvNOsWZ%7G~!4Uybt&v<;?=swB;PhwtJa29h5u=*)+dT73*GSQBRy7NcBtv{wwNt}a=_@KM%+lQZ z(dmtQ;`E%HMi0|K6@VSk*7f3MN7M^6S>pa(dl0RSA_i2^2F5K_BZq3+*!2zM>teIe zu>-Kl9YI*l0$+!1BC)z{1WrJHB|Go&14NbbgFEVhvbbskljVCoL zmhLz91g`~wn1-I!_9g3oyq^^*S>JV05m_Ih2+aiT8+$kOEGk}bS(7q?%XDjB1w&4l zp_1=HHG`8Is+p-C%IHkVP>TC z8~D*nRw&)?<(4M2U*|5CFxd4ZC2w$$zF_}m0Yww^;k`>XG>qZ6yM(%tA6(6PC}Wvb z{*GAXZ_y7+AM4Fnlw$a2?s|Ru)X4<<6!4MQ@U#{a(`z~h4QAeT{&Nf^C5SYB1-Wco znq#CyMAotE{Gf<6q3UiFZA^Z2<-8CkBa{Np2Do-Jgm(lIkoFzgPEb9K-<)tffjX)gk={*(wMO?*>FggPSq|PaDj~y1;f@#v z2uExd1#%P4%Tg6ON`W%}m~L|^?eo*Mwvb1+93ZES!iB1Ik~P&KeFVzY2g!5_lSNno zZ_qtwlst#TIxZbHdCJX9bO5L@N zRj{*=uYT8SrjaQT5KhYlRU*;$L^3;~;Cl+3T0yi}zD1t_tLkq`X6$q}xa$@U>`|_? zNcRj^YsCfB0HKN?G))Xmwae%VAX+|85{Z0r!FVnUI^)ZH&5x|-g$g}%W8B-tUvNPro#4%sqsT_pMU6$q{Z^s>LTb(5-);a>v^IU|ZU18z4=FOEm z(Z?F;eE9sa$dHM){i_W6)s)0BA!XLdqpP>-i~sw9n;*pCvg1(AHb>j33nop_ zQqR8hM}*pu(jWFif}F8AbC+knPVC9-0!#w26e7da90Kssh`B36_MpHv%M@@>l&24# zIC4N1E3@Ca*GR}p_D5#iCPxnVqu>JDM(G5n+aE>Xx;@;=>JAZkQ{ozG6nSI6ZmrIh zqJLw6#d>Z5Wu=Ns%elD{ib`3(tGPMDSb=wEz|AG=)*sxOd6@P8a&Xq)`F0O0S*(9d zn{<^{9Jh&*wXoy((? zuQc4w7cQ{Rxy&{3BvN?x{YwpAH7#N1kYR@rm|1HZ>C)oEx19yihJRvL>gaDt^?|F0 zxyl=Q;oMql?{@tyG^zfY9_B)QW^RRVrNYK*Jr=Q;ZP;{cUn1^J6?ReU5J=C>fvqFp z&78TCoc?aw1M_Pg1>pGN9->dR@ssmVml}<0rg^8dmjFFddK-T9Qd5-MbN=6Gj(Lc? zSimQ1J&MJQrnSOMPGQpOx|0C8kr6A-j_mgqiY;KKnm4 z`KT!|8(-zay!l|;|K77O$g{Jf<`eZLwfCUS6`e$7ea-jQ^Q|60OsZh~;xK_IM1Kq) zSj$Y--m~!DKvu3K7d*EnVn4NmBy8@0Pb|(qQ;uj8ajLC z>LTyD?vhMltXC5kO@B5_H2HMRD8I*V*_K!`TeeZPW=F8Ak=3VxJqQ(n*(@$L0xjc_ zJ?5xgV8bHCr&gNvBBCt#ehy(CLhMij)%})hkU_Z$5(SV;gOme#^9RKUAwodi{$YhC zm>b0NRPI^qpo{n=UtP}PzG%cosJllf{7uHhvF%^n4lPXnIAkI(9D6BZ_KT7dfgAwm)^^Ptlrr z;8NCkZ&<^j^nGSaQq$lq>F@rTB2xq804|Iv*-!pKzh#jIP#k6v2 z*&a5VV!T9YNankF2^z^vX73uD%wGPw2jv!-{URG?v3-%K66T)}Iyf|QI%B~41l^uq zk?=AvaD_ZQw6X=n!2x|nHTrO-NANM%Oo|SSvk1}>mT2rM*C7{Y1RArjf?Agd`xLjD z1WQ$JgcxhC<+Nh7{KtnqbL)IC{IQ{_{BVR{-IK)z4L7wjSWjHC_*i71MCPtnF1S{Y6^H4=ZxX`Zyl|A*g@f~z&MZ`B8M==+T0UYgA;KU0S+BO13J9q~ zU#PT%6#Im&;s~jZSyONo*7<6FQUkK|MOytx>GS>2HOHI$ogr#31iwLwJp;_-b+ENx z2qemlh!*L^&FD|UC=!-oai|1O7Ga>MAoOr^D{sc>LUio@xxPzxi|V=zx=L`*#T=^hd_{0#q6{N;OtQnUYkkx`?$JR*v2^q0EUDjwA4!ZZ=WNwV#|_-Y(mdauu&i*WfL8k)ZE1_u za=Ch({GH?Dlmw4@I_g$xEfgOc@|P?Y82Zql#|**Pvfx-X70#n2Dh?HYXxy~r%aPI@ zkpa4kL=ts$wflJnPwML|>NXf)n(tT-J+n1z&`s^JK1M|0vp0P$)%Ud?uVGj>K&H8o*kx~NvD-8)mmDgFRCes$2W{AL$b=w;&%Xr51hm9S1OiQCT zC!4b+WRms(H@-c>`tBLimtmGT`K%Zjx;tX6Ad5}!)(BKnW5H5;)xoc6v;FFS+k=>R zF=>IWo0xo$jNQry;tB~SZT@xjB%X08?HzofcYXO}W2v$VGd*0YxDQE4OZ>G|teni% z>$#QHlM4&0!F2Fu(J#0+mv@)55Cr;*eG}+BY8S}rH-_n40=P!Z9byGxc@O-0MpO-novY_SYjJ(cUp~96saSoS^-I3|D2`ezFeYms zyTpVAH2!LsOTBvlTDKrD&@djrSXPTmY=%-4;6Bu(fLt28C?E#@$9{mt){JgM;S11I zRHH1b@p`#6y4jmdw@A(GGTi}~F(NQQ&1~)hL8o&Tlmzy~TNwje5)6jo(NCm z_$Rzc#d5ht*F2;C+hcX)4N#eCl+tcAgZ+FxSYtv;apNm7HbsTYp^vM{p>)ee)^Tsf zk0b`i=T2#Pa1(biJ6iiUAL)$b;*wuZ)e7u*OSq;8k*_O33s+|cX>}FDn5-tDqq`_( zm4SpJS~)n~pe4(+NzNUiI$MZrg%c=OuCj?COsR&b(rw2HsWSvTPK4~TtYA-@RlRx~ zws+VnWW9@Ew6ddS#6gG^sfg8_KO=XQ|2KC|HsZ`5h#;p-3p0p*IfY~FqJFf;y|nDJNXho#o_+-;cT zzv|9iu%%1#PpwS~DQjX>SV=i$i>Y<|f(8+@*2^&%O(4f!e$qkqddjhH38cQ#H&S1D zr)Q2XQt#>}$Cy?{1Ur{if;?M2sY7qAl#~HRaFKG>6$5WB=lrZ^!Ir$coCS^>^cD zW08}rhnewhV#Z_iqc4$jl{TfK1BaJ4N^q;Tw{`tLbk zx#jtMZuR8MYVc+iGg4f{9|0QO8_1d&rq$uYsbJ0jV947#M)Z95o&D*9p9uIan??24~uN1r%`z z>UG5j>bJ>4T6IBqp$h$F>tTx9f_yvrso23yYf6T4( zf{j^d{XYd|@!b*O=^8mNlz>sPSpV(PSvxXF&m*HM4ORblnzGa+*g*NlZkVVbwy02sjp1~Bq(7cegj6EK%Y8O~a5=eq<;NQB!j z408!SZ;hhonuH@2b* z?>kK)rN1K3j+Fije)O{Ssw3#Lf2?J|UvL*|XW8TWT($Q$$mfvLhy6lCpjTT>T&&iZ zZ03;aj6FpAmB}9^g2mO`hd82H@wdf_|Bin2b%cG7yLR7cVw)3wMWgHaba^~WS~Tw z-}4h18fqfE`CS%Srv6kr`o^;aBQLsT1zDdpNN^55?cEN|&34e-49+T3AUhli5Xdfn znd(A+!x>BifobuM!D;b@{TliP43_A5p8VK4VMR%v=J9H<80MNS=&ZttiU+OA(Mt7r zg%Z3)F$JzxO4{UBwZK z%}`8>^TV9%qeG?;4cGRtNA`d(UiMfxGE8T>wXa+Wo7erhTbOUt3xS}Bn}$4)D| zQ7DoqI+q-PjzC3MsE(9Ewfj&5vGhK|^+@Tx_>n}ldFiJ#NxqxASn}loa_ISaUPiq@ z=8#+w2WQe`)V-AI@+QmKH>A8VsYK;7Th)oSv@;7{%PD9ev-o&EhL#ejRWEU=YY90EH)qK_VWHKkt`B=bVwB=gGOc;@eFBzu#SjH@1#W*p3PQB8(Css;rk z_BRGc>^awLRQfW>j}FA~SoOPjn~b`cr~ z8bKE+HWpfq21WhB*d>$=(x+M55Gn>#>{sDS8hWLd1R?!S--L9ADiWVQ!Mu{@gvEmvMJPy{5p5BHkBK^ zQ6f@#RhwJyMrK~$5%c5_{4PeijwuaMj zY8ojlSD)lGG^Ki5p_tGJ~jZE*WO1Dy_(WBSPAs0tLaByB)0RHJo9(Aj7C2s z=589oVYY4cxD$i+ z7I9ezNc z2a!=r6 z2F5W6P}@ZxJzNUBCo+*lr}+7^uTUzFS_q%wuk?_E1t{<9SYRZ~B7AI^3-S31)Jm1F z018*fg(jXho8pykxjn9$t64(x2fY+Z(r1@id}WCc|D zmkY$zVFIy-g)B;St}sc+KR-Oo$@be7yMP-4Nqd7(`AwvxK06IzB#r#irwEoKrJwHyq#U3f8o_GTYrxdL46Bl%`H|BR zy^)di@=YX?-keC1oR6-352d>Nv=tfc0qLZ)xFt2 z0f?SE<^ho<9kvt8H%U!qUKbgpJM~$`=brxw4MH^umKhgPB_o1^{zh9r@=Q3M|E&*oROb z&A4WI#>|OZ1}C!@KCfY9rf!iL=OmIIVVuPPp0LepI=YDHh#FZCHO~OjT@mzoX&}y{ z`AB4tL^SZ?H}(3|B$x(LL<4de8mMjisb|{plwf_%fYUVqDHKmjN^lG}os`h6nVvLL zLO3`jymZBt>WFKmZc)M$ri!jHR4~@NnKS@lxx1t^}UTfQ!?DKxky%63qskOq>5@nDh48LZXn4XMYQw zhSp8D_O)d7zy^{b3jx_ZGAi5L3e8cCI*lWp0XpU^p2S{|5^-;~FV%je$)a@W2qW!c z{77OExZ*RKiZ9|W7JL8cgV3?Wcd>O;aNR2W8^Yn=LyzMuk~j5G?NI1+d7weJvQZ zK)AO4v0&7PyWWSrt-}OGhsQzzO38ejgU=%pww5pcPP&}fbIUN7bMJr*rirdS=tv&$ z80p;2Qf3h>NjIP2XMP~VkI78##;2Ac=+*d?2DmtMc|JNj=+6y^8nLA zVEClG7LS6K7r9Od;tn~Sf=m#&SP%6-aho%%D$RNk@f~H3xQnDqOgcue96~3a3G3B~ zpn~Lz@`~16wy*tj)($ktC?Lt7j*z5#K1jF?|3YMto^-8JXXoo08)_1)kz7Q_PXeK} zNb@Y|l;gJA&kl3S8*$}X1j(9s+n$%6MZ|B27QBooC~RoIA*|rKajMIkpDPz}>C=y# zQWxrBBf+)6>B6Vo2NuceV?N#D;U4n&$%LR*mntrnr(J5HG+F1FUOew)f4&w_FBt0h zQ`tZ6h_2cx>FS;Iv}e-dnLFvfALgXbp0H>ce*pi{$;RW~g2*3fi1g&N!Y`mEo7L?P zbhkB91TJbVLfFG52R*UJ8J6rtM1Pz070A>-+Qp1Bk;HOxud6j9nh zBjaEh`9M5LSRfhEAn6HACJC!spCmy9U|R>7b7DxM6S`9#=pKwLzUL|wNh}$X_#VA} zoYH#O@l?AQlIWCdJ|R&U=A?Zj5uI&3Nmw9xRD+}^FqtH*ZUd795s6Ig^on4CetEFh z`$i|j{oq=lTQnr`e!c$QB%#{HkVGdP`=rDthB;}Uc{DoPc#^O{@?H&+c#^QX8ItJv zwbJj?2IFiwMpH38+o>|wfrvzDiKksMSd6VRiN%#uRThhNgl6rsk|LvzwR|uAF2M!C zO8C+1%&jUu)IX+G*WctWmPU2ko<;W7A%q-f1rc#Arb6qTpm#d_O6@v*^)a*l6EW*Q zq92w)vNviU2ZEaJ{C$Rj>r?&%zFoepXB$0D)&aj61K9-DjAwtt0lJv91^6Dv?|&44 z`iH&&_1rf+-*g>|*P&RR=NN6Dn0qyG!L;Fxy$LV@n4Z7vYGpAX+Ux+6bjht0!*X6< zFXukgb1Ger`H*HpKl-Au7q)t4;R5bWkvNy>JCq4x_2f3Ipk!=2Jzb`>&4YvR#lxP- zC!@5x1i8(=TJ+w)BqV9{_8liPz*cg8*w*USai1A~L>Ww@)QH`PmVKj976KthUg zF}o_|s0wzV09BF4Lf#*hJ8os-$V3&6ETJa(;^HC_RgkqqqPD@e9ieucy3~t`u~0%x zSYIe2ofLkom1;{WD$o$#*ln8T9MGc;xa432>8|pV(3El{GDuHhuJqtneph2aO(O1S zq}utF9y!@KBg#95IoZt#@ou3bahC+xNmO*}J;Pk^y#o@iX5{vuBYD7M)U|F78jI)* z+na789-lRzXtpxP;~SmG<6qbMNfK>uOx{(+dE^vZDC5Q0Oe<;y$b0;hzSTz$4dl|N z2=_=_z>g$4;yK^ZJoz)+#o9o=@et+HK@+3`U@dVuG%9#>UZADo6msQabA|3nM7<5t zDUsRHY~`sWWhkp5$YLVd;L$24c;JRvPTEuICd4OV00=sw>=;rR?fgF}N+F2qA8ppsCP3)usTNVyE=ts^~aIcyfHdC4nKL)QO88uX<< zpp}l4e!m||VYlQ3hD;2!O5cCsS!cPPhSCpoKKdc+(d^>+XPtE`3IWzofETx`LaLOJ z#gs}RBf45gU{XW<1|LTaso6-=uBk^gz?*bBlm*GFXJ)8uZG{Hq*X3p^h5V>s-weu_ zQ{}}8RI27{!^XCAY3h22f@l?#g3Dn1EK;!3vLPO*mcjUVFf$fRcD6tls$V9%jE#ww z{6(ZgkE#5N>*(7C)z+etImm;x1(_F6w z)8?&%qRvv_Ao?i=02SoQOIb8c&t?wR4A)0RhRbF@L?{iZSbmhm^CM;OBj<;b1m zVIKR!8fxoNs2z`_gn_Q*O!GXJs~4+?cQe~B?VL!vY?blm8F7D|E*X&8BLEVvgz1@y z8CvY_Z~>MjSuw|qA~prm4O)#bRS|zS(0h7nDus1CE+8U?BZqV&ts0HN_C_V$sWk|% zTwQ9^atmQP1N`CtrWt{!NLxRsYbiAS1x=Ti=P`gB>QM1%re}shFT~!4ZsU7w^xi_n z;suU%?=c=M2Gi6};YYw;U|f&FgBVy~j+n5-vBWF4B{1tbIPKLu9tDb7z6j*8xKGu} z&Fo@v34E9a0pSm7kroi0o=Qu|j7`udB|0X8&Gk%XGZJ)j^SH%uN3a?1K zg5Re%12=l4G5KyqzC%y>q#Q0G^-oTMs+gz8r0#o0;+~PX5JG1p?iq<|ZK#&C)3Q=Y zx}JOM8Hw9N;%Y^k4by)OExVWDpm1y9&844&a$Nd2{zJ*BAH+|n=YOfybCty59_30u zjdr=hjZdGlZU`*BpE{2}d_^##jI&>t<3uM>`r~N-_J)P9T%K#L)WTEk^XTMqisthk zF(0Mzv(0z>ZCI3L+IY*8WYvI=i!2L-iMf=mPHyoIgHTLZRR1lVrIk0{MV zsEu!boT+GF-Naqiav=QZ1!tW#^uym`9Vem2QM$(kS3`@VC7m;Kb#d}Kb2>b8I{asT zI^=D?=MfR73r;BO+32)7N17wtIw>_a(0ZIDX&w;2A{`G0WJ~g7`C%GI;EBaza}H(= zcqhYJbH{_Zaw8NfnCgn7WivfHu_B(TbFLayx02`B^sHzn&(ZSp*@bGjI49mj5b9e1 z4^D^Sfsea3sFI#}dz_wmVm*)00xs)vgg&!J5k;S|U?PO!j0IDt=hoNVf@#YTVX2Io zF(V?)-{{O91)s!(+CLbm?nVQ!o!O%j+GS6>JxU+n*yyk|=Iqwh=rBWPKRzq3!RWB_ z!}=sRDaVN(Ap12s(23HWk97>c`Z&(HnR55r<+uS%?2SgX=M{722%F@1J$}D?y&OCx z@Cwje$ktIUuT-kblp+YdO<6Vi>7p@NIzo#dDILa-C>(tU@qPch2LDssWi1Es^Hb}5 z5YMvHbFLAVL-y0DgE<%8o@<7U=3JwOxR=zK)BAe*cHx~8uHr1Ylha$Kw{C^E_Jr6v zOQ z5uJ+`lcZ|+;udndQN#q2Y>P|e%|({&3Fa#JCRH#LbWz?pTLL7a7pw=j%0B?Bi$(jG8Eor*P=(sG?u^LdZ~z&PL{VI!eF7?LX3mh zv0Dua&)C0EJX&mK9w8&Hh%6YENk!4B_$hSZB#{OjS0bnkaR#C{c!i&^gOwJe=+$aR z6yDK{2NT4*LtBfo+L&X{O%2(rHNK&(Jkm~L8F*Y);;HG@OhHatB2CH3Zd0B>-00j$ zkIsMf{Df=|RvXyp#NY9TRy1i%(skEEh3{!$5*R%INXHLkmK~W6zYv>6!clt8Nen0F z#33j{3miG45F$8@;{#vc!G1=eJEIVuQ3xw;_@v(C&M1WGGYTQ;H~j{E3JS8%EoT(M zGYTQ8W8HMlGYVnXAYum1tF91g!z7!SYcW>It|sPXvTCl?#616J+QwOu<3vrP{hFA` zR_e~jGBJY#e0VC-r9B3_7tfgE=H!eKHz#JCX{Z@z?z8}^J8^o2TBhoRz(cdl z4tSc$bc1Kw0_fBp4in2EnY4`G@ho1lVg{~a}W z+x_>@BoRQ~b8)e4FH#bs9;;QHBWawm5Zw{bpRFQJz0+IaYREnBXNtSU_rA(BQ<2F*# zOuQ`a;Y30!j{&dG^v$$ddJ$%#D<%q9QUE23imk%{&HksHyk+1h))6c?=iSnv*NVQ^Im?} z#~RjZ&)JNUAfB0fu5(o5r~(2=U3_3D$hW35qyhV0k#c!rP4^8-iUyAZ59L{h*`a)F zPqlt5)DzniK{^53&mSOnhq*(Y*SlE7mFg&TN)axUoK`KfyM}((A!-u=@`_~5hLn}a z?)Yv{?}$RZm7t{b*+R8d&Qp#W+vCOl*sRuy3yqYxMFQ-c=koN6)Tm{#^X_5H)uONA zj0d!QE5eZ=oQ`VZo;D8*1#H%WhBRQ`E3z2onr=umjL@}-=ezL){yrnyd;~uGv2)Hk zi|9rZF5jdxFCWy9`im@DxEG1Iaa%Gr7W}QN%wPjdHFI^`0XB*26l=IqkvvGb`7*fU zIBlAg$t{Q&Ls80UwJx%43Afnh$eWW*K@)3C1VvFAELVYAh))WMv5D%tK@kvhs#tr zCl|0+UM80!B?Ai1GqU2b24FVzV5^St>&wuDFsis(S}6s>EKUwMVT?vxSUqe|zAq~U zbWvpv$?=r+(4tE-8^Xa)^OgHYhh7n+dFceTMgh6MM@cu))q}=ZC!zkW%26oTl|X0D zHd1I+MWL&eW&AiP%9-O53sOt?hKVr$;wM@aRi}+ng4?lK>_KeJ1F`fSGs4O$7b|jy zou!8uk@2tqFe3LQPm2{vE+q6?^YAz<2;VZ9k7f36miKjR z1+Ts?qPKX2nWxRp!P&%G#xO?fdo7k2!`3i3$6D(d&Uk~YXx9+Lhqa)g3^&Mn9vF%! zvsN^u0S8&r-k}OsYi&awc#t)2bIu{bS_XXvw^a*ssIvw>;mJtVFKekI=)#; z6n}drz80K2>zMT~50BmDeR7z~+qPznPEV@6_AxwQj6Voj-0VFFeemqF&O&iwGU=HE z#EYM(hxn^O%D=9k`Qj(&z#ihn0x^w@SQ2I+wP0BEC^bAK&Mkr785XK_%&Atb8U17>y_n8nEUDdE91KpiFf9UmZ>4#aT2%Iu?=^zsSZ> zRq%_=qQ)djbHQ|};+I&0icVGbu$0|`)D{70qy4{iNO&oaJL_Pn(;1 zQ;9X%xR#zd^pJ;11;$q+%j&uNt3C50MgdSuW(6RKBXVoG@ohZ|?hSP7=<>i&xXGIC zkOmxNO&?RFblUN@1*g+^l7qnhnr_m|>~vk5c)lCIky$L}-N^n38`*(Em^!py?n1m< z)t^omn$22c`{ZOpgbK5`&sOV8lb{}0>?UE`YGGTOL~$%}VbY{53PEwUDAch)b}AIS zQ{(QZU9g@lG?&X`DV=@*dso!)g>8h5WY|c^KL(q15Il%q2y)~cq(ZN-7ltV={XZXK zQ{^&}v=?DG+LBdZy7GZZV;OZO+Y(wW=PE0rvUMs~MoAUaf8~YeMS=nw?lFN%RTpjL z*EEUcZ15C0_h^%YQJvE=^lsm|J)-np6<6jKmZ1Sidws5aJhy^Mi?R%CkV~Q5;IVLp zk}-HY6lE28FU|!Ebm1<5M@CM-lue7md$&w0q4M0kp{nY#dZG+1voD?tG+PYDe}beg zx<1}Q)zRkb+WU5y!Y_Zeu9fG=V?*8@9bhDjOriVUQCbDLwhyf%ZBlWb!e7AG^W$c3 zQp@MPslykc-kLKPc9%U+Eh=Nw%yjvX=>Z&em^#rk+kdbLEMRk15s9-{ynzycp?i*@Xb zOO+z(J%=xTB9GFpmArC^r^JpWdZVFpV14lu`h0D+vb)wiai;4JpLG_t5&W}?`y9`2 z4=@_gvJjE8xJB}!aJg2tB{F9B4UtO8m8tdk!#`~{ zEHx+phNa$G(I&fxfk#Vb3}-wabUj;xS&wyaOD85OgMs9IJTr@DM+-qdcAWhfh;o0c zR;zMU7O|Of1GC+PI5kE8;Y|;3ixs%gU;Kpfq01~@n1@JHE`-IU0`z2Xl~GQiaW>#2 z&ZZt}K>?O+5HyCCh;QNLdpsw@~hIzPPj$)?wNT7Z-~Q zMKZ8q8q4e##$4UGEJIQiWVW&4jp?fYoQ&*g?qLgUtyKp-56{3rT4iS|#i{ zd<}!{DO6E1hVW(c*w*q@+=78HBzyqj*2l1U(3K38@VGW7!GM^$DM1bY4j~jU6bVqY6csk`WnI zJc7Hb@1Pin9GWko;g5(0D2)KSC^0zFjo5hX?0=Tb;bJx1Gt7%p#~77zDa0S5qNtem zgkA@VstEWPfx-}CV=c;(!efg6YpF*PO4+!{xGZiNBc0Ur&(f+X%aIpNzqo4-ZaIhp z1Orz1fR)hX1+2``Na*XVBrS$SO(OML{&Jgwn}8M&E)TG?XYfxqjQlgizZQcWEA?$i z^j_&}T{9N{R&NFB2;f>ddRF3pZdT`Cy&}hnPNLt8MWw4)X8iBbUXGU@q00rKP~e&; zFf2zo^BGs9Wbi?jVi9P+UxB3y9E<;u*9L#Gb_0KvirU~aCiADba0n-65(FaL8g8X= zEA|=M$at3AUn#DRkF#$R`CyP%Y>wm95=tzgy9{y56J0`OMTlc8(tYSSPa;HI-zaSg z#+bHi2y3%+i=hmMt?jBC;{8&dHtD2Lwx*oCk7r(`We}hyl*?S)DQjQ!vF4$6^bNl7 z=E%xjzKEoxDeK?!-={f3L4~b<=QH(Kij(@qqy%i8cD?&VM<&_@pqjndnQ6h=yW`k@HERPbjFIhgsoS75&< zwJXgbpmVr)PW&=G5oEL3K>c~LmYH51!!CDt5eXM3r&mu_Ru3c5X6|Hh$JFW({1qH7 zhl@?9ZAB;{axnUh8v?>~u4%f-xICf-a!d$d{dQ3(Gp0~Cq0rlvv zVsl@D3*=w;S^YJ?~#6EK1@aMW!cr=SU`QKAD>@{~xFl6g`c88v%` zHcv|u4RKKL$WnwyeA3L*=H9{JW-VvXk@{YP(wnf-tNrNH^xUn1=bQ1e@x`z7BBq_6ai&r;bUNB5O;EAHsSal$}PE z#xa@e?&RhLqqCzajv->bht2=sC~{^lR4YwbkU;Qs0W>@abUg%~QlcM*dXfTk$go#B`iMs%g?~BmC z8`G&X9SRrXYO9TH{kJ9&7I`0v4ANWmSU@%?9FJc#^q?les>dZZQms|F&plK3v921~ za$4RuAe~rK8h9wrDqIAdJ*yM}{N@7v)58RMM|hF}oFy+F7z%%wb+m^xpa+b-iNRXa z^%n0%xz(C!&t{(QyJ*yW)0>aJGE9_0AEN_b4-up3sU6?ev&49jZ?U-1V_bD|JSDWm z!$=tB3L?8!PwL;L=C=x6a%v)_IH8{|dWkHjz7F^fsG7a>t)9DBdlRWoq??VFM)?e7 zL_VQ|Ha0EVFH&tSIy27z@xP1EgG*FrmlQ=YBy{S`O5hxUP{dy$5-hDqBz^GrFOjxES^qvL1e>3E9hI3vh?rsGgV=bgTe zMe^XJ&PC=>bUI8$gP!^BHsE(azI$b-CQgF_>qwN{i`<_g%xLV0L!dM(iqPE1-qs98 zXC^BsIsgu=&XX>udGMy*{Vbx7YYED_J3_}Uzn+UZD30adT_Ej_bWX$s=ae*9)W?}; zFSggJzteUednO#u3f2b=9$b;0PdTT3CTx*FOEv-95P(|Nb4#c98gz_a{LyHbvSLv??W~u3g3r?7~XBUtH7QMqjB>% zLX>j#oQ%HG#|`$M$-kyYv#H16F%tQ5eBL+hgpF>I^78NfIu;ou5eshmRgFqD31-3b8Rph%<%6V?omR3!PJY@9#Z>tIVNUk!tQo!Z z!SmRBs8(a0ojQ&Qo;GiY(3Tmv-SH|ZEaFn{|J5*IaV6r9;gBghOrRw|+xyzHka$^Y zQH~J;t}NY^swn$tV%bSnQ&J{a=1vb=z!e0s-Bu;IuCaR2dYl z+xQwhjJ7`<9BunD9$;B$yF`RPv8_d%cyJ>c)XY+=U{{Ag);EJGdS?;Cq2uyfImvnGNdLLcQHkD(-T?Q3|WDS zlae*oYbafO6Z6*w`q9@-?$18&nYD|o&5W#-Ubun6Xz-G_y`Da!;2O+*K(Y1QfAB2E zLcxXhVV^9g>Tf-h(wG3`=Dq>>{D1R&+Xdu|EvP{ME85XE?PmuggVxejFN(ptb#U;$ zaLsj{ve*LeqkBYff=DASOMSSA>;Ni=XRE+)k5w!i3~`3mgZYr;m}Ij3jce&P{r1Eb z^*{vkYtmUCjSP}VXMOQr4PG^gIGt6sQ`(?+vT-_VZkUtZV9=-OuWdc_Ksa6}vmnz= z30SguD^>~U-k`U;+c4y-_B{fv!@s|)uvVx{3> z3qp9Y*_CHDs)VieDoSmlwvo~g^h0#43BKI+%-a>Phx;OeXkdd+2J1ivv%RhFM>L=q z-5(ts-LL$T2W%F)uXYU;K!aChA?i`C&Rz+pqK-AA=Lpnv6a&)y>FQV%HeYHL8{wEHla!gEg?sxc5c-V>bY;~;~b_cXmwlaVYXSW z7yk9c+w1!s4_e~c$^waDD|eH4KZiaV>BqJpj&wvsr1hBZ5j_cZqC=PXV-``2RtvHW zt&SZzu(p0iH9U=Gq>;(*`s~uRUNreuEoiK;@aekGr-U6u2`7%T#t0(dn#5N0 z*m_iiZUg*ibF{Q-X?+)nd0AYdo`7KWKFJ&OaB%|Ic)ej2vd4v|zO#U4U#=A93^p8jdzJbPY!S>Ffg3aTZj6yD6 z99c;5^475jakLTFW>{$$*ez6fyj-5_hT{HqhtY({iijfK-$%8x9)&@ShD<2NT_(nx zMJK1xB;U6PBUyV8&PO7+aQQFW--;Bi^xMrPkB>*Dl!&WB!&PcY%vGBb09k*&oust~ zXz}!AP0!GmyWK7Jz1;=3AtEfdJfF`6dlGJaMm2o3KBs}pGVND^+X3~y`d@4LX+UXh zg9kVu-cVpQWsAFkD9`e~Kt`{WtJR|cibwEucB;~+ zFkyfX*Jp4IqtbsQ+Q;>A{jdO=B%i(VhF->;*%q=VL{SjBCJ+@vk5i#h4rGv_Lm4D{ z!mvvD>Ifo2rEOUh6^TQ1)F;!8NcGft<;d}&MliFhKqHE$!;M9vs;c)&G7+XUL6w+8 zxMRqYdt#y7;)3CnDPIR}tiRLaT*KR{SXDX786UytKDu%Y*AfQ%i%ZK$oJ9euVE;nl zbHBP0R>qOQ@eqYRk!6AkcG3K$f0I{*H&hl8<1Gt(B`y+=q~@tjzc#NBVI z3O8AWb9Gs$M0S0IpY78n0#d0|rItnYk$iPI+d#>r^vu-E^jL~A@B-x5A+eYDjTJG> z*!C3NQeVNT4&55C)t~{FOBv%Axc;wIukgFpa$0uIs3zm%Gh>q#e3Ayo<~2vUb$WIp zSdrtJ?wC8uW@l@~%CYop_CWBKV0MiD!@sUzh-SSNa(_(a(9A?ImEAfKY{}mFbSia1 z%nm~|YPp3ljpSyeHBVzzD{|hLn^;FJn``9i_1sGO&qtXHsKO?548Q=w`rJLG`L!IZe>nQ4TBV6Jrcd+}ju8`&&}8Vir^< zppj0C78A%Dqz2UVmS7L>tHOzDvJ^2J+S#2{#}s*b_~KvSa4Ag7Q3RT14{e_aYER>H zVsWVy0UUq2uoqucxscd!#kLNW!TUv!7lw!uQkurpbKUVE3R;9?q9P9AHvAtGo*@}t0E89^bW3O!`{<8V7*N(IP z90|+bc)eLCShBrJZ0+xy`|sRWx2j%sy%`3n^`SRi@7;IbJ?EZ(_ndRjeK32@qI7lV zux)Uph^X@b3~X)RRtK5j%Wb;B6Po!9m>4))Knczv^ok!a zi719XbQVY)&6e{sh4Q|ghhWK3M|e)$PY(^0whgctYvMy}8ybWLClFl_bsSPwl%@-& zLWiFqk#wSQ_@wp{v_aD4he7fkS_ZF?Xd?a))JpbU3KUCp=@IO%ELBl52Tr(~S|nVW6nWni~d+ zw15JPyeN)dd%)UO^b`Npd=wkQlr(yv+n|P8He$6(f2$ zgS9Ljak5xNy?8#}h^$&JDbH`@>vZ5OP6xz_w>*a;VUJeNpoV;nV`*G1YzetJ6o?}> zFyXIw0>UVtT1&+&5Yud+PM-=#IXjGs$Ew4X;O42}tP={b)!|6QnTh0y9nDa5RKcQO zCU+CnDeR>c1%~ZJzTW=d}AYIoQy?lMFJj zI&Gan#JVrwT>W7J<`k`_GP5*uM1IYvHW;X#s zAD#3w_WhQp*URt%h7>yM2bMb%i5@ZDgf9ffwv1y zw`e{uAvd?piMzf$KnxsKTO`0+H+Gn80sI0Gd4 zI{}cpTOnd5Lsm#MbF_**k;rUUyZGs#VKN;^Oa=%WHU&K1{M52Axz$>%36zz#r@fgO zxDcPRcrE=)n%IHTFZVLPoc{E4j`khK{QSD(8Ivq`0)|$DFM&+Wta~~=acTmg7Wi`*2TBfU&e)9}m_am# z7b;|4QuSI>VAs--<{i5mccy^QRl(Md=%WV0L~;8OEFm7pq2VNgkX?Y|*pYe%0UnBk``f@M{i)6O&hL8z+-{N1LW9DQ(ZIp ziG?#OSxWCek`Ah!nUq?O(8f1{+GHi4kO>%Qb2ycG7a*n7Vb2f=qxMi5R4717r@ww6 zI+6}c2$<69u^*`R^~IqAm_9q|2L>r8toV=Q-9Yc%*A|yO-cj&JKfS-U1dpFyGJxLC z6zj<2WC2HMV@o*Nz?K^2{R8k~s_ja`>qiD*ncSBnBb>E|G)SyC$m>s~6S;nb67pCj z#@8Y2@}#C`3gvQ6onHl+UcvdxZdZcy;Z3GHU@`e=_l)^^GA>KM)1L9PnGt2yUHX@s zc=LFmbS&0KN9VxeF&w3^#m%`H0DA_WOnp+^c+>AYfWGeJE*C&&12ZCMK%Z8u}P?l`DOo zv#WPy_k+zL7)ni5Wf&1Tg!N$F=(vGO_omG989LXUqJ3mkama&X7_gslj$E@chFnV| zLC!bJDz;%x&YdgFG#3_NUX8`G$M4yD-+f3Jqr}_2=})O>g7Y;Izho@6pl4pY1CjaQp6tnl7{gA-s& z1C4B7bCVFI+PdU;GiF7K!N%ONX=lK6dGiXDYU_u|i0{-IB&-^@k{M zvH#xjAf-8$JBq;NN3!cEdq(8S$h&qKINIZpJq5bsY-xbBMwdzSZ9 z4BZX!WXCtGKeVEsVI=yFO^N2c8D2*8hS9NMbZi(M8%783RaA;`!|2#BIOrP+brkjtbfXQ&V3J6$1LaVb?ct2>we;cNHfcMs(2~Y zMX|PKmMfX$d0VxKe8aFu&}d{t(%Q(#-F0p0&zm%6EPaixChJu>Np*2t?fmp2W@8yz zn%+3=;Q6&>EPA6aO^%Z5H#( zh^Jw1_B^dnby*V$gX>HKfHmhs8FS7Ph7N0$UZq&uEDGr_(CFYZ=9(Ia|Ms76f z6$SWxb<@xIA6UOQaNO2MCxNkd4+{v^>@O^^bakhqDi=Xmzza6OPlv3ESxVYLZ2=uc zcp4{Q&gE-!Poq*+p>|&52ybu{0hZDm7(~d~0u}Mlz}{H2;lb>#Nirr(K?-F=unUpO zuc+5^dv(mf)Mp|88A;-JS+esVHrRE9e4n%-44y^b^XU>}Or^}Z0wTkKj8vEluCREE z#bO-R$V0?F7TW!xH=Aj<^fVu3C@-KGR}}?DzvYe9bWat;yhJ6LNX^&iK0?6=5;`aDr91Lqp6JR z9I_PJwIAhfU2j|5Z%jI6Zd_df?_kdTs(|LljMc6 zY$c&&VX@X8pMJcx!_-1!8-{mTQJ$-Bi2W(;$|G(V-oc!l4Z|DRoEwI>)tPJr!5TKz zqGP&q!|>+vqBfZ-S;p3e;Vn#MU0Pwo@E%gDXQjh}h81jo+`o8Jw>AuKtP44df5Y$| z3fstL`TL|wdqwuxo7nKS-g*uX{{g~b@yM(Ug0&3~|DKJbnIMQCN$VIMp7>Q8Qu`g< zPka!mhKIlJ(n~JU(JhJ$!<$Jv3aLCV65J7RRelo=cdCSF9X<5}v@k2&z8#$-PCKU& zf+ephq?}rcl1nc?+rZWtB$n$vi?po~w~KX@?w!pex0*dQwIro;$eOXMU-gBIKHGeJ z5Kp)#M0BZ`5lRs8cMuUSW8Wh|W;glDB0VTvq(ahJZ_A?c{>t5(7;^=|VLoSrcUc;j zZ^pWl^Z}rVKJ0=2Sf3xuSW&)O8owSMJQfIB%wflY3nCv*Wnx4KEu{_-kovl%aT_FU zIjbPh##KdGFOdjnQs}H_NDN0Kta&I6auwjD)8D!wiq|ROjo{-ZiNKb1@nz(VJN=!r zzMzxRQq)L0@Sm-SGbF-U0}rg5V+9ub=w#if12H_*r_2x#^xaioaP%v?EPX@|_1jed zj_z@9L0m5G?Ou!asF*Oezvmi-*~Ll`aVp*jpIfTd3b|?}hopSES6RqWC}}Ofc-zS} zqU31YzKVdOeDy)Z6>6vpfW0{8U5+H+-M8W09Vgc)Vdg>XQ+#3X&AKST;=5?aIq3eK zW`87%jw?^D4J|%wZbbJ}%6;~TNx!1~7iVNv6DyqeKK}Hm_CnIeDb{I*YuDd_oyOAR z#Nq>`@4!ECLnOxEiw^<9r?$P-lwEj)UvZT+rmlsp@hB?wpg0X5QpU+%I30VoiRzeh zaUg(9TYZC6Ig01;iR}4|3YZ{ZnlG>5m@{oaP$@w4i&F0^EpqoYjT>fs}GUAy}R;<#o2xL%D6Bl)BUO2U9?c5Fx2_he@A`iT6-pgg~1E{ZEhp zCXL>DhDGZ6G-yS@b10Sm`hjSF|AcGhLzF^i0a%>GHsV)`(kCHkj{1T|OVjsR;radp z1qe)R0cmRG3Q3WCmbTGWs!K)Wbih}XcR?Y#g(I-^_asx+8MjQmh4{q+OUjUmn!BMd(ee9X*Yq+l12)#}(F)@X6~9bWkfG z;9bW0!YVy1WbmU_U82(lbjQ#_c_>@{Qbju*K`{+w_-Q2Mi8IFwDJyp!y1 ze%4bg=bDbia@rm>%5!jj3gbLAfj}a8ET}q)I$Tth0@3L+dy&(M7>-Dx;Urhpl1}kN zL76?REnN02^*ubSTAtT~PIsX~3rcp-_o(m|s^uV1in=&yRL0p&5Q@S&6{@V$QZ47c zSg{;^+Nf7J1{HHCRehXt)c1~!j*Or%4NZ=U_mBmz95TYeL^mnpzKZpsUi0W> zM=TI!<*~K}Wdh_nw(;MaEW-pa^Wb%uRbf6}U8qg^_dR@&uS;nmO!QbdNOL49Co&J3 zytyXk)Jm4s+)quPrwR*d24gAHe6wr8Aa2z@jXFueAf5890N)4dQU#1;Wgn2OO878o z*=EFM@WIBrvIh}xZDMgsC?*2uWiB1Vf?BSOS={9BY6?hF!Y4*FZC~Lb$|PT^y<~xF z$$i#O6+ZRT6zbYY=qQaKp1rHEu8Yug$=v0!X-tI*;l>~ zzhM__*afB_7#R;z)@|4YvXCHSX2UK}nR%=}Z7!-Lnhm?448GyQdFR0{uucp1DgG@y z7M-H#ZF360VO<@@+v|I>JLVM5yxqp+|62DGA4IP?g~ofrr>l&-2SB+ve{>tp`r{3l zh`qUJIBeTu{UPIW?bya5t#sQi_VnCJ{~%w&)a3AikLMv!c(8WT!feLcoN*4k=p7B1 z>Nyi*%3)EIqfmWLm#N56ZbhzIuIFmSxrIUw_IR#LXM%`)ko&0%7p9_hRcm?%x){xJ zA8gZ`K4N(xE^8u5UdaB(iwEB1pbMWSA2#jQ{;sY*%{iyuQ&>t`nU*D(&~M-+C))xhF9l{uuBt5 zba+YOfI#78oQH_nsG}@9QY&;uOooR3)HLM9;^zd4kh5Dikcm98iv_s};bB1UB+%wS|D+;-Nu#%(VKJHk5F;t0 zPpTEbq|jMEupCZ>e*`=!b=VI~@9vMA1u*^g)DIL7Tk)_?p(5buzpr&e(sN|^`Iw05 zXpNYpf)6=$XT%RD6U734%nfOJ z$it?x8B>hOSJ-SEDHop@2sCXOIt@PgMin8a8qzbft~ZQaS9a%GIi$8oh%23lf#JLf zbp(rc=p)xApV8T%b*lTau-Jgnb_SK9D0kCowoSm4o0N!ug1TOeO?ybQ1lk-*1LtBm z>Gan#JVLuXxPAfewvs z{q6!ea^gNDkfUd@y9{y`OUfplM#b`W@B{=|VYyx`S1Z%S%n2Mml|2EnTf_qH+~n%e z_-GdQ7MAj-CWZ%D*3rVjd+S^g*_Y@9%Y``vKXIwxSo-2vVeCO(^U+y+Hv~Q{W-a&_ zIQWAYyZCBzcMoGXtIwX^v6IET`&w);?Ri&hJo-U5Ck7Z+!CNdY1scT7uE1}Tz+(yw zfGziZAika^5SP=mi}9=9fuV=gmO+OxIM8N$fP(DsgHHqB>dq%(P8^^3=bZj#2FZFj zH|ZR9NRZyxOI_epSG^NT<0xrAD**cD~P(ve35J@e8PCE&2JG80Q>2t-=6YyVw6hUh<)$&|@1vFkh$7!k% z=``+4g?6Vg=zNe;zJ|R@I8zCRZgF1uZ|s7M5kuO+j+HhuRp;Yi!NfOZE1_h_NrfbY zA}Vz26A;f}Llw@ut~A7ErR^|}V^bjHPUtBl!`+|`>6Y=e$F-7Yi$;9L?)v059_uiX z26*XJOJHU|tO2g|h(APw)6t^=z8x+RP@Dz=*!l73g8Z z3|)nw0l9}ae1JccF2KY_9>9Df0B~msVTd2C4yGXB#fVbrZr$+swC#&Gy?w{O=}*>A zra4(0cS)@0>_DR4-V30ryZtLPlCytZBrN*Pr%)6H``7dWQJ4l(+AJ{1y5!I+LRC|~ zF70Vw$qrUd;Iw(vI$0j%UCl5cv_~vIL-B!-0W!#CRfPi(TPVxbDu)|YWGPdFOHQ4F znjM5mMLsvTr4q~8Ec&jI>&|4-4&wXFy|ntgMgkSRB%H;dl&=-pZTczdYY)((Uo%qs0kVefbWcAp-HY5 zJSx8G-468cBll8kh~qCGao6AJTl0M}l1jbqEJED(qeRWS0atqdGC+WO8-&tj*E@@e z!8D-@#dqRt(OpdlA>`DAdo9|JC=SW!9M*iOU}6eqAa6ybD5!jvd``q=a<4upv7H zd-YJ*j7|hMXKv{<7Dn+*F<;Nw7#1Q^_(FAle%S76hYl{Bh0_5Rr^`1QoF9@p@a(Gg z38Rf#x45-GhE}N65T5fl>pO4rYG+?_MwNcU30t%9dth*=>BhxzRK=R>qZh~}@Wvv` z@$f}48W7^qm>3C3??9V7ObSnM)&-bd8U{w-%lGSB2*_|*;E823l8`^N^%5`${#Tlh z!I(%9e9m#6fuAj^9c)W>qF(`hg2r7jaGzu#;ezVS4hBo>eA~lhX(derwv^yA=oM+2 z*GD(E<7PXu_DupPby}!ha)Ov|lC;XL>-+fk!N>2so-3>HySGs1ZrMc6mi4Un*^7mr zt$-iKP&?_71pe$-vTrlRupHn{%|!qG1CW}_*t~tV2v0HPH$rKcoK%&?NupMrTEaPq zq><~@a+7OQ=_I|@1daldBv^O4Oh)vRu{{sYE041KGDp{@XMGy{!XF2zWa->k9U8V%{>C=-~|?Ie(NUa<;`e-F_=-c@#iG zcYPwr&xuuEpS;V{Cvbz6=L#gtC=M(X&D5&=rGjl^R-N?MiG`ZA$zKD*eQJ}}DwgXU z*bZCKb`Aoi5#)XkeNxg<{IDnIJ|`I;kgL91EhR~-lOXL8~ubHm9~6WJ3gy(LV% zMC#cg)GjKb<#0BaIWAa$_=$R;JS>&W#aTW4&T+iehW=^rs8a)Oa;~oFViv2BlPG@F z_BGE$P(}e>p>N-n0n;V=beleQXVH%wjFu4d=T!W=lURtufgy#!$Oy>-==I1MYrR5@ z(Rvb04t%&B(gToZcem|EO?J<7k)g>TW6n;7mkFq7$cjfNDKQR89c23T<^kXsVd`4j z#6YC?1!d?4i@vMvDJIWk%>6CLB35Iu0eYr?CB5_NtZ}C~tIR3sscmpd zKU%;(pwa9gGzMo-DG8wy=z9=Z5C%5Y-g(hH51Cpn&<-Vv%fw?YlyAv9jWc}EwW{*# zy&qb9s&op-S+xy!&HOLexAXf{H5MT9f2YUBu(lawctYiqSdqbak{&kv& z-|BfqvI5%pldb??Wn_T?%-!$-cr{G`-mzZ6W5*P&yNduyy02>lA3G)CUW?u;{6Gl0 zmz%1%hY`7=zK3k7WUDust*5&CFnp9T%zVc`Coy2 znlnDL^;LuYo7`a{A$;JN6~by5ZsJ0&8gc}RSN@Ve zKGL1~Km%E*-^V%jkZAjs;$X_JSRk@d@w$Lp(az7k>-oBvu=Tx<8NA_c<+3s&*PqK< zJeRkC&&ZYb4J=QyMR$AbGl^#SkegvMOjfgZB}A?rLX`B4075!b6_pPX!z8K-H~^<^ zg|+F~?81|LLZEfnTc*NW;hYtx&Yfsfmsf>ZI(+I}ZttpWanwCNwt8}(_qKR8wc6oT z@{=65UrEx9Z9KJl>fBK=Adh(Xw2ytQw`>Sy14&wVgpp*4nfd8H@Kc{|%%{G~%1=kO zLLfeJlrjv+{L&fT92h}~D6CT<5%tQ!j`N>wY&sbvMMz0~@%m?%9$Qe~iY zxEJBVwcTfax-;xMb>xQk|M{UItNeBBAxTp=;%C%|pYD0Yb6rOK&{1e8Gd43hMq>pQ zlv3(Yf``>aJG*xz^iGo2n4x?O-t(koO0mx*(lYZ7vu2pAWi}C3?94^{GXP3={UAQz z7|r*dAGy+b&kN6@Mj7@SkF0tQ$hI1TT${8P*#wl>uih$W;ly@rylesOwLY0WzMi*8 zkY<@kB%SAcRSdyHGq+Sf-@823p% z#2{W6lP%byH!^o3Ac0G0g8BkUPTn!r{_@Tg2xHW@VFFeN0yW`z?go#7WVK9;-7CL2U>-|Q6ZmOp0>dc33dXi4LB_KW9Voz9^jic#)^5KR?rD&*@`#sLjhDLP;pCrfxN{A$}mFr>M5WcGvsCaopqk5%znAP8W z8r80W+4J;4tg-v_Jk|bPNAJtj?bvT?w~_EhN)BSf36H7*x)6a%0W)+;hPxG^gl0HW zwZD}_NSIv1ca=PrEWme=l_cyS^RdTJjj2*uMahICbYddL<}!v_EFO05>P{##Q?g}9 zPtYzHI)ZVgkuSmrha_dxkXM_3l=CeU_xHq}X9r7z*yJnAzS9{@^TpXBQk5U1GTm+; zl>%=SIaUGZz;p>|cCgWc?Yl0;PfnjJl~%J{Sb|)NI7V7+a;5W)GgMf-QNxMLNQ|I_ z1r!;l@~ph$7>+BBZ3)+7K~$HzwJnth$4!SuaOeDB%w^@BB7fWqU$@f1Ob+|wuGCB>rq zI)|A)6I(=^78i9~WXW-K6K`vQad`Y&y7UeN2=pF-lIxlk*}qK_zqfUev+lc5$7TU& zI%Cxo@$g`dDq$&ozNEV0g$w;zpIN?==49SPg=|_Bm6*mh4c0w@nC6Lfs6mT@b-f)J zZTDD)5Y8@^A%cc~jLMie-UJ(dDajC((zz@aZLm^i46K;;_26L_K+$JJ(oHN)QIxFo zZ5NIUsC-EDP^9!ml@*C<6b0l~BJ~1=KC3I)c-i<-RW_b#+oN)PegT#i?OC2H-~?Oj zw+H46|G)~ON~1T->82rwJh2}vBtimh?o5PrG2DbN1V-eGtG?2OZi)e$$iVmq z&cHB;U<2a@PTBJ5M*tjr900E~!i~&EtEbNgZ9187Ub+v!+g)e~ga?Jjuqra%ru;hC z(VkXhluZwBScH@wAYvIPy`vYJ;cS7AIf8NH_X4=l=S6l8mPy40q|nI45(_A#@X#W) zwselg5Ox?-%T9SZwueXq2s=Y3ujxtj337!7X8mB`YUzZv>!AgLj=Rz2mq-)i*R% zygNINWi<^*DNkrGa+1(`Ssf&&?Q10Lz@U*CIb7X80Ev`%p~v8$XJQ-_l6dhpzR;f$ z>*^l4B`LCusJIZH@=0(ehMq)9>7|2~=nU#uDQ!{Pr!nVZV#LkBboi*gT9AstiFeN* zd^63#@9mv5?P#KPyc|GAckLuFA7uAk-~uvWRW6t~d)?TWC}w65o2ab?vGo0f?19q9 zdciqsq@Qxs;>PT2q~8(QLxIGe7F~35pK)*=Z%GF*?oN18LkcO2AR01JJH1AGef{Tg zW!0(Y6CAZ0wZ%5RC1JB8xiN!aaex1M(JRof{yw^*q_4%= zZ>5Q-trk&=cI=!)@h1VGx+?`SKSACZ()%@vI6n4_(hsvjHOv*~=TQa?aSTH|$zQOL zl>V3yHcZ4V2N*IGs%Zi!w{Ztjzub z6x^pWd-d~HU^k6rR%s6vc-gZdI#6Evh~zw|Oh=YuA3-pS_5h!yHzn2k>~3%NgY# zNlfARYV9bM#fIzE933cto&#lvb0|BE;bKd>)7%G-0@^;OKxnd5x`PD$W=w*H7Oa6b zLrIB23_1BbJ&#Mc4j)a6_{7N5>oWkEZ&`WkPaU80?)l>5EyKP`JCTL;+JCYF!%Qt( zSlMnYCQ#D7ZC6nHvY2vd0{(&)OK+r{u=5&S45JB5UU$n-(3Mj}?zLz_1X2Qf#&Zz) zIkytZkz-iwBfn3)t%6_PK5|wVtqZBv?d5K#bCT`lqlf6cWKs#FptyK&o>bsWex}H$ z!^o52j78hqOLpc3ZUlB_GZ7^cri=xLf`Fe5It0`}k z8J`19MfM8}nk{yW&?GNnxeR7O70;*g$<=0KxyfpOuFxpdmJkKxa>u63BUC0-XS0f6 zp((VRX;)m|o*P)H7VC6^j+^##C-l*+*eJdL+oVDzm1&TwGE7F|TUCakRSs{)6jTBG zpovpk?Kq7?3chHaW+u+kIg|#XgltTwKM=xnOvr_lbD+(;*iW&}ITLX7*H6#mH%i=B zh=C&dS}g{iSXN@&x*0=XBIWos$5aNjtQ>D-X%iz{fvoWJ01|xgZVHmV5hQJ%`e1$} zT`&Vo=Ybh|9ssj5t(v(;t9SRu(PC`<_Oxy&{3&$>Wi+9>8~((7I?ag-9gt%P*kqYG zT^uR{LIoQH^^;187x0|=0@Mne{_|Bx*Q`4J%tqCef{3{@sk#^EDd(k#HZe zy#|ybcZV};ou5jnJX&6IoG{Z)&(n&S#A}0=qo`$yrLaB@H&DJTgA;D?=`%`MiJDEG z*6RSfNx5?62J#uK(-+Egf(+|qD$~YC5g5F(!h9s(tk+Qh%^hHank_6@D8NaP(T>w7 z7^{0*u+A4_)a*0Gx{7ywDb{5o0sajqz|D|YfZx`E4h5_on+3oO$vN{~!*@O^?vf(E z{k$6;khHlkTfUqo>f%MV?5E&S_861WPCEndWMLQK_?Kw{FfbOB6JpxiwFm&zdB%li zJCcqV5~ED3$9|wXoQhNwK&8}SKQP_WPDYDiwr>;o@*BYMJ4V*w3Th`TZwM$Ifs5ed z02&gr%2r>YiBH9!1is=UdlQtQk)8avl~%kc*q0`n?jqHnKevd zy)G=}Pfe_@o`9?7+=+5wzHvOK8dC9c?@%LEr#ti&6hK}@f2Ok&wc^5J+1Jv zIR)i)YlJ0nvgICwuVxwL3Mkq{2E}9aSCIlNE7?Ohv9Z>KfQGwis92mEHq71QTNd@D zI*EXR(z9K#Z`^F#pRp*e@GH*E7X1jUXuWD66W^>Hcx^$RFlCu@ktwVZum*AK4+hR+ zAHd~jXREb2ay24_giHsuz6zT02(sbJIv2k|J2GrfR54knU6Gytwk{(SqPA;OAnku5 z+=(3eo@DPwVlOAM=WlfAHy>wv{-|>IrHfX9jq}rg0EHnXX&^N#2()=?mkGD@-L++4 z3{GOS^9+b-ATCgX#@8(`WtK_KLrn;GE*3aA$GjnL>8e^6UWd3exi`a5?S!(f_!$;G z?KviauGsnaJV6$B?nIwbCtB7D`8jYEq<*oY#1Ujsxp=xzE-qH9bL9MhQ=tIYJhq%; zzERM%sO!TPD--37RyCMUp!Df1QUNkv4Cxm&4L zW!_Ld0%B5#1kX~Vk_5_9a~24N(N*H#OcN&tvMnP_PS=o36%#AW4n2A3$S_B7VHstY ztMFaIIcwyM)F0$irWhqj5uJ>rQj{otiQJ-U^zOs~jD~#BR(|~G#SE$-=Zkxs;Gj!) zcd`aOA(Hfjy-cjjM3VAeCn?R4SW@1%K02{tEm~&>0n~J7PyvKvP~XD(@363ZP8gLx zJK0*qDOlL=8^?x$3Khq`&Y7ZjO0OQ@w&FVr|y#*1tng0wd01k6>J zXd5}CwGq*X-j$oy%B(hFKi0EDIM2@mZ$)})WW%Qa=1oy57Iw7Ff_dcYkJQTLRaN8$_ zcg4C)Bsib?Z+804kXUfOFZC&Nqm9l(=K++4memGEV>O}8yZa<1I|+UF6o8?-(hw8q z_}5n&`@KV)53TMyK6G|^d~{`cd>ECs&Q9;epL>T#Ps~*t2!-PY>j&KAe}_*x(VId~ z7F{gBPW3OqW^UPKlHeQygaG^eV=7gg<<&x=Qm8>HuL(|x!^M!n)2~kHaNa9t3m?U3e ztzbl|@$^X>S$OcCf8!t!O#4QT0-RBaNw*FHGwE{mOsva90<-pzBRBm%3(U|OEk5Ru zL<+Ro-3yUeeu{c}Er8MPvPQzL*S9#~=rK4ehQE7lb!}*IbM*5u1NBe`?_b;;yxD})T%3h< zyQIf0B$D0=XvfXjoJdw}EOM+PW1dr7mOPI6S{@GzwYm<382_l0hB6pCno+{6UR|Q? zkF-XH#Yje?&at8F(b>f!R>W*b@(_#9Qye%iBxlAtO6wFI&NoJRhl4tQ&y2x>3LAH+ zI#(!@57d1%!=@>|M>m(M^@iTbQZBgI%KpAFCRjg(&Oi%ak8yFxjtaEd-}lT(Nc!)r zXGlaU;o($hNWha)hyB2GFa?Sh5T(*xKk&Sz19w^Z7IkVB_`1^!?E>>Z=}W+PRCnsA zO4(O?tLyb9q|hP?Ft7~aEOrs=@lx6R^b>#M6*<)4d*e=R+*Zk&Wh%st0FINBFxjw` zZxE~Gj1X=lgVZMw_!-MST0Mh-)JEB-*yK&v)PmEOc$2!4x0I95+s`fWK%C7smu4W4 zNx;&;XoZckEkz^`p$-{GuBzyI9{aKB`JQSmLz~0+21)I5VTtwtP!M~$+MswAviaEo zR>rEI#C!! z&$9rcpI!oE_buPeOiv~aO#iYR#!I3HW56e}bAQHhl0gjHxtl<_0yh2`7-VMz3t_&$ zZ!Qp{NujfTV4>vg{`pbBlTwHMz;rMT>0s$GN~OC1Jl$1;AZRj%0OO5Dlg3A z!l*8Jr9(RvWcbi&2o(9(6iZqI%lYa8+@_7iB@&Iek%0im+J6@NjS#8fEy5HZ!gi}_ zMJI70%ma@Ah38$sU%ia1$OTB(N@FauzYMdw!ot3c>c}L@Io`xt20mHd(F*lat#D-_ zzKw0jsHprr4=iKrndq+FK=EbNHBr$knA@?@%@xe(td%XqI}GA}F(l~wU>eH#D*;eiJ@yQfkf;y$OF9Cc{`=|&k^}wIuz)3v-ui)Ni?Pzi#Gu5@K;PY^ z+CO({IwaMie>e#vPHo8Kd~7YdAXloDJz8Gy4=ui0TVoXH*}C(e;(1^Qd> z?S&o)KC_4}{RYkWKC5(;yi58EY>Qs95SVdrac;fMm9?$v+O`7 zgx1uf*$NzJTpSu(vP}gtr>c!vFoQK4aiX#FFUYqXmVQ@Z=~eni?04w6kl+e~pY!ph zj4nS)_G6iYUK*DQW9-YF02a>ml_s@{PO!xH61}L?UQ(@AU7GyGKTtbiyzKjO7n)9x zB>&1~gC?JV$USZ|N7t&{bC|nuH`waNvAcFurPmc1{nwt!=+)uCID9gCF-%4(j#|Mc zx>Y79Z*Sgi^;GT2pp6;+nPw}iv0ZvHg+Y<&uzPoU!r!as0;(6|(Cnlzsr+64r1IL6 zfhiKw@q3QuPxE#z&yQuSjuA-7k(iid9~81mmJ3L#gUlqGBj8AbM+R=F>OzGUKC~*D z&*M~?B94`zB2}i6lM%Ew+wG2y;}OP1mxfRtTXVixEG^w8vH&&#G$Xe8Z;Sd!JsK$8 z9P6N?TrHK+J2phTe#Ro)41gUpW4)nKK-%k@A<*Y!0z?Ae)b5@@*Ly+Qfxd^_u)Pq2 zn~I)a4FK;ohOeL41&m!O7onUC_`VNXRS5KL;PFb-oJcM9x$bZs%na^w(Z zvy&N>K;>NaDNEK4MM91|u2c^Bvbi!FNrf|oayfSzR#(O2d7HLbm2P;BR%$Q2l?I1X z=~eJjN*#jAj>b5oXZP>PSO=Y@i)OLyzhS4r44^sFoxsIwZuWcvU&*v9Y~*S~b0z70 zt%T3);N=l&I_HB0q=j|F%GN8q!|v;DKn&i`WP$st^@OYuMf`R1G(zLY-= zPk=pijZavsW93Tu=&BGfIT;^E;HC;8x&X~fp&j-vRtgBmJ5$EnG+E(G7VNhCY*{1zgT7;Kb_ohCHu4 zz4cw|YSyh9ldNWko}dX+wJM*s=^E?-K9FB2)bA!s2u+rZRWZ`#h1b686ZRE{vL96_ z`(zgyG`S@Q-f0o_aehS**`M#v&cZQ7$;5hS$RQTAL0KgJ&~>(lX^r$^RJnGvS*v3A zg5K4(z}sw*i2URVJXj0WTHaxRKC@83+M`ih8O!*eppO>LHh?EuU+WzaFe5pLS{Tb7 zz~)@~iPJ2qWH&I4bJ$du2|Zh^F6n9yHkCviD*Vx;KnwKu2e#5u(BcqMKHnI62eKU_9h=UCRPIzh5OWfPQ)F zPvZ+mP`%D(7@XR7oOU_T_6D`HLGNogscH!h9@}>uJ0NK5gMO~}JDe1SVed@z zDM5<~UjVIqF-9xCiF0e)(HjBSp@p4hwoktl`%ofN=n*#uW}v*#+RP$Z0IY9x2dFK& zEkNyC_yGN3ngE4zIt|3hb2<)pTiX^Q;GGB|$q{joL&)yD9>I15$T<1*G ze;a!<5m7(ph}wL)8&R8fh7`OJMwcEl#EZXtBHcEo$iTeV*8nf4sd68P)bDNxj4me{ zG9y$!{6X$abC7Y-P})kp%SkrzS|5oR*fBXu{P0J4ZhqENR1|;W4SjNPWRhv+ z(u{kBDKSnV(D}O(FsJJ!S@XbXd^BzX)0(;FzkMV=nI;l9vs4iv=o{S!2|;8BNq_r; zoK16(Hw6Wr`(o6lL02HIGBVRZ;%@lhJeMXoTdaTraA-#h-Hajx%OohU$(1qFtgu*iktz#o23^gjEEneZn`r|QJ7r)c)X1l#p zj|PjK8`*C~o!eplJ{=oKN1d^F4F2%PoklZIkd$f2?*-n$L3KG)P3vvl%ceBzHLz&t+NBr=I_T~>Q0+8AGLTl z2no@qdA2x|oHj+*?P&8IEwnkRX_TXvpiH9FeYmLKQ}{hK?BWLNe%~US>Gv=Cr{CH? zw|I2G3P;pqIGs#6J88=$Tl5A*UP-{@mun43M|CC|1tw|Hp$K~Lm@xLY`dtcz6Mq8A z_-2eUe6!?gpw$e=kg<2q$JbSR)?qfk@cR}+W-x54WsN&cEVVrF1z_FVb_2EJdgZn> z;kr`KnHavk)peBD({$-oReaHsOLLSVt#78stctp;Au+c>>c8)?d2(d3BS`Lm)D{4U zzrrj*+)o@%TrVhnSAy3CniDn06dZ~n(&ve77>FE?;ft30K5sk`>$IZ?HYEf<m-9-^G)O)>qIWktZc?MWDU&?MzU7?rC-bh>8FK6AFhA_o&l_i8G{nj0 z${p=qVYd^%V!f}eAqVHKPZDyt@d+fJ7pX+hlkPMxGZo>7c0Niqnu~$bGXCjgo|r(w zi+^V&)DpiUi==&6k_R7Y4bIOYLA$i0UO3-SSpOILvliVv-#^Ao*M620Hf=Upaf z>QL)uB)ZYCuq>`FqNet_WW-h(<|IWD8FtITDk}8qY8)Xs*1my7Q0WUqDFdZn>4GC8 z%U*umBI5J>ijZY{NRHqfKsP2Kb+Ef=o^!v*s0YOIVe$L^@41-0*+QXwQV?9HtEd7jzy9%@m3v+1q;XwhwJz@MTN}+iFD(?--uEa=^O;y`b_${Rz;qrw?A zwgfL2{z8A&;ch_T?ne4Yh==ynWt8jh`dAEc+^()1=R6FF+S*|=AMETh-wF8OwO{o} zK>;Di;bxT-2m_He(VG%=x)>eit?hF=($c*O!4o|r`1SuB7@KdsywC~3$;6kHc}BLe z=ZDU*iCaXbffRmj{w{{0Sfn#~JlsEdTz1{v#>yc{(Lx^Ir^o{<6D_nLFGy&OWl?tv zWxK0q>RD7%M2=B`&OFI;#n=K0)y!h20WvfiE3}lse;n?f$C}2j-fd+Zaz)XBgQAK9 zUM$pbY-42sX9dG&#cA@M%j!G)fr(_Ao`)tZjYs`n?b@^VKB42Ej4^~~kaYAO9gDr$ zQC1lnWcxepRGKfd4RT91AI7_YyM3FkaCGu7olX;u+dC17fTeT0uJClqLRd``o+}hp zi;e~HCNH{`;To?&n9bCv~onL8Rgt|a22vld7MSeBa-(A7) z8j1vA`w~8aQF9BrgGY4g7j9$lGHziyb16yJdUz3h6MKM}_}4Ke@I-Hfc!Wi%Sq^&l zd^P)9vCa}n<3TsaW-u&`hZ9es0I-Lb0>HzHYsq3UV=_#PHdceTBN;KE?rj6(@FtEWD9IJbjPNy^M zt2n(O0BYAF07GJ&9!W{mSjLai>M;P7(|qlGJwkYpf@rbI|Ar)?#V|BPTs0s*W>f>_ zLI~JpR6mCazF0)k0}+<(L@O3L!qb15TFDNuNd%SA(24LA(414kYo&@^Gx-J{N|(i$bnO}~ zvHi6;7`aw$E-XUNVKY#kPDQaB)%JDnLoAzvz1WE?9U?XuC>`j98#qL@+p&U!(jg~; zbHFpobJckSki)U!9kI3)2JN;)aK?6$CBn|b3W3z~Gxa>&5BS4Jx3L%p+cp(gb&{!c zQep5o{za06EfjuS!HRX~Szp!Bj@>X62nEo0v&ZJ1Ch$Jt;Q zc&1QSr~Sz@sX4$!;x;3T9=R@ z#h3&M*?56A@9vLI1u*^g)H4ji_TSqxyNH3Wpe5_-$!1p~`=;*EZcWL5jr&=6! zQdI(|B&`FmCC2R+-}GA1@@(|xhb1tR&X8{BOArE??8p{h!V|FP%5-+wG0*IRwV{&( z=+s3Rk0G;#J!^Oob{LlQFn^pCa;5H85UaffIQ}BWGak1^n=lqDnCNdii!7M_CiZe7 z#q>_6n9RpnG2QDVZ*rL3fTH!ip9BR?HUq=%i;RM+Cw!+yUnscB&Ys0HWV4;o*4gS# zbpqQ#s%vN?6$Uw-Pr$1eWQ)-PhEuy^)-e+!%5NfMRxVS#!mbB+$$&+NNTUU@4vvrA zkAySTEevUuFwP`J*=MKD@7LX2o%b(+js|0-5Mw1^!m<+mZKs&&C>MJ<5gqMwtYki} zY`j*C+pk5}w>}IEzB^@!k2}iplSXc)q>+7lO%m%~st`7IzuTGC_PHG`o})7~8VMQu?X$w87FNntfqnQLHW%2fogxTkCn zTDh`2qzGq|t*(TcNoPdWn97BP%8_(*p}!0Yet(REeD=GXFmCuy{qTuwI`(=Z27Kyw zELoY)GXvhhE*O7m1$Kgu10aOVL-QRGFg%nB(O7gu>GT(Xr@K%Q2z5fmy%z1zepPZ# zHpwm6$xFFYHztnb#`H%wH>Gp;C2>=3Kg7;G9KE2~w7g=7EcD3Yxb!h1w}DdBMYNgR z@0o8{Li;Gc;+&AtZ>vPoaV2vanR63 zG8#JzrC%qU4U}Gqk$Xq2%wet<9u45#CySWbInMO47u4}Xw>@(S|4(P1wQXp}$<*@i z`=^!{tASAq8Nc=npWx#Mz-T!|X=9*|N_HNbaJ6JL5ql|ic?HITmyT~ zlS;_w)IdlZo9FH=eUv1k?LUFryS=;Pv{`!JxZNd$HPZbcmB-Q#0> zZG#6Mc>5|=S>yK*L2Ej`n1^U5iCO9Y?Vpui{z72N1S@^#QHszOaqeJeiq}gw-t?Q# z4&FlxHp+yD>;;R2yl^`h-{jXoxsP>xLzo-u>;Mw;?_xCWFM~D#3_}(Cw~ziS?^%a8 z>y@usgqx4^A}HGxUH0Qq_qGAhy4wh1Xif>c*P`unWWMK$4bmU)u(LQYHYqiXjC`)%Xpke{;Ukj+f(;kQJ}bGP>_VoN!i(Sg!#{L{&W zV%8b2UOjBAmLYycFwQeV%-d9OBjgvb#{_x<$BiP1gi=YGygQc88AT>T97I>~*(S(* zK=d9^BJ1zp;o_YN7Z1=sy=_4AvC?YG9YKl$=Ngc=2-V`@Agv>D93~mz2PRsB7CW3< z4Y@#SIB-yrG5qGLI8#Da(?b3DYI&}{vV?!m9a^Q*l9d8ZB*)o(R5Tlf6i9KaQhw}J zJ-@>dV(Qd+{t_tnk=Shc@@%V*8pAO0+m;4c$vheBvZGOHcJ=(8qX9D{mU1_@n+^NT zsBMz~cuqLATla{85Q~P^J^Q_59S*JTJB|u;RB?BFcr=SY_fqlQVO@QfZk(&+_O9Y4 z|2urr?X^pxCyNKheGxP#l48C3X^RNqj>SPm9-qOeV>}xrAO6?fmOM4@qtu@Yh=LoHyD4bnB$Hx^Mt( zDEtsRJqYJejgG7aKIWerq)s$ADYlls*@kT=lgaCQ`zNoFhyn`TA*Y`fQWVxT6#{Wk_~d*{SuKIrAG? zh6o|I&bui$hPpIdv72mIU8>1z4fXO+7Fp!ld__EgmzdCEF4vG1T1)OZvl^+NAAhW( zLK9MrUsU)J8thYMu+E~nxP}Vx^zoQ@@>SsW*2y+fFZ9UgmmiCDok$gK|Fq>0Gc;C( zZ_#2mKw|>qomzDTHm=z90ylo!2l8K}3FJH0%N}-2(Ym_`pr^aNAZDwby|6o`s+3-8 zQCEjnMO~qu-m$Q=kpAudh4jw<-IAaK$_eS{>6m!tBLm6HN@upgET3&66MO#?YAd;FCWLmVTeTTO5`Ci0E~o^oPBWEn6iM ze-*%rZi9O&jNfy6_*i2xX%CzxyY;19yzb#c_+_+5CX6he)TK z&lSq-%EzWKDWDInwy>2(6;kN2Qs$5VWuBYW;K&Ha&_+gR1G4N%y*B8aPB(@=F9dJ+ zISB0=F+%h8%aH%s_5c{Ill%pLhBn>X!@Aro%bq%9=fr@)x@aJbO@WSfEdmG;b%Zzb zY#31WFAGOx9uJfTdVz*(O1A^$Qh5Raegp>S&)ALrj447fLk>T*5HPQd^fLBtGjZt- zfpq)`48OY)6S#5m-n|x$&d8lgT<5Y@oI`p5HmKxiw@EeUd``l}r*=3w+mH6M@QdtW z;}@YW%<*J_x(C)Pki`49&H2GthnP8Oxo@YV^dwEiKxq>HbXqnW|J}@K%MS11SAO^1bqDS3&FLSDX-D#d~o03`%w@@1-ayrLwWE+T-CP_v?rL}d2hqEqmGL6e>V z`lvY+2^|TS$JPMecYlZz zko4PA&oGD`9oeq)XGu5e-Zn52$J6cnUvoZ*w`k$7Kn)^DIZB7xfj+T@CYz6{u{QaP zLEeE!1jjq!bU4Rb18dF7B7hpwb}_UrH&IPZ#?UR*S{l~-U4p#{uN3&}kvxqC{c_%{ z`$G2lLJhe95JPr&gC`pK=BWvnM$2M@R@(Qj2UI~x>w6!wwi@FBU;l)boqC}bdSoev zbUgIRn{ zmz1NIMaQZ>xK_A%&s+cgT>NhUuan)9z};Va(KBgUA>Jv(*5Z*wG*Yk3DM6;w z8ebQ^utn$iJT~l8jxYT$$hRy8OTSOVF;MzHdLd|)1ocY+ocWG`|0hIXEo)P_d{C=a z(Oi*fjv_@DJUo#W)qi^dZ3&4L>mT>eiZ6aSKvr&rF?9_>Bg!OU%b!m|N2M{C_s|Kp zkQ!S&@WJ8GL_QQZWG#(l_fzORGgnkauy95<31W1E|CYe)#GB6*Dn*<1iC0)adNCF(_>aGEb|(3mSM7w`%UO+bQ@b}-cRV#yk66TM?asGIe?4YBc;F|m=Nk@i1gT_-Zq zUb=F(ai5u?v5|JAo@@cJy%hi>wB9oPiA@m)`X?PRpfq~x85XhbI+O+#3wYA$uOEo+ zO2$%R%t_w}jLIkd&CI68h}6G)VL5W&I((7s2ps035eW7xk+>NcaCaM0e5{=f`3sqY z0gtiCPgS}eTRD(3F8#_YtFcr9`A{v5huyo=6MjO$ZfhG%C|h0oyXiZv9r^vZw39dC z$nDIsvONg;>C^6gdCZDSH^ENXM&s0ThN;kNKM|QbU59+7#)}!S z{cZc=8LXx`+^g&y*pFaix1E<(EO+OBff+|0zKQ?SZwUV@s&ky3oa^5-l=6}O zdHvPDXUW`6rr`AnyD}wTrivoWk`-(+j&@|Tlf_y1CyVvU-NlG-d4klG&HCC;1aRQ^All{vIvnf~Q5^M%4R`Y>o~3Oago)xbk5j~m`}5;8 z@%t7{rOy$(1ErU{(1MB6y#8-2u%G2u#AN^O(JIyyHLBc3%N6xaZ?@TvfjZ9%##WOo}{PDbc8~Mh(x}rqllS7M*#&Z3^J$nk3 zu`|Wf#pS|WF+WzVE$pFRd-OZgva8`?^)>chJ8$-A7Tt0x8IvRVtV2C(RaLA%(H7#guv$gz4N}mF{ple>3QtJo0rs+Woar(`bJ<>{(?P$?<*IZlJ=?vve0CIO)OYy zHPgjLfpBvq}V$EG@hGz4kh<*PCk0b~f1v zInuwpJMb|}L2gooyrWb;z1q?KE<$s<&aU^zNZ;q3EB5Z8Pa0$0{PZy~9qS?y_dNBB z7Irg)?rtvA(uhufu-2`mGjHr~f4H?YhwEHg_9MQVD?a4ztS&I)sKR%1{p1_L>OFb6 z1dqQt25YB}nfo91&)hSAXbId+qhRhGo|h`%@=$5+@E;^-X-A)=F%-~0?u7@~_^)ls zwHp79n4@iFPC16eQVGV=SH+Jy`Kp|6$oQ&6&+j`^D4fpS^Y$F}q%?}yBZwV^IsY^U zDv4Ick+S)k$az*N(S=I?&sVDrI;226e9xuPwRL1f zprG3HS>!}7t>MmAoHJ5fo<w`%c%QYs)y{ zw25+pK=m9*|FS}5ExG|^H&N7KTIF}(GdR5iD7gZG>v@0)LMYBn<8X}m;@P#uE26FW zW}`aoZbg?BK~2}rHJ6sBEyV$&e1rJ~l^dwTZF~m@%2et!@9x%ez0jPiPM=*OTDp$B zxb?!ce3QvzwlV!&5hr~xZEhp4Y<{LpfapEwA9Pj0(GkQ8922TGschMK{qK z)FZty-6%lhfMeF8t8hPe=lH!q%huZbZ0s|1$9?85^BFK(9=Mr3jVgo1LStTitU9}A zv9VOf$F?-`GvI=`#^Tz0u8QseX)R4#_OgIX^NERTAB_fhT*V4bB%%G)pkTt*+}cN? zt84i)_9uAny823`0dcrkTv#mQ{~*}Q%T=QFYf$e9X(yAe%Bi-`1WF<(7?T$X3N$5>FL5!zF3|{-ace6$%KzY*Hf1Se7#Vf zSCZ?a(WXM>xwXmWOmvl!7~oV89g2z-sru}6BVSu6Kmvh8uE%IVL=6Z|^nqO?^|jTt zp=cYOw!$Usa`abyZE;I!ewEcaNQhTe&6`sw;P*U{jdt0&5yUdBkW6Q{bGL6 z>5%exMwc($jLUD(<)vG2`4nAlyA_vP=<GEIb@}6N_9;FMSikFlY z>GGA`xcm`a{?#Zhze1O_94S(@a};L?n?5Ct?=mK9|&4NUU9lILWS=VOxRVv^@!lILKO@juD9pJcpGGR`L% z-;<2%NyhUe<9L$sJIT16WV}u?PA3_klZ?wr#^WU8aFX#i*<8F85{wyU@mn>UaSpS}Vd58Nm0zR3m{q<(oWrbgBk>Ef z$`|M_W|i+I&S6%0lsJc3<<4#Ri&^D%pk1-bTZvzoRep{BVph3@_=Q>JPv|dZl@AiX zFsuA2{l%>EuZVM)Rjwk=VOIHV;v8m`pCZm-R(UJ&2D8dB%PMR17qiOG(O=9ej}YfD ztL!1pVOBXvoWrcLjrfIG717N|OxwLG}#jj@}y^%k$kE#|BI(sr+Z80NJGtpsk0_obg) z5?uw8(>B|Rv(#XzLAy|KY}KK)EWO)T+uN+^Gjn9@azW;`Nm_J8*Uc|YOQl8lXwHoN E|1r=#Qvd(} delta 31047 zcmb__33L?2_C8fLB$*@wB$G@g`@WJy5*8usAuI+MR#{|^AYs@80Ynjq;DSLQ4HT;a zq9BNngvWq@tcp(sB!bGKAo37FWO?{NQ2gJjn%D;BW zmZlWeE#;ztY6T{0bM`^>PG|IlG}Ag#YGM1rJRa4l1EMo)1<}6vAd;3xe{B9*TsY;XTpGi!RIZ3QgYb_?n%puc@FJ^` z*1h1mo6SK)e{6-V(uuem_zsfx#&^emB56sR^q_~lW`0oss^hK;*Ygw7tasQ2R7eU_ z9EE9);sWO*B)yuDgzMwl)W zuBeqM-cCMeZH{Z|{yyay+@9;Ux;d8(Q-qsKGg5mQid{%;hWm1{C<8IKp5lVS0BKQL z3Y{5a@|6~+r5QpUPiull>#Wx@z_d8dl!=>TsaH-TyXK1C340rsVLV6YgblAKcF0cH zk=fc@#>m;6i!lq#M)rzrnp(m=+^tfZCw{QL()>IV@KfBRE>xTwj9nrxqMYm7E zE}dOfj}(9pv2;4CpGVoxJ;)x&PR9TNE|DS|`-SLyQKwF-J13Fl^1 zhvlFI$FRhZN}GIau0yzE9%EYG#FQ+}Z0d_2;mTK-UgL&yFSi19M zh8#Qho>HLwrEX95GL&>X2c^_1nbkZJ`Aa>Tk1>=#H6TptksCmF%qQMbyWC9Z{%WHz zu0>k_nv@q0WqG?G=}L>S@UNaZtt^j}7=;{?R_1(zMo5!dwv;mnPgV?hgy|`at(8?R zo!C0rz!M`nv_mts6arT$!jq-1+O$BEIfcnmX4_{C5o%bpoWaSNrG;~==b1IRon0zy zm!P1Rx9ebtF8jNpR*tW=?=Db+Rr{bv@t-{t%Q0*R$-AYMki9aeMwp zcresgd-S*iN=V7^_+{6o*gDQsGc&=(xR~1_r)Bn4MS7gH%+b_P?0ZKNeyw(~ydJT5 zoUmb>w4ldo_W}EJs)bb=+OwA-Dv*)9^BYKSw}~=j)1gbudJ#9)$RxwUa&C zyR{((bIk=VCXdis*O;U?4#ZVs6W2GH{@D}xN^yPDYK44IJCjR&+g3vy@bsWsfr9F4 zLr}+!g`nLhk+rwYIgZ7Tw6Y%$LEeM!=~rcbvThwEVTa>(bz$xl8{%}2&$$*N_xh}! zgu-K_OO+;n&G{zM&;i4(ec@{L&i>SZ;dlt=CO2Z93fEUE8u$e63+i$jg=x+=aRc%3 z6f~B{gwk7y(EoNH5iCW;#!69xW=Wqsa|KVUs}1$uO%$R{>e9(qNRVV1{Eqb%16oH0 zH^Yl`v=-4!bgGLX6F5s)6M9>_Y1|23E2Dy4`~md{Gw+ z8}2Ir-_(^dM$W|F0He`3OFB2Qy8b&OLH`4t{)#A*-27k(AJD#e?rmiC@iK1h052Aa za%&&%s7U)rsbe}|A1~h9`AAE~Y*i0NI9nP5$`&=N9kkKd9)_YU@}8<)tns)+t@Bt^a9XCrlGD%V~MmU3{MU(r~f+MDnTXlrBSbD-h3^l~mbkUb#13#7vF1HH8Y!W>6FtCOh3WmvWn*{=iL5bP|br$*vrm8mqmBuuI*{l&Z+@W7B=-ctF!Wa-92UtH-e-Cuax zdX!zLHoLDr*rMawMhr67m1F3rw=`$bi|DAdenys*x!4atc|Td{SUeoztDtORJo1)K zFR4IRSsjZ_m6fZ0SN1o;kG**?J@&5rYpEAPk9p5gx>ex7Ofki2tvloACB|6Ix!jF~Kl!B_Eco)dM&8CB=fp(%g-9B;wY`one(y zc}Ya&zc#*&aD|W5;HOaO?e|jgULSVT4r%rXnWTsB`N1_^#|>3kPtyBc1siSIgJoTn;1OwN@-rVSI$)zf^qbGfZ$ABK6#y?0H>cfBrz@mMpz325v}t{rD-tvLU2 zO*9gdPTkH%n`)K1u{&PTPEXavK~k4JGZ_<>i5i~IxDDf(Sn2=KhY0^u=e4TuRgaiO z=M}lH6rr=sFZirkI=R1<^=wV|NKqfpLpK;dd2qXI@|K$s#IAgtZYUOiAPKs+TE*ZV zL{-ght*#z8$yl(+)+l&JP?fw7H8jWh@}P_Hlsy`x(Y?xsTwoT;fRCd=WUo;42@h=bkaGLHFSSz-5M zbFkBW{_kVeNJ}A~ZN$&(G-Q9K)xZ-w>Yvxl%Y}xezHEZypL! zwp9O08s6%wz1zxD?ADPOL_@Q1~c=E4#;k~@};??U(}XDIgxy&#snK2AqsT z_%vLH{ssOF{A!Z*bWJKe%kWhlmx9&`3_;9T!~+eKu{-V&N=`oWgW2NH#i6R-e<#E1*t^JKi3PF_=)?j6X$&K5?|5H z-aK~)f9S_2*cElmatLhotF&H-M0md+3}OB8r+%8@Yw!1d?EGincd_oTNsGtlIbFN$ zRB=IW(GTa0_p#&i+sd^8RwMBHUN1cXg}JXmNQI zsAy&B>n0#F*Z1czgs13iy1Pad_ZOqUtJmfr>&phrYs`*InHj;d8ThCI^|HTig7PxA zN3z`T!+rh5U3kWg5{x$j73d`3&zYMA){Xy*%3V5@CRHVX{Q6rL4J7L@#@|}2o)M#E z-tC#jnqbsD?*Y8yopcq75Bpp0OgBOj(}(*Wpgz9)fI+d5O=AFf=Eot>#O8Z%`$SrJ z6NgJX?qwr;_w1HTZvn?$v|k=Fn5g0%XNMYzDCO_;bqU%5}PTSUSveAwSSRajSK z5#x5`BR`7{x3cM@+wfRR2bDX#2GnbhE3JzSoZ|R953l4F$&V6xzp4n!0_>J%DP($s zPQJ?!vfCh`*q3AUx%rCl z)MdE5Ihj7h$QNI9;Mkz6p)Px4AO9&4m0c7u_hl zJ?SSVR1z6jk1eN;C=6i_xI^I)diao~%Sat~_zM3?bK_FvX?M{X1sX`PwKdn4y6|Mp z8V3l~B9->~4X8D&r@col`BRaPqi4KPM;up=^#>@9w)H`sjbQko$`%Pb*NzlshD8d2 z6s$7QJlCPS&1eE1T06pMU(~~m!2Uo-LT~w^DMo~7u^$?MXL2lgI;^#;p01f0tM9V> zRl8-s&GARWk+rO*-z4^!c(sn|YpQsetFRt2V_a5`C8)pV%0&pDAbO+$`qX@gq1v1W zpbo5qmBstXDY>s6{UiXLHlGv~j@RSqXkV(WAsSLd$Mrx|h0lWt>hqZm`C2r#P)wtP z&?xgw0eM3}()~dYYWv*QaWL1JAk+xqKXhn^SJXhe{W=Qw@vF}*Fw^ycD2wj1A-@nE zm^yyb^&x0KT^)i>p%A(x07cUs{ZWJ*u$Qk^$9J8yBj{R_nF4*+O1p=lW~dQUsSeb4 zE&j6elIhVfl#G+>^N3BR4Z_hj_YwQUV&*^LD4zWrnud0bKzM(l z7_~4`z3$o;iJS>WGS1)l_L*urECrjhaG*|3srfc#NPhhE9t? zEre7tT0)mZqcl9XKHVRU!s+-}6bYB!h((=QeKztuP;CKTFuAztvqw;JG0!E_jCi25 zv%V`Q9;G1sNqx2)L1!hPBy@ri5px}jsj1?UPwLal2`~+Vs-VEGxw6ykU(xi>B%-en zx>mD&0tYKtJL_{-xfUj)0Azh&&_3)Y9is-?u`doLE$+LJ2lZ*oRP-l)AlyGD4FXWl zVs08rM>wp3Hk`A!C;)B(v_U$0!JOWJ2VhzQks-OKG}Sa2;<}KIU{gwS9cJ$bn#!X6 zZ)BqO=6r$K#f@1IZIQX_dCqzQ`5@d+hgsA~Q(5%9*BYTF*!mm;tro~nG=@N!$Q>n* z@Fn9F`RC}W#%Pq0Aiswysayzjq!pFc9FlRj0fPT>e zZMGgWu=tf6u=uQw`O>9|#mA_Y&M`qpbA49>5qi(IL~+P^#{hptD>My1)P){Z@$WQn z^=u7oW+cid{}x)FgEHjhR-L3AOx2cKoq1vn?+zzx*NE(L(-S>(yS8X4P7Kh7^oG5P z;cmm|-+8F3&yxW>q?e8rCLZ3j7`$NRb+43r&oc!$mP9H8gXcpXuR;Zg+~MR) z1>gb!$euogyrikAMP1zyEypW3s(jau-&5<-Ii1ih^FIV$WdM6d;D21Xyd!`Xbw<;C zj)?1r6ld9kw3RgaTrIkwX$YUxnGttSi^zunZpM!S_M$MOK|7OINO*nOyF1uL#W^45 zqbvBP4mCGdQ&%|W_pT_*=Wl`fmqL~f>jo{W!*fmVhQ7kcj~;ru+89}|xH_t9OAmAq zS;GSbAIf)|+T>HhjJG-GFn?vES@ho1$mO$GoP3ks z?u@eR^L|hapXGLyi0eQ5p`~bz>S;L)%j!0e+j0_kr9P;(4_*@N*A^@^Uzq=7Ixov2BqqK^V4h`44ygPugz9}Jje&4(8M zS;yq?Ohx>MKsq@MCmBgDe+2{|dJw!m812SC1oDZj267g?Gz9JRiLvqqh@y8!qGbQ7 z^@_@HdU7b*Zq5|f)2(8JKCP+1uE=@hIyns45qp@}A1RyOE`O$?Sw$kv8jVJqy9vyE zfl05AKz^a&S~PXuS=ybg8^zPd!;m+9vj8Q)#tJW5P=JP_0XkhcBtUMqaJR1bXVJH2 zr_fa>bXfx1Xsf~K7)1MyM9X|$7S-l5H)NxiXu>Eo#JohDT;%5Y(J0{gQY}2o7|(Q< z?{laDLTh!@vY}UXN7PGNH*6R1+uZPxh0vMk;9X4%kuO4fb!ffMY0>!2;f672IkJA? zW~S{&x{ieb^|Vzxmn%SlGMg`~l#D~UMv^?QfA%2Ab`Qz>8@{menFGKDj7QHI!O`t6 zAP0s=Dw@5_dLm&H&|yPKz5p5QL6Rqif$`jg6c2)W6X?N@Nj4oe38qAHE6tvSs5v)? zCrG;>o&dDRWKn@ltmwqaXoq=_s4y^y?m$Pe|S&lI7p*3s;We$vR-VJC4?@|C02lFgzNdV4knJ(9ew|I==L&FrdF4#AYMW@(%4D3W&b$kYe;1UdijqcWY(S zG<8K6)0}aPt=txu?RZ#kY zG`qpQH3N0Bp07!Q_i$Hq91h=5q|XP@eb1vt_&j$SjhYErq}SeTCTRalr@boBDkp)g z{g0XG9jhg{MvIBiwFPU(;>#@*aSLtpB5H3W$m1`ycDcHBU*7J3TuKjlj9V+8OBG;R*}ch_x@F>DJx`MkLQhZSWvYMjof}Yl?X9U@^QHNnT=- z6Ff_XM>@wdr8cLEpj{VQF$XZ_c)-x6VsGJ+F>@i&z%HI>ug^uB&F=~o?*#M7B=nx{ zPIX8g2Xo}nxc{^L5@dP%kli~-^8 zPB_g7K}4L7b})zlS`w%aYQN{aX!Oj(G_LLAFIS7 z=)pJPgm7694O@ni@J*eL(JH%N-0XfaV8{H1>d^Mfkg{z-FR5ksi(sdza8Uz5 zMMn%Ja(H>b9HSD8rE`~|WNU0l&FEuZL(O%(w1OeJJCj(NRfe+sH5H=h+@+`?J+K^F z7`)afJTcfgiJ70KZD@!*-amy>RAwH-SlFDSL-<7C9EFL%NP3l`7ks9RE5(eBSQA*a z0y_MWZrVJ7H68w?6)1|HTLJU>mqWDqTyjv!o{=GHKKm>a@R9;=FZ)?jQ}{;o66aUK zs``2z>V|s3a{9}t%U7WcpGtvRuAtICS3wV@BTa)=qXK+Dhj(~z4LsKC(5>JQnzsz4 z(yx4o7tLG)3l(~>j%#3`{$2-mc}op2pS9?k^`_f(+Q}~KuF-#Zi_4*M_mm=jQ-q3v zAn$L?cGgeH&$YC2N+T5p&=GH=H2Tm9XcKN?(uqvq1G|$nn{ypTE-P)}LqUC?wt>?0DLic^~JbSG> zP3PK;HtYq)*yON#iNbD9dB>i-(-f$(J+bun3bf3;K~P;Ms9NGWE3&K_n~MKbiGDEe z71j0#@7>;s{4#gJssDp9ssBKOEckfs#(ZvQOztv;&@m%pcG=msSvuwj`qBI!LGqp; z`TSAj*W%9qi6ra(bb>D8{|vH^{x= zx8Ns*ISz$^%vOEJ?;TFv_2pUQjm%l0JUpKWFCvxa%0|+I$Y@W85#4-zxSCrS|Xfd+tNHUn;){cjEWI$5ORZM}EMotDT}3@xp4S zhFro++^2&3G3`7}v@=kC7r8i8d;d8vOJNUmv@=j32^r(BsnC(O{~K9d-j{Jlc%?(w zusM_#{e-u;57A~}!_!yrd^klfT*2+|5vDLj+D3wj&FQXC^e;TOnoRmtFjAnpVV3`l zJF_z^aVOn|PJj~#qAP#K71mpdg~}_xbp0J1O1ECa(e&&c9L)ZefgkA1l~i+Yh0?HJ zaHb*9pkHt<^wX1Xsl&Nm^5QQz0pAiX0HD{n7cYF8vuXrD_whoiqPql8%Z@3NWq$9Oqwj$m-j7HqT&2Z~5>D1Tpfd=wB zZsS&HhzEJ^+jx>&K1h+*#t%oB>H#714jyjE9a?54dews@U%xQ2dih;E4XyGZ>G(HJ zaiKkujd*d2d|e#Z~3Un?F`S_rlev_ltkumevj_5il>AufnEaD>`5I|FFgBOH!%JfLVh9WW8H-J;>#5oVQaXuI|Z zZ=j1^ID@+W#7)gZ!+9S(IGpvtmWub3JQx`6O8E=kqQ}l~?k#6H-S9UaY;`gL{lsBe z0O?4aFY2i6RFRzF%Qldq(FTI5eWgulgU7fj?Tbivyr_00zEuh+fi0vrqXt&mHM$16 z+3}u&uEpdn`~mQ9nG?yDww8@??~j3b_u=CjQ~5l`^%fzw5#A^4-4{-G)+J4>`)b-t zAJrvE_=JvS)jb6cB%k#nd4`f7d6Ay@M-P(Syh(3E$#QQp4d3%1ImU&{1^bSPZqGZzztSo9h#JQ#pe9GcIt6rnaz@ zL#s?(!>lBdxCUBD7QqX3s^WHQgvB<1n>NxCS=Smkt)-oOgx}Lyn|)D{UmHQ63?+>W zCHdB^Jsu?g3?&74t*{S(=7o{whCnC7$Vhy~15j~zEs_VrNfUg{gJiP^vVc)weUrv4 zh$7xM1xAv=IM*(Q3MTIeWnFRK}&)^CVIUY8HcU!*JR=H zX+iSg#V8$(k_`&L`;l~M4oNc-rKfVp$M_QulBZgNr{9m{ISN31T9E=npsF|+a)0yy zl+_x>4*?Xc8ljn_ZQ9ow-tK(h0pf8R(n;mQrvS6sl4A5ecQ9R;#aq^u)s8HK9ZXT2 zb66DpwLKYU4U4K7i~N12bh@M|R@S~2sBQzv-*ZW>3d9vo>Ofqu`wX5tXIV6r<$!uV zkBm0;1YPt}6WmCGH@@s|s*w72&+15)B0Ly)sv8H({;4#Z8`721nZWlV#&L73;|$Cg z-i2I4&+E+Ca9e>H7eyC5MKTQ~hYcV(sMLdG=X~-K8YisPV9M5pbR~m)HZe*z=erDs zuZ|_Wt7_L-tajd|U(-9iND4g|0(ZWTyOKo+SL*yw5fUbMGqN9UcPB8NI#SaCY#9Z8 zsk7p68%6v`6n(x2DK!!lecVM4k{f!G=I98szm?Kwr1v7Dd>#s;A1Fr09aUAOy3-TCVjtHheMvCJHmD0jq%C@Re^R6#@p#Z`&+-Dpw8e%cY7ITQogCfDAU2qc3K!G|RRXPnKuVSxI zKB_z%M8=z)j30c|D_YEEDis=|qFwI{CZ`deCQO_bP1g=3tK5gd(-gZj{REr*VrQw2 z-#rx)ARY{Z$)GB$+~&|8!-<0~dkSaDYkPXRi^EBt%7nXj@CdTdBOA(wV06F(8|_^J znAs~Q%U@+XDq74>%+Hc=Z2eZTSLMuR8R(J@|Dh`XZ8Y683Od$rx&I6SnmEZMbl(F2 zpIsQK=Ccdm7&cbRE2}!yBIwu~18~1dLg{y-Nurwx`@`;Nn-!9^)lT`3A*-vMx-^Eg zua!G)&Kbx}rei>_I-!Ra#>*;4j3pU{VC%<{G~Cw%m}ML(WGHMxuTe9~-6~@0abO%- zfzWt*YN|T*z~?m)Epp(a31kMY4C6U4Y$BQHvml1&z-vqpwqFMzA4?z(mpi%nF`IlR z5oBG?YE_%obKY<3cn7JH%VX&4lgUxMJcf4?a5$xi3}uJR3{A!PZfIdf_!O9Cp+h<| z;!-ty#2wnzsU!+nPu0fioQ_p?PeuMz41IYT>1ia%78~$Q4}yi$$pj-o@n&S5Snb?V zuTq<8r?>+wnhNt{T&HG`&e&>?t?6VwnTLR^J~&z?{Ux<=a3ZKM@OaiTM9;gEWJ7hCT;nk z1O^2-{CqBX3*`%Dnu#a(BL{g@{KKptcNdwu=}Y73&|jJoj9KL zk7N02&xR2SnMbj%O$*7l2>ZwJcJz;<8yAs{?nC5H(LNkRs}_^a^s~iq%OtAF=bv6e zXth&`WiTv4EjA`^)anrf%)T=60O7VmS=%^zZYf!AZOhnI>uuOlJL*NzA$8ucMTG-^ zUw6m3aZp>v(nfnOCrfmv;DI!SPF`pewSwqx6ga~vMo@$JC{_^sb_Lmo@O(jVejMGi zihSrkgy7-cShpJVh#K-K=NhuC+NtJiNjy8n^z2ZXgC4f%;M%s9WFov<=-C}d(c7R0 z{+7Q@vix<`d5fZTaBn^$g12bdI?~eoy=c+%pn$F4y{5G2cX8Cbo?NHje2QBn?9`qY zPOi3oQM!(v$}`z${ddS4^v(w2!uNrbdM9vryB3+Rm-uZYPngZ|Jfghg#qN))Z#303 zid=7OBwY{=j@M#d{!3GtyMs1+kDSMe@ypiWB>KcA@LZ#K-UntfHEH>G6C&jlOpK>X zH<20EM0Uj>XmFm8bT1U>B^PZbGeHd>lQdVpa|a;z4ej$j`3w)yF~;2#{or5jg_pLF zJJ#{Fk(j0<5jaH=A0IETTg7S4>IkU!C-mS4!YmQd&tqjXYb^B^ra4>?9HNjh&i^7YH+Ft$tk-@f}zeIekAF*l?O?9nghGxQW#yEQ`N1x{h&Sf0EWW@ z#>aa|b8JbF%5TP_1RC)XDKz9lKZ5C~(*q9KOOgyF`Gjzm2g%KQNhd=|v=7z`7kiML zz7LiPRJ+yNklp*hsCPU_YUf^HJd0FKbGpNUzO$dSzXjYtI2V0}?*RZgm!R$Cde@Jt7KQZu0k}uHF4T58NcP}6 zTD1ku6oKn*weXj1D7I@Z>Gi&Mh?HS-qE>Iw5CujyclBX1#~PYgGx#%)z{*S9vJpgC zc`BKwNQWlUEl0@U7z05yqn-8y`scSK3^lDCrQ|3{`!6UB6X?3{0Ht&7DB6|{1&iTL zU6reB>B>)FJ~7n5(%er;E*h;xpQlO>P1Ma9hbHp4(Nq9K!m-%yH)YttBI~|pOH?;y260XyZz zRW}rmxlIQ?hlR?IbZx((N~52>H5yD4zJ%4vtxfXE|;tc#@kk_i1Uk)1L(6Pm-8HX z8d=ZQ^fawH50CdQ>v$xnqGyw+;{x>EXQ3%!k^_gAFOY&510YS4$<{sgfO_zIcQroKal?a1zC1~XtFjK^n5cfk}lPdw_hY{(OK?8i>C5tNg@p2x&$k})|})TVMzH* zq=Qabs_LAaWLLM#DdXlbYlLk*tQ&VP{+*e`}R zCG& z25!AcI{y~~vH;VIgYDR-bIB1Hm?{tr2Vtdk5_w0=O+ z;i;F_Gri|1@{>~N)(5aOJP8y)?*fO`Kgbq#$XY;C5zOVuQvV@&7H5lOdHW%0VcyJi z+MMr!1bilOlY;zCiYw?5`4i!t0(xf(ZT1(bV21!r*CrA_%R6qq!qV)=@L1tXoux%5 zGQ!?=YZKHpn= z-UI%uBe1(t2j)sTibD`P$6TgiBc+OvTl5L;jLd%v2mB>`mVaGk%{<_ay0<#Y z49Ij%K8XV{9rsThhG0p30S-!hZemf)$Wfz<;FpTVOo%BOF=6zmlt~lEJul9s=J>|I zfAr1*9BfW;pYx^1SK$|2jw;-^F5iQV{=WmauZsiHYTa31FHO6%Uf$Au*DFrWV81k@ zE#2T`zO}yiJKWX$L>kYAj5HCKI8za3)!6>S(6jhW0@23;Gd1L?3DYKyWxocMJL36K zQ<7t*7LJ-&G-|y3yH26O<$S&iA#OC;w=BC z6}?b_FSNevV;YRC1+1DrqSc2SW7D)9f=$IJNTbVrP5HPWjYXJUbEShDPjs#DRO3B7 zFHPGk#l2Xa32p3eYG__A_%0QWvS+`esip+uCFBwhA~xtSb3fNq7R%LT^-M+BT2a$F zk*|Aomi4})$XBG%^#P`yMv|&uPI!=hCD610S8%`3RV6Uk=sog`)ie|TtYcqbkCy|= z-8S^eAk%*HAHpN|-5x1wtEr~-NXKB42|jk*Q0wKA=QNc?H~E3hwA&ioutsy!=^>`6 zII5wxsmm5B;=v7RhTYT!2RG!4LW}LDRGih2MYDTRNMHAX-4tc0B{kI44R`dQ#T9Cb zVp`ZJ-L(`(7;pwl5VLKJJ11zbVoNMh;bQj$rJhnX6i$1>Y&&M^Q1FZZlk z2oTRTbhQdM^+R|%cqb3O);6Cspr1yV7Q0Wt-mmscNDy5VY3jvJG2Ls*xFCgRkwLD; zQKn~MrNY2#{5wwTb@puDsd#NgL)tjTw77Q3ogO5I$JQcA;!KV3F%Odb$G28+JM0=Z z&$;=Fq`Vnts=}9a;Zkv4Vg6i05iavvn9iH;3YXsI?cKrNx3cA^d_>_ORpJ&+?yfrJ3Q)-v0d(|$k$7!-=)F?rCvIpIKW{_8gwkmAs3D*muYs&aKg(2v@H;yCfuAb+ zW%^!?O%=E@T?}7wH?`OK@a5aY)S8Z52T54m1d{M1Gt}lh4$9d*Y*3;cFrTNpt~4=K zA$&famdC)i;Z8L(^}|<~6)D6CAAIZkq^Yf?i7Ei~);$T6&hx?!&6UsH&J^L?sk!L_ z{#$28ZcB|?HJp6{GB~;Vw0x_3_TIZdQ&WT`M|G9wn7lBK%+NM&!RwmJ0yeC*=~J8m zwO~G*kwLxMn3k)DJS=$(W-!J2QI>!D7j~{o1)C)S}T^a36 zGZ7x4qgetAneN@_pQj6&zPYB6*2xAW=H{9jp_w`oWseo{$r&OB4J7%dzJ(qHc??d@ z;3=em)YG@;nZ_afo{s*8#H8x_)OY`=?&IqmW+hFvT+Ij~%8a;n^Tv8Zdog(a2{f7EhR(F@DU1;;EyiEOq8(QcDk0 z>g?0d9L|%hSdO5OSh1fKAF<*AhT?ZtWYvWtlNFzNLGcMIcEEJoxs4U@wd*?0Z&=aa z3`Jj7yy*+Y8?4yu2gSRr=;aTEgB8IRC;~0Bp}fHUd6pG>;3JpLU92dFZx%W~WJM)> zqs_U46$9YYT+XLiv5gg5STPg6vgDl33cD4GU{<`qiW#iv1K$~O_GHCc_~e3f6)Q%- ztG>>mtjK{EGG{xVgwk2biXE)@jTJXokqU3OIg?nC%Zj$F=m&3|IQy_74qg&)Mzg{T z_F6j$D+*u>m~$8_N?_NDb2cj;#6s~qE6%`EY-bfKw!yP0=N4AXn+@ym&bh2O$%-#o z5eBQI&Jb37!ivMJm+`cGK`_9ryOYI%gU5h_x4(n8y@R*AgSWYZx3`10 zwS%{_gSWARx37b@t%J9#gEy&zH>ZO)rGq!4gEygrH=lzyontm{HwSMs2X8M2Zz~6H zCkJmM5Qg?}@TPI_W^wQ)aq#AF@TPF^W^nK(aB%m70pN89cRQ#AZ#$gxz`YLcX$SYR z!#SI|*umZF;I4IWw>r2>9o(G`?n(!Dql3H9QCt|xe8$qIuocgl!sQ}iR<7O@!m!`^VF@!EVvi*d&lvXOAtGbg4>5?0VV}?!8N)uh4jBUl`z*Og z81~_3kudC2#3EtX$6ZCjuuox%gkc{h6bZw=8YmKmebUZ3n`aFB(wWE@_JuByG3+B% zB4gMmYedGd50{99Vej;dgkdj}i-cisVT*)euR4o_NoQHZ6NbGkDiVgh`Xmx2F@ohk z<8FT4W5{NGURg?M)LI^N1x7<89LnI7)(kc>$J$Dlc!yapigo$KX!xQE+_Aj0= sY-v#>3|o5R3DeM-%hHBt3|npx8N+UtM8>em3eOlP%pCgnHRYiH2dxh2t^fc4 diff --git a/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb b/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb index c565e7b07..39e531a60 100644 --- a/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb +++ b/doc/LectureNotes/_build/html/_sources/exercisesweek37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "cfebae03", + "id": "1ff4a4e9", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "353ebb07", + "id": "0569f7a9", "metadata": { "editable": true }, @@ -27,25 +27,34 @@ }, { "cell_type": "markdown", - "id": "c02815dd", + "id": "6f34e507", "metadata": { "editable": true }, "source": [ "## Overarching aims of the exercises this week\n", "\n", - "This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercise is also a part of project 1 and can be reused in the theory part of the project.\n", + "This exercise deals with various mean values and variances in linear\n", + "regression method (here it may be useful to look up chapter 3,\n", + "equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome\n", + "H. Friedman, The Elements of Statistical Learning,\n", + "Springer](https://www.springer.com/gp/book/9780387848570)). The\n", + "exercise is also a part of project 1 and can be reused in the theory\n", + "part of the project.\n", "\n", - "For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "For more discussions on Ridge regression and calculation of\n", + "expectation values, [Wessel van\n", + "Wieringen's](https://arxiv.org/abs/1509.09169) article is highly\n", + "recommended.\n", "\n", - "The assumption we have made is \n", - "that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$\n", - "which describes our data" + "The assumption we have made is that there exists a continuous function\n", + "$f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0,\n", + "\\sigma^2)$ which describes our data" ] }, { "cell_type": "markdown", - "id": "fb152043", + "id": "3ff574a4", "metadata": { "editable": true }, @@ -57,7 +66,7 @@ }, { "cell_type": "markdown", - "id": "9aba7a46", + "id": "096a3165", "metadata": { "editable": true }, @@ -68,7 +77,7 @@ }, { "cell_type": "markdown", - "id": "be52c097", + "id": "cc9fb258", "metadata": { "editable": true }, @@ -80,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "c54979e5", + "id": "daf4eb5d", "metadata": { "editable": true }, @@ -90,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "4b1eb811", + "id": "3b491679", "metadata": { "editable": true }, @@ -102,7 +111,7 @@ }, { "cell_type": "markdown", - "id": "036a5440", + "id": "93a49739", "metadata": { "editable": true }, @@ -114,7 +123,7 @@ }, { "cell_type": "markdown", - "id": "f7fb8094", + "id": "a3fa80db", "metadata": { "editable": true }, @@ -125,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "d7d92995", + "id": "e6e2defd", "metadata": { "editable": true }, @@ -137,7 +146,7 @@ }, { "cell_type": "markdown", - "id": "29eb9701", + "id": "9aa19afa", "metadata": { "editable": true }, @@ -150,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "aa106511", + "id": "5fa82a47", "metadata": { "editable": true }, @@ -162,7 +171,7 @@ }, { "cell_type": "markdown", - "id": "58be1091", + "id": "33099abb", "metadata": { "editable": true }, @@ -172,7 +181,7 @@ }, { "cell_type": "markdown", - "id": "4cb58e41", + "id": "5d03f001", "metadata": { "editable": true }, @@ -184,7 +193,7 @@ }, { "cell_type": "markdown", - "id": "a04df28d", + "id": "94bd151d", "metadata": { "editable": true }, @@ -195,7 +204,7 @@ }, { "cell_type": "markdown", - "id": "c095c96a", + "id": "f0b84941", "metadata": { "editable": true }, @@ -207,20 +216,20 @@ }, { "cell_type": "markdown", - "id": "5d425a79", + "id": "60370a63", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", + "\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", "\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "fb89cfab", + "id": "af251217", "metadata": { "editable": true }, @@ -233,19 +242,19 @@ }, { "cell_type": "markdown", - "id": "aea8801c", + "id": "02e978da", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\mbox{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e885693", + "id": "3656a896", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/html/_sources/week36.ipynb b/doc/LectureNotes/_build/html/_sources/week36.ipynb index bdaee795a..ededd9147 100644 --- a/doc/LectureNotes/_build/html/_sources/week36.ipynb +++ b/doc/LectureNotes/_build/html/_sources/week36.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "7411173e", + "id": "2b900fea", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "f50db913", + "id": "2286e211", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "4a90e380", + "id": "edbcd7f9", "metadata": { "editable": true }, @@ -44,28 +44,34 @@ "\n", "* Material for the lecture on Thursday September 7\n", "\n", - " * Linear Regression and links with Statistics, Resampling methods\n", + " * Technicalities related to scaling and other issues with data handling\n", "\n", - " * Recommended Reading: Goodfellow et al chapter 3 on probability theory, see URL:\"\"\n", + " * Linear Regression and links with Statistics\n", "\n", - " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)" + " * [Recommended Reading: Goodfellow et al chapter 3 on probability theory](https://www.deeplearningbook.org/)\n", + "\n", + " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)\n", + "\n", + " * [Video of lecture](https://youtu.be/Kc20CFK0z7Y) \n", + "\n", + " * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep7.pdf)" ] }, { "cell_type": "markdown", - "id": "351d36fd", + "id": "5dbf3ff5", "metadata": { "editable": true }, "source": [ "## Material for the active learning sessions Tuesday and Wednesday\n", "\n", - "The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples" + "The material here contains a summary from last week and discussion of SVD, Ridge and Lasso regression with examples" ] }, { "cell_type": "markdown", - "id": "0f9f4d2f", + "id": "2dae8175", "metadata": { "editable": true }, @@ -77,7 +83,7 @@ }, { "cell_type": "markdown", - "id": "9ec36bdd", + "id": "fb3d486d", "metadata": { "editable": true }, @@ -89,7 +95,7 @@ }, { "cell_type": "markdown", - "id": "614493c6", + "id": "5f8492a1", "metadata": { "editable": true }, @@ -99,7 +105,7 @@ }, { "cell_type": "markdown", - "id": "dc4be423", + "id": "a35422b4", "metadata": { "editable": true }, @@ -111,7 +117,7 @@ }, { "cell_type": "markdown", - "id": "b262332c", + "id": "aef923ea", "metadata": { "editable": true }, @@ -121,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "ab1829a4", + "id": "5da8277b", "metadata": { "editable": true }, @@ -138,7 +144,7 @@ }, { "cell_type": "markdown", - "id": "7c82dd9e", + "id": "eb34e530", "metadata": { "editable": true }, @@ -148,7 +154,7 @@ }, { "cell_type": "markdown", - "id": "5bab8721", + "id": "a2a33728", "metadata": { "editable": true }, @@ -160,7 +166,7 @@ }, { "cell_type": "markdown", - "id": "72273a4f", + "id": "9bea1837", "metadata": { "editable": true }, @@ -170,7 +176,7 @@ }, { "cell_type": "markdown", - "id": "d4af03ea", + "id": "af711505", "metadata": { "editable": true }, @@ -182,7 +188,7 @@ }, { "cell_type": "markdown", - "id": "c0d35853", + "id": "f02b3738", "metadata": { "editable": true }, @@ -196,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "db5ea827", + "id": "d3fe4855", "metadata": { "editable": true }, @@ -208,7 +214,7 @@ }, { "cell_type": "markdown", - "id": "a6b341ce", + "id": "7b5fa376", "metadata": { "editable": true }, @@ -230,7 +236,7 @@ }, { "cell_type": "markdown", - "id": "db723709", + "id": "d212be4e", "metadata": { "editable": true }, @@ -242,7 +248,7 @@ }, { "cell_type": "markdown", - "id": "1e600268", + "id": "eca13576", "metadata": { "editable": true }, @@ -257,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "b9452938", + "id": "fbf0e2be", "metadata": { "editable": true }, @@ -269,7 +275,7 @@ }, { "cell_type": "markdown", - "id": "53fea1e6", + "id": "acbd86ec", "metadata": { "editable": true }, @@ -281,7 +287,7 @@ }, { "cell_type": "markdown", - "id": "ed10baad", + "id": "8887e338", "metadata": { "editable": true }, @@ -291,7 +297,7 @@ }, { "cell_type": "markdown", - "id": "a4d0db96", + "id": "e95bcbda", "metadata": { "editable": true }, @@ -303,7 +309,7 @@ }, { "cell_type": "markdown", - "id": "fae3cadc", + "id": "e66200c2", "metadata": { "editable": true }, @@ -313,7 +319,7 @@ }, { "cell_type": "markdown", - "id": "e973c775", + "id": "a0c3692c", "metadata": { "editable": true }, @@ -325,7 +331,7 @@ }, { "cell_type": "markdown", - "id": "f88a33f4", + "id": "8d13a9ff", "metadata": { "editable": true }, @@ -335,7 +341,7 @@ }, { "cell_type": "markdown", - "id": "5bc7214d", + "id": "31c3dd91", "metadata": { "editable": true }, @@ -347,7 +353,7 @@ }, { "cell_type": "markdown", - "id": "561d4c91", + "id": "87a6b291", "metadata": { "editable": true }, @@ -364,7 +370,7 @@ }, { "cell_type": "markdown", - "id": "0eda2d1e", + "id": "3bb9eed3", "metadata": { "editable": true }, @@ -378,7 +384,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "c95dbc32", + "id": "ba123060", "metadata": { "collapsed": false, "editable": true @@ -392,7 +398,7 @@ }, { "cell_type": "markdown", - "id": "77079d25", + "id": "46fdfd39", "metadata": { "editable": true }, @@ -403,7 +409,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "cd0ecc44", + "id": "36e47fde", "metadata": { "collapsed": false, "editable": true @@ -443,7 +449,7 @@ }, { "cell_type": "markdown", - "id": "41d896dc", + "id": "53878874", "metadata": { "editable": true }, @@ -463,7 +469,7 @@ }, { "cell_type": "markdown", - "id": "f594852c", + "id": "87546753", "metadata": { "editable": true }, @@ -475,7 +481,7 @@ }, { "cell_type": "markdown", - "id": "b78264f2", + "id": "d8a4669d", "metadata": { "editable": true }, @@ -486,7 +492,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "d05eff95", + "id": "b6523ecb", "metadata": { "collapsed": false, "editable": true @@ -520,7 +526,7 @@ }, { "cell_type": "markdown", - "id": "2fae32c6", + "id": "2fce573f", "metadata": { "editable": true }, @@ -530,7 +536,7 @@ }, { "cell_type": "markdown", - "id": "8631011a", + "id": "840991d3", "metadata": { "editable": true }, @@ -543,7 +549,7 @@ }, { "cell_type": "markdown", - "id": "8e7701de", + "id": "4df7aac6", "metadata": { "editable": true }, @@ -555,7 +561,7 @@ }, { "cell_type": "markdown", - "id": "aea7230b", + "id": "3f3acc05", "metadata": { "editable": true }, @@ -565,7 +571,7 @@ }, { "cell_type": "markdown", - "id": "c5029213", + "id": "2d1218f6", "metadata": { "editable": true }, @@ -578,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "535ad70e", + "id": "01624e31", "metadata": { "editable": true }, @@ -588,7 +594,7 @@ }, { "cell_type": "markdown", - "id": "159ae526", + "id": "616fe159", "metadata": { "editable": true }, @@ -600,7 +606,7 @@ }, { "cell_type": "markdown", - "id": "2f25dfa4", + "id": "27cf2acd", "metadata": { "editable": true }, @@ -615,7 +621,7 @@ }, { "cell_type": "markdown", - "id": "9d187806", + "id": "2ff1e0d2", "metadata": { "editable": true }, @@ -628,7 +634,7 @@ }, { "cell_type": "markdown", - "id": "6c692db5", + "id": "aef1996b", "metadata": { "editable": true }, @@ -642,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "3404e0b4", + "id": "75a44cc9", "metadata": { "editable": true }, @@ -654,7 +660,7 @@ }, { "cell_type": "markdown", - "id": "4bc49666", + "id": "e2ba6037", "metadata": { "editable": true }, @@ -664,7 +670,7 @@ }, { "cell_type": "markdown", - "id": "3f08dfa1", + "id": "95517053", "metadata": { "editable": true }, @@ -677,7 +683,7 @@ }, { "cell_type": "markdown", - "id": "30bcc0cd", + "id": "5f15d4ea", "metadata": { "editable": true }, @@ -689,7 +695,7 @@ }, { "cell_type": "markdown", - "id": "d311a7d8", + "id": "68579bd0", "metadata": { "editable": true }, @@ -701,7 +707,7 @@ }, { "cell_type": "markdown", - "id": "32c46b73", + "id": "53887938", "metadata": { "editable": true }, @@ -713,7 +719,7 @@ }, { "cell_type": "markdown", - "id": "3cceda87", + "id": "469c8c35", "metadata": { "editable": true }, @@ -725,7 +731,7 @@ }, { "cell_type": "markdown", - "id": "10a16675", + "id": "a7521a38", "metadata": { "editable": true }, @@ -739,7 +745,7 @@ }, { "cell_type": "markdown", - "id": "6dba20bf", + "id": "cdfddb46", "metadata": { "editable": true }, @@ -751,7 +757,7 @@ }, { "cell_type": "markdown", - "id": "9325f924", + "id": "e87c8606", "metadata": { "editable": true }, @@ -761,7 +767,7 @@ }, { "cell_type": "markdown", - "id": "745d61e9", + "id": "3e06d524", "metadata": { "editable": true }, @@ -773,7 +779,7 @@ }, { "cell_type": "markdown", - "id": "3866af64", + "id": "798d77a9", "metadata": { "editable": true }, @@ -783,7 +789,7 @@ }, { "cell_type": "markdown", - "id": "710de288", + "id": "8808de88", "metadata": { "editable": true }, @@ -795,7 +801,7 @@ }, { "cell_type": "markdown", - "id": "5893fba0", + "id": "c3825a07", "metadata": { "editable": true }, @@ -807,7 +813,7 @@ }, { "cell_type": "markdown", - "id": "5ef8c59f", + "id": "868b6a19", "metadata": { "editable": true }, @@ -817,7 +823,7 @@ }, { "cell_type": "markdown", - "id": "79b257ae", + "id": "eaba5f13", "metadata": { "editable": true }, @@ -828,7 +834,7 @@ }, { "cell_type": "markdown", - "id": "46270996", + "id": "c9cc77c3", "metadata": { "editable": true }, @@ -840,7 +846,7 @@ }, { "cell_type": "markdown", - "id": "785aebe9", + "id": "4b3e1232", "metadata": { "editable": true }, @@ -856,7 +862,7 @@ }, { "cell_type": "markdown", - "id": "443f3a3f", + "id": "d59ed944", "metadata": { "editable": true }, @@ -869,7 +875,7 @@ }, { "cell_type": "markdown", - "id": "cfb8d4c0", + "id": "e81dd0be", "metadata": { "editable": true }, @@ -881,7 +887,7 @@ }, { "cell_type": "markdown", - "id": "2be1ca1f", + "id": "f26a385c", "metadata": { "editable": true }, @@ -891,7 +897,7 @@ }, { "cell_type": "markdown", - "id": "9ed0aec2", + "id": "aa00d97a", "metadata": { "editable": true }, @@ -903,7 +909,7 @@ }, { "cell_type": "markdown", - "id": "943b1cd5", + "id": "17ba8d1c", "metadata": { "editable": true }, @@ -913,7 +919,7 @@ }, { "cell_type": "markdown", - "id": "4a8a6bff", + "id": "8cf9dc8f", "metadata": { "editable": true }, @@ -925,7 +931,7 @@ }, { "cell_type": "markdown", - "id": "b0371965", + "id": "f0c43d20", "metadata": { "editable": true }, @@ -937,7 +943,7 @@ }, { "cell_type": "markdown", - "id": "38e7b497", + "id": "5b119d73", "metadata": { "editable": true }, @@ -953,7 +959,7 @@ }, { "cell_type": "markdown", - "id": "682f90db", + "id": "1efba158", "metadata": { "editable": true }, @@ -965,7 +971,7 @@ }, { "cell_type": "markdown", - "id": "b5fb7771", + "id": "e5943a40", "metadata": { "editable": true }, @@ -977,7 +983,7 @@ }, { "cell_type": "markdown", - "id": "0c228525", + "id": "167a1a9e", "metadata": { "editable": true }, @@ -987,7 +993,7 @@ }, { "cell_type": "markdown", - "id": "08a400af", + "id": "924b5c9b", "metadata": { "editable": true }, @@ -999,7 +1005,7 @@ }, { "cell_type": "markdown", - "id": "ec0808b1", + "id": "56ac0a65", "metadata": { "editable": true }, @@ -1009,7 +1015,7 @@ }, { "cell_type": "markdown", - "id": "92d3d92f", + "id": "4323c860", "metadata": { "editable": true }, @@ -1021,7 +1027,7 @@ }, { "cell_type": "markdown", - "id": "07486e29", + "id": "12ef13b2", "metadata": { "editable": true }, @@ -1038,7 +1044,7 @@ }, { "cell_type": "markdown", - "id": "9868d4f4", + "id": "d33c685e", "metadata": { "editable": true }, @@ -1050,7 +1056,7 @@ }, { "cell_type": "markdown", - "id": "1793a225", + "id": "b30d436d", "metadata": { "editable": true }, @@ -1062,7 +1068,7 @@ }, { "cell_type": "markdown", - "id": "c7d3fa52", + "id": "37e47a2a", "metadata": { "editable": true }, @@ -1072,7 +1078,7 @@ }, { "cell_type": "markdown", - "id": "47894977", + "id": "3f87ff41", "metadata": { "editable": true }, @@ -1084,7 +1090,7 @@ }, { "cell_type": "markdown", - "id": "d1a11dc8", + "id": "b14d10ff", "metadata": { "editable": true }, @@ -1094,7 +1100,7 @@ }, { "cell_type": "markdown", - "id": "a6cf82de", + "id": "672b64b5", "metadata": { "editable": true }, @@ -1106,7 +1112,7 @@ }, { "cell_type": "markdown", - "id": "bbf7ccc7", + "id": "ef321df6", "metadata": { "editable": true }, @@ -1116,7 +1122,7 @@ }, { "cell_type": "markdown", - "id": "bc6dbb83", + "id": "e67e683d", "metadata": { "editable": true }, @@ -1128,7 +1134,7 @@ }, { "cell_type": "markdown", - "id": "ef5a8292", + "id": "b9bd1d3f", "metadata": { "editable": true }, @@ -1138,7 +1144,7 @@ }, { "cell_type": "markdown", - "id": "d77bf73a", + "id": "b705fb0b", "metadata": { "editable": true }, @@ -1153,7 +1159,7 @@ }, { "cell_type": "markdown", - "id": "265ce93f", + "id": "03679605", "metadata": { "editable": true }, @@ -1165,7 +1171,7 @@ }, { "cell_type": "markdown", - "id": "c19e720c", + "id": "b58e9ee7", "metadata": { "editable": true }, @@ -1175,7 +1181,7 @@ }, { "cell_type": "markdown", - "id": "8cad2962", + "id": "42d8e1b3", "metadata": { "editable": true }, @@ -1187,7 +1193,7 @@ }, { "cell_type": "markdown", - "id": "bf6316a2", + "id": "db624348", "metadata": { "editable": true }, @@ -1199,7 +1205,7 @@ }, { "cell_type": "markdown", - "id": "78413ab6", + "id": "8d37cd9f", "metadata": { "editable": true }, @@ -1211,7 +1217,7 @@ }, { "cell_type": "markdown", - "id": "a77ca41b", + "id": "e88bc71d", "metadata": { "editable": true }, @@ -1221,7 +1227,7 @@ }, { "cell_type": "markdown", - "id": "67b3f67e", + "id": "7bee2106", "metadata": { "editable": true }, @@ -1233,7 +1239,7 @@ }, { "cell_type": "markdown", - "id": "d3571c8b", + "id": "fc1dc754", "metadata": { "editable": true }, @@ -1245,7 +1251,7 @@ }, { "cell_type": "markdown", - "id": "943f822e", + "id": "e5c6d86f", "metadata": { "editable": true }, @@ -1257,7 +1263,7 @@ }, { "cell_type": "markdown", - "id": "2ec0ba01", + "id": "1413c7cf", "metadata": { "editable": true }, @@ -1267,7 +1273,7 @@ }, { "cell_type": "markdown", - "id": "85bf4c9b", + "id": "1b181b9b", "metadata": { "editable": true }, @@ -1279,7 +1285,7 @@ }, { "cell_type": "markdown", - "id": "c81d05f4", + "id": "ca27bd28", "metadata": { "editable": true }, @@ -1289,7 +1295,7 @@ }, { "cell_type": "markdown", - "id": "ce36390d", + "id": "f2ce6515", "metadata": { "editable": true }, @@ -1303,7 +1309,7 @@ }, { "cell_type": "markdown", - "id": "a47dd697", + "id": "bbe27e9c", "metadata": { "editable": true }, @@ -1313,7 +1319,7 @@ }, { "cell_type": "markdown", - "id": "15fe5a0c", + "id": "3ddfa9f9", "metadata": { "editable": true }, @@ -1325,7 +1331,7 @@ }, { "cell_type": "markdown", - "id": "62d7f0e9", + "id": "2d31b195", "metadata": { "editable": true }, @@ -1337,7 +1343,7 @@ }, { "cell_type": "markdown", - "id": "f2d4576f", + "id": "4b092e9b", "metadata": { "editable": true }, @@ -1347,7 +1353,7 @@ }, { "cell_type": "markdown", - "id": "fb2bc0ff", + "id": "7c845085", "metadata": { "editable": true }, @@ -1359,7 +1365,7 @@ }, { "cell_type": "markdown", - "id": "807ac009", + "id": "0fb000c5", "metadata": { "editable": true }, @@ -1369,7 +1375,7 @@ }, { "cell_type": "markdown", - "id": "fe72889b", + "id": "bd4d8817", "metadata": { "editable": true }, @@ -1381,7 +1387,7 @@ }, { "cell_type": "markdown", - "id": "48844e10", + "id": "d0d10a69", "metadata": { "editable": true }, @@ -1393,7 +1399,7 @@ }, { "cell_type": "markdown", - "id": "72a4bc4c", + "id": "d9db36da", "metadata": { "editable": true }, @@ -1403,7 +1409,7 @@ }, { "cell_type": "markdown", - "id": "cfe1db6f", + "id": "a409bd8a", "metadata": { "editable": true }, @@ -1415,7 +1421,7 @@ }, { "cell_type": "markdown", - "id": "05b6e6f6", + "id": "c3be5f03", "metadata": { "editable": true }, @@ -1425,7 +1431,7 @@ }, { "cell_type": "markdown", - "id": "0e58f37a", + "id": "6533b635", "metadata": { "editable": true }, @@ -1437,7 +1443,7 @@ }, { "cell_type": "markdown", - "id": "78593de5", + "id": "3c6b3074", "metadata": { "editable": true }, @@ -1449,7 +1455,7 @@ }, { "cell_type": "markdown", - "id": "26b08a56", + "id": "747eecf9", "metadata": { "editable": true }, @@ -1459,7 +1465,7 @@ }, { "cell_type": "markdown", - "id": "3df661a4", + "id": "87542c6e", "metadata": { "editable": true }, @@ -1471,7 +1477,7 @@ }, { "cell_type": "markdown", - "id": "68460e68", + "id": "13f97260", "metadata": { "editable": true }, @@ -1484,7 +1490,7 @@ }, { "cell_type": "markdown", - "id": "6c660112", + "id": "bc02bc84", "metadata": { "editable": true }, @@ -1496,7 +1502,7 @@ }, { "cell_type": "markdown", - "id": "6ff54cf1", + "id": "9f9d8193", "metadata": { "editable": true }, @@ -1508,7 +1514,7 @@ }, { "cell_type": "markdown", - "id": "f3f88f94", + "id": "ba404c5e", "metadata": { "editable": true }, @@ -1520,7 +1526,7 @@ }, { "cell_type": "markdown", - "id": "6918fc7c", + "id": "9af7054a", "metadata": { "editable": true }, @@ -1530,7 +1536,7 @@ }, { "cell_type": "markdown", - "id": "1b2321a3", + "id": "eb7fd44a", "metadata": { "editable": true }, @@ -1542,7 +1548,7 @@ }, { "cell_type": "markdown", - "id": "d5e21457", + "id": "041b60b2", "metadata": { "editable": true }, @@ -1552,7 +1558,7 @@ }, { "cell_type": "markdown", - "id": "e03c3617", + "id": "f95dc455", "metadata": { "editable": true }, @@ -1564,7 +1570,7 @@ }, { "cell_type": "markdown", - "id": "6c4d9a4a", + "id": "28176b93", "metadata": { "editable": true }, @@ -1574,7 +1580,7 @@ }, { "cell_type": "markdown", - "id": "584bef2e", + "id": "b6f75bf7", "metadata": { "editable": true }, @@ -1586,7 +1592,7 @@ }, { "cell_type": "markdown", - "id": "87273242", + "id": "f6ad6840", "metadata": { "editable": true }, @@ -1596,7 +1602,7 @@ }, { "cell_type": "markdown", - "id": "f7e8277f", + "id": "fe93ca4d", "metadata": { "editable": true }, @@ -1609,7 +1615,7 @@ }, { "cell_type": "markdown", - "id": "e7167c28", + "id": "4d7921c1", "metadata": { "editable": true }, @@ -1621,7 +1627,7 @@ }, { "cell_type": "markdown", - "id": "6a3fa7b2", + "id": "d106e526", "metadata": { "editable": true }, @@ -1633,7 +1639,7 @@ }, { "cell_type": "markdown", - "id": "92c33067", + "id": "2ab11733", "metadata": { "editable": true }, @@ -1643,7 +1649,7 @@ }, { "cell_type": "markdown", - "id": "1b7be5cc", + "id": "c871f92c", "metadata": { "editable": true }, @@ -1655,7 +1661,7 @@ }, { "cell_type": "markdown", - "id": "9e2db51b", + "id": "4fc85aac", "metadata": { "editable": true }, @@ -1672,7 +1678,7 @@ }, { "cell_type": "markdown", - "id": "7ea5c023", + "id": "c66960ef", "metadata": { "editable": true }, @@ -1684,7 +1690,7 @@ }, { "cell_type": "markdown", - "id": "efff1845", + "id": "6c742a89", "metadata": { "editable": true }, @@ -1696,7 +1702,7 @@ }, { "cell_type": "markdown", - "id": "80496d8c", + "id": "5a6fe016", "metadata": { "editable": true }, @@ -1706,7 +1712,7 @@ }, { "cell_type": "markdown", - "id": "bb33463a", + "id": "63472c22", "metadata": { "editable": true }, @@ -1718,7 +1724,7 @@ }, { "cell_type": "markdown", - "id": "f16b1457", + "id": "3ede415b", "metadata": { "editable": true }, @@ -1728,7 +1734,7 @@ }, { "cell_type": "markdown", - "id": "4bee5a43", + "id": "90397204", "metadata": { "editable": true }, @@ -1740,7 +1746,7 @@ }, { "cell_type": "markdown", - "id": "17176990", + "id": "acd78254", "metadata": { "editable": true }, @@ -1750,7 +1756,7 @@ }, { "cell_type": "markdown", - "id": "ec2c8f34", + "id": "114e58b4", "metadata": { "editable": true }, @@ -1762,7 +1768,7 @@ }, { "cell_type": "markdown", - "id": "5934144d", + "id": "dc251844", "metadata": { "editable": true }, @@ -1772,7 +1778,7 @@ }, { "cell_type": "markdown", - "id": "fe097c51", + "id": "2fb1ac98", "metadata": { "editable": true }, @@ -1787,7 +1793,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "92cd1361", + "id": "8c64672d", "metadata": { "collapsed": false, "editable": true @@ -1849,7 +1855,7 @@ }, { "cell_type": "markdown", - "id": "9e4426f4", + "id": "a87e0be8", "metadata": { "editable": true }, @@ -1859,7 +1865,7 @@ }, { "cell_type": "markdown", - "id": "5f2b6935", + "id": "ac5bf925", "metadata": { "editable": true }, @@ -1870,7 +1876,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "409bbeac", + "id": "078628d7", "metadata": { "collapsed": false, "editable": true @@ -1937,7 +1943,7 @@ }, { "cell_type": "markdown", - "id": "67a24e77", + "id": "f035fb09", "metadata": { "editable": true }, @@ -1948,7 +1954,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "8da16a74", + "id": "acb14894", "metadata": { "collapsed": false, "editable": true @@ -2037,7 +2043,7 @@ }, { "cell_type": "markdown", - "id": "9d2563fa", + "id": "391b11f6", "metadata": { "editable": true }, @@ -2047,962 +2053,850 @@ }, { "cell_type": "markdown", - "id": "1232037e", + "id": "242933c8", "metadata": { "editable": true }, "source": [ - "## Linking the regression analysis with a statistical interpretation\n", + "## Important technicalities: More on Rescaling data\n", "\n", - "We will now couple the discussions of ordinary least squares, Ridge\n", - "and Lasso regression with a statistical interpretation, that is we\n", - "move from a linear algebra analysis to a statistical analysis. In\n", - "particular, we will focus on what the regularization terms can result\n", - "in. We will amongst other things show that the regularization\n", - "parameter can reduce considerably the variance of the parameters\n", - "$\\beta$.\n", + "When you are comparing your own code with for example **Scikit-Learn**'s\n", + "library, there are some technicalities to keep in mind. The examples\n", + "here demonstrate some of these aspects with potential pitfalls.\n", + "\n", + "The discussion here focuses on the role of the intercept, how we can\n", + "set up the design matrix, what scaling we should use and other topics\n", + "which tend confuse us.\n", + "\n", + "The intercept can be interpreted as the expected value of our\n", + "target/output variables when all other predictors are set to zero.\n", + "Thus, if we cannot assume that the expected outputs/targets are zero\n", + "when all predictors are zero (the columns in the design matrix), it\n", + "may be a bad idea to implement a model which penalizes the intercept.\n", + "Furthermore, in for example Ridge and Lasso regression, the default solutions\n", + "from the library **Scikit-Learn** (when not shrinking $\\beta_0$) for the unknown parameters\n", + "$\\boldsymbol{\\beta}$, are derived under the assumption that both $\\boldsymbol{y}$ and\n", + "$\\boldsymbol{X}$ are zero centered, that is we subtract the mean values.\n", + "\n", + "If our predictors represent different scales, then it is important to\n", + "standardize the design matrix $\\boldsymbol{X}$ by subtracting the mean of each\n", + "column from the corresponding column and dividing the column with its\n", + "standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,\n", + "the results may differ. \n", "\n", "The\n", - "advantage of doing linear regression is that we actually end up with\n", - "analytical expressions for several statistical quantities. \n", - "Standard least squares and Ridge regression allow us to\n", - "derive quantities like the variance and other expectation values in a\n", - "rather straightforward way.\n", + "[Standardscaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html)\n", + "function in **Scikit-Learn** does this for us. For the data sets we\n", + "have been studying in our various examples, the data are in many cases\n", + "already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a\n", + "survey of your data, with a critical assessment of them in case you need to scale the data.\n", "\n", - "It is assumed that $\\varepsilon_i\n", - "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", - "independent, i.e.:" + "If you need to scale the data, not doing so will give an *unfair*\n", + "penalization of the parameters since their magnitude depends on the\n", + "scale of their corresponding predictor.\n", + "\n", + "Suppose as an example that you \n", + "you have an input variable given by the heights of different persons.\n", + "Human height might be measured in inches or meters or\n", + "kilometers. If measured in kilometers, a standard linear regression\n", + "model with this predictor would probably give a much bigger\n", + "coefficient term, than if measured in millimeters.\n", + "This can clearly lead to problems in evaluating the cost/loss functions.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0e71daa5", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" ] }, { "cell_type": "markdown", - "id": "e50205be", + "id": "1f0eb0d0", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "2ffc4acb", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mbox{Cov}(\\varepsilon_{i_1},\n", - "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", - "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", - "\\end{align*}\n", + "C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70eccda9", + "id": "2511c028", "metadata": { "editable": true }, "source": [ - "The randomness of $\\varepsilon_i$ implies that\n", - "$\\mathbf{y}_i$ is also a random variable. In particular,\n", - "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", - "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", - "non-random scalar. To specify the parameters of the distribution of\n", - "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", "\n", - "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", - "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", - "row number $i$ and perform a sum over all values $p$." + "What we have done is to single out the $\\beta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\beta_0$, we want the derivative to obey" ] }, { "cell_type": "markdown", - "id": "e6bdcbf2", - "metadata": { - "editable": true - }, - "source": [ - "## Assumptions made\n", - "\n", - "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", - "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", - "which describe our data" - ] - }, - { - "cell_type": "markdown", - "id": "d74471b8", + "id": "908beb7f", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "\\frac{\\partial C}{\\partial \\beta_j} = 0,\n", "$$" ] }, { "cell_type": "markdown", - "id": "f7b178d9", + "id": "5fae2a94", "metadata": { "editable": true }, "source": [ - "We approximate this function with our model from the solution of the linear regression equations, that is our\n", - "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + "for all $j$. For $\\beta_0$ we have" ] }, { "cell_type": "markdown", - "id": "8ea16c62", + "id": "d7729a27", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n", "$$" ] }, { "cell_type": "markdown", - "id": "4a8d7bfd", + "id": "7fbb2ded", "metadata": { "editable": true }, "source": [ - "## Expectation value and variance\n", - "\n", - "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + "Multiplying away the constant $2/n$, we obtain" ] }, { "cell_type": "markdown", - "id": "5367d943", + "id": "8cef22ce", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mathbb{E}(y_i) & =\n", - "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", - "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", - "\\end{align*}\n", + "\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c7eefd85", + "id": "c38ab3a0", "metadata": { "editable": true }, "source": [ - "while\n", - "its variance is" + "Let us specialize first to the case where we have only two parameters $\\beta_0$ and $\\beta_1$.\n", + "Our result for $\\beta_0$ simplifies then to" ] }, { "cell_type": "markdown", - "id": "27ce343a", + "id": "c47e7bbe", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", - "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", - "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", - "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", - "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", - "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", - "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", - "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", - "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", - "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", - "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", - "\\end{align*}\n", + "n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c67ea20b", + "id": "06962abd", "metadata": { "editable": true }, "source": [ - "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", - "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + "We obtain then" ] }, { "cell_type": "markdown", - "id": "b4da473a", - "metadata": { - "editable": true - }, - "source": [ - "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", - "\n", - "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" - ] - }, - { - "cell_type": "markdown", - "id": "af812d38", + "id": "14c0a845", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "9d3aaff5", + "id": "5f50f223", "metadata": { "editable": true }, "source": [ - "This means that the estimator of the regression parameters is unbiased.\n", - "\n", - "We can also calculate the variance\n", - "\n", - "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + "If we define" ] }, { "cell_type": "markdown", - "id": "eebea8b7", + "id": "188ed44d", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{eqnarray*}\n", - "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", - "\\\\\n", - "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", - "\\\\\n", - "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", - "% \\\\\n", - "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", - "\\\\\n", - "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", - "\\end{eqnarray*}\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", "$$" ] }, { "cell_type": "markdown", - "id": "714f281a", + "id": "0b0db7b8", "metadata": { "editable": true }, "source": [ - "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", - "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", - "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", - "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", - "variance of the estimate of the $j$-th regression coefficient:\n", - "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", - "construct a confidence interval for the estimates.\n", - "\n", - "In a similar way, we can obtain analytical expressions for say the\n", - "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", - "when we employ Ridge regression, allowing us again to define a confidence interval. \n", - "\n", - "It is rather straightforward to show that" + "and the mean value of the outputs as" ] }, { "cell_type": "markdown", - "id": "66015ca6", + "id": "e7550136", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", "$$" ] }, { "cell_type": "markdown", - "id": "267f01c0", + "id": "c7cc678d", "metadata": { "editable": true }, "source": [ - "We see clearly that \n", - "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", - "\n", - "We can also compute the variance as" + "we have" ] }, { "cell_type": "markdown", - "id": "998de4b6", + "id": "9c9abcb2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\beta_0 = \\mu_y - \\beta_1\\mu_{\\boldsymbol{x}_1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "06a8d452", + "id": "2336bf55", "metadata": { "editable": true }, "source": [ - "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", - "\n", - "With this, we can compute the difference" + "In the general case with more parameters than $\\beta_0$ and $\\beta_1$, we have" ] }, { "cell_type": "markdown", - "id": "6cd3ed9b", + "id": "e0111655", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70f9aa69", + "id": "709c3af5", "metadata": { "editable": true }, "source": [ - "The difference is non-negative definite since each component of the\n", - "matrix product is non-negative definite. \n", - "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + "We can rewrite the latter equation as" ] }, { "cell_type": "markdown", - "id": "3bb18388", - "metadata": { - "editable": true - }, - "source": [ - "## Deriving OLS from a probability distribution\n", - "\n", - "Our basic assumption when we derived the OLS equations was to assume\n", - "that our output is determined by a given continuous function\n", - "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", - "distribution with zero mean value and an undetermined variance\n", - "$\\sigma^2$.\n", - "\n", - "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", - "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", - "the design matrix are not stochastic variables, we can assume that the\n", - "probability distribution of our targets is also a normal distribution\n", - "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", - "single output $y_i$ is given by the Gaussian distribution" - ] - }, - { - "cell_type": "markdown", - "id": "84b546af", + "id": "a9ed060b", "metadata": { "editable": true }, "source": [ "$$\n", - "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\beta_j,\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e90935e", + "id": "deb61dec", "metadata": { "editable": true }, "source": [ - "## Independent and Identically Distrubuted (iid)\n", - "\n", - "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", - "We define this distribution as" + "where we have defined" ] }, { "cell_type": "markdown", - "id": "b40b33f5", + "id": "dc60f20f", "metadata": { "editable": true }, "source": [ "$$\n", - "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", "$$" ] }, { "cell_type": "markdown", - "id": "23e0c7ac", + "id": "b43a095d", "metadata": { "editable": true }, "source": [ - "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", "\n", - "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" ] }, { "cell_type": "markdown", - "id": "dcd4ed9c", + "id": "050949a2", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n", "$$" ] }, { "cell_type": "markdown", - "id": "5c37adb5", + "id": "7c288c56", "metadata": { "editable": true }, "source": [ - "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", - "in case we have a simple one-dimensional input and output case" + "If we minimize with respect to $\\boldsymbol{\\beta}$ we have then" ] }, { "cell_type": "markdown", - "id": "27e4eea9", + "id": "72d25a93", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", "$$" ] }, { "cell_type": "markdown", - "id": "5dd95110", + "id": "6e4fe379", "metadata": { "editable": true }, "source": [ - "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", - "We can now rewrite the above probability as" + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\beta}^T\\boldsymbol{\\beta}$ to the cost function and get then" ] }, { "cell_type": "markdown", - "id": "bc7cfda7", + "id": "b0f0e6e7", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "2bf6dbf4", + "id": "6cfb015d", "metadata": { "editable": true }, "source": [ - "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "bfbf5d1d", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_beta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_beta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "beta = fit_beta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True beta: {true_beta}\")\n", + "print(f\"Fitted beta: {beta}\")\n", + "print(f\"Sklearn fitted beta: {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ beta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "beta = fit_beta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ beta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted beta (wiothout intercept): {beta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted beta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ beta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "e8ff9386", + "id": "7dc13bd7", "metadata": { "editable": true }, "source": [ - "## Maximum Likelihood Estimation (MLE)\n", + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", "\n", - "In statistics, maximum likelihood estimation (MLE) is a method of\n", - "estimating the parameters of an assumed probability distribution,\n", - "given some observed data. This is achieved by maximizing a likelihood\n", - "function so that, under the assumed statistical model, the observed\n", - "data is the most probable. \n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\beta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", "\n", - "We will assume here that our events are given by the above Gaussian\n", - "distribution and we will determine the optimal parameters $\\beta$ by\n", - "maximizing the above PDF. However, computing the derivatives of a\n", - "product function is cumbersome and can easily lead to overflow and/or\n", - "underflowproblems, with potentials for loss of numerical precision.\n", - "\n", - "In practice, it is more convenient to maximize the logarithm of the\n", - "PDF because it is a monotonically increasing function of the argument.\n", - "Alternatively, and this will be our option, we will minimize the\n", - "negative of the logarithm since this is a monotonically decreasing\n", - "function.\n", - "\n", - "Note also that maximization/minimization of the logarithm of the PDF\n", - "is equivalent to the maximization/minimization of the function itself." + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" ] }, { "cell_type": "markdown", - "id": "cd665e76", - "metadata": { - "editable": true - }, - "source": [ - "## A new Cost Function\n", - "\n", - "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" - ] - }, - { - "cell_type": "markdown", - "id": "144c5c88", + "id": "3ce23f71", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\beta_j^2,\n", "$$" ] }, { "cell_type": "markdown", - "id": "76c7c5c6", + "id": "f34b4a50", "metadata": { "editable": true }, "source": [ - "which becomes" + "but when we take out the intercept, this equation becomes" ] }, { "cell_type": "markdown", - "id": "f78405b0", + "id": "6c535531", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\beta_j^2.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dd7b69b5", + "id": "dc9bd9cc", "metadata": { "editable": true }, "source": [ - "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + "For Lasso regression we have" ] }, { "cell_type": "markdown", - "id": "a1430a20", + "id": "b5911fe2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\beta_j\\vert.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dfcd0c7b", + "id": "4d4103f1", "metadata": { "editable": true }, "source": [ - "which leads to the well-known OLS equation for the optimal paramters $\\beta$" - ] - }, - { - "cell_type": "markdown", - "id": "043d37a6", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "28fe8edc", - "metadata": { - "editable": true - }, - "source": [ - "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." - ] - }, - { - "cell_type": "markdown", - "id": "b600d21c", - "metadata": { - "editable": true - }, - "source": [ - "## More basic Statistics and Bayes' theorem\n", + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", "\n", - "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", - "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "eecec942", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", "\n", - "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", "\n", - "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", - "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", "\n", - "**Union of events is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "d513cf2c", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "d18008b5", - "metadata": { - "editable": true - }, - "source": [ - "**The product rule (aka joint probability) is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "22f55ec4", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "29e1bff6", - "metadata": { - "editable": true - }, - "source": [ - "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", "\n", - "If we have independent events then $p(X,Y)=p(X)p(Y)$." - ] - }, - { - "cell_type": "markdown", - "id": "333869a5", - "metadata": { - "editable": true - }, - "source": [ - "## Marginal Probability\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" - ] - }, - { - "cell_type": "markdown", - "id": "bf8de993", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1c5349d1", - "metadata": { - "editable": true - }, - "source": [ - "## Conditional Probability\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "The conditional probability, if $p(Y) > 0$, is" - ] - }, - { - "cell_type": "markdown", - "id": "dbede038", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "c1334a9c", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeBeta\n", + " ypredictOwnRidge = X_test @ OwnRidgeBeta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta)\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", "\n", - "If we combine the conditional probability with the marginal probability and the standard product rule, we have" - ] - }, - { - "cell_type": "markdown", - "id": "ac8dcf04", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1992e354", - "metadata": { - "editable": true - }, - "source": [ - "which we can rewrite as" - ] - }, - { - "cell_type": "markdown", - "id": "24e5179f", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "42979e5a", - "metadata": { - "editable": true - }, - "source": [ - "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." - ] - }, - { - "cell_type": "markdown", - "id": "86a8cd07", - "metadata": { - "editable": true - }, - "source": [ - "## Interpretations of Bayes' Theorem\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", "\n", - "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", - "evaluated for the observed data $Y$ and can be viewed as a function of\n", - "the parameter space represented by $X$. This function is not\n", - "necesseraly normalized and is normally called the likelihood function.\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7d67706b", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "598d20bd", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", "\n", - "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", "\n", - "Let us try to illustrate Bayes' theorem through an example." - ] - }, - { - "cell_type": "markdown", - "id": "b9a4f9a9", - "metadata": { - "editable": true - }, - "source": [ - "## Example of Usage of Bayes' theorem\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "Let us suppose that you are undergoing a series of mammography scans in\n", - "order to rule out possible breast cancer cases. We define the\n", - "sensitivity for a positive event by the variable $X$. It takes binary\n", - "values with $X=1$ representing a positive event and $X=0$ being a\n", - "negative event. We reserve $Y$ as a classification parameter for\n", - "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", "\n", - "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", "\n", - "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" - ] - }, - { - "cell_type": "markdown", - "id": "888c64ec", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=1) =0.8.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "5086b046", - "metadata": { - "editable": true - }, - "source": [ - "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", - "It is however not correct, as the following Bayesian analysis shows." - ] - }, - { - "cell_type": "markdown", - "id": "609ea2b0", - "metadata": { - "editable": true - }, - "source": [ - "## Doing it correctly\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", - "Let us assume that the prior probability in the population as a whole is" + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta) #Intercept is given by mean of target variable\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "687bfdf3", + "id": "cec80003", "metadata": { "editable": true }, "source": [ - "$$\n", - "p(Y=1) =0.004.\n", - "$$" + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\beta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." ] }, { "cell_type": "markdown", - "id": "c2af1d2d", + "id": "16cdcffb", "metadata": { "editable": true }, "source": [ - "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" - ] - }, - { - "cell_type": "markdown", - "id": "b8ec8d93", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=0) =0.1.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "48a6fb5b", - "metadata": { - "editable": true - }, - "source": [ - "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" - ] - }, - { - "cell_type": "markdown", - "id": "ba37d4fd", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "824f1789", - "metadata": { - "editable": true - }, - "source": [ - "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" - ] - }, - { - "cell_type": "markdown", - "id": "07f89fd5", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem and Ridge and Lasso Regression\n", + "## Test Function for what happens with OLS, Ridge and Lasso\n", "\n", "Hitherto we have discussed Ridge and Lasso regression in terms of a\n", "linear analysis. This may to many of you feel rather technical and\n", "perhaps not that intuitive. The question is whether we can develop a\n", "more intuitive way of understanding what Ridge and Lasso express.\n", "\n", - "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit." - ] - }, - { - "cell_type": "markdown", - "id": "8fcd73d9", - "metadata": { - "editable": true - }, - "source": [ - "## Test Function for what happens with OLS, Ridge and Lasso\n", + "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. \n", "\n", "We will play around with a study of the values for the optimal\n", "parameters $\\boldsymbol{\\beta}$ using OLS, Ridge and Lasso regression. For\n", @@ -3015,8 +2909,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "id": "4402e113", + "execution_count": 11, + "id": "e04dce1d", "metadata": { "collapsed": false, "editable": true @@ -3043,10 +2937,9 @@ "X = np.zeros((len(x),Maxpolydegree))\n", "X[:,0] = 1.0\n", "\n", - "for polydegree in range(1, Maxpolydegree):\n", - " for degree in range(polydegree):\n", - " X[:,degree] = x**(degree)\n", "\n", + "for polydegree in range(1,Maxpolydegree):\n", + " X[:,polydegree] = x**(polydegree)\n", "\n", "# We split the data in test and training data\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", @@ -3090,7 +2983,7 @@ }, { "cell_type": "markdown", - "id": "b700e45e", + "id": "2ae202c3", "metadata": { "editable": true }, @@ -3100,43 +2993,475 @@ }, { "cell_type": "markdown", - "id": "69659d9d", + "id": "4186f9f0", "metadata": { "editable": true }, "source": [ - "## Invoking Bayes' theorem\n", + "## Linking the regression analysis with a statistical interpretation\n", "\n", - "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "We will now couple the discussions of ordinary least squares, Ridge\n", + "and Lasso regression with a statistical interpretation, that is we\n", + "move from a linear algebra analysis to a statistical analysis. In\n", + "particular, we will focus on what the regularization terms can result\n", + "in. We will amongst other things show that the regularization\n", + "parameter can reduce considerably the variance of the parameters\n", + "$\\beta$.\n", "\n", - "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + "The\n", + "advantage of doing linear regression is that we actually end up with\n", + "analytical expressions for several statistical quantities. \n", + "Standard least squares and Ridge regression allow us to\n", + "derive quantities like the variance and other expectation values in a\n", + "rather straightforward way.\n", + "\n", + "It is assumed that $\\varepsilon_i\n", + "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", + "independent, i.e.:" ] }, { "cell_type": "markdown", - "id": "80b423eb", + "id": "7293d2ab", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "\\begin{align*} \n", + "\\mbox{Cov}(\\varepsilon_{i_1},\n", + "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", + "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", + "\\end{align*}\n", "$$" ] }, { "cell_type": "markdown", - "id": "de547949", + "id": "efee3703", "metadata": { "editable": true }, "source": [ - "is given by" + "The randomness of $\\varepsilon_i$ implies that\n", + "$\\mathbf{y}_i$ is also a random variable. In particular,\n", + "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", + "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", + "non-random scalar. To specify the parameters of the distribution of\n", + "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "\n", + "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", + "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", + "row number $i$ and perform a sum over all values $p$." ] }, { "cell_type": "markdown", - "id": "99d9476a", + "id": "85d2303d", + "metadata": { + "editable": true + }, + "source": [ + "## Assumptions made\n", + "\n", + "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", + "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", + "which describe our data" + ] + }, + { + "cell_type": "markdown", + "id": "c1a189c0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0b63e61f", + "metadata": { + "editable": true + }, + "source": [ + "We approximate this function with our model from the solution of the linear regression equations, that is our\n", + "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + ] + }, + { + "cell_type": "markdown", + "id": "14acfe14", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "681408c6", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance\n", + "\n", + "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + ] + }, + { + "cell_type": "markdown", + "id": "d956e33c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mathbb{E}(y_i) & =\n", + "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", + "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "992bee5b", + "metadata": { + "editable": true + }, + "source": [ + "while\n", + "its variance is" + ] + }, + { + "cell_type": "markdown", + "id": "fd75d1af", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", + "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", + "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", + "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", + "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", + "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", + "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", + "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", + "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", + "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", + "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f702411d", + "metadata": { + "editable": true + }, + "source": [ + "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", + "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + ] + }, + { + "cell_type": "markdown", + "id": "b86e729d", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", + "\n", + "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" + ] + }, + { + "cell_type": "markdown", + "id": "5af888a2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3d21a65e", + "metadata": { + "editable": true + }, + "source": [ + "This means that the estimator of the regression parameters is unbiased.\n", + "\n", + "We can also calculate the variance\n", + "\n", + "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + ] + }, + { + "cell_type": "markdown", + "id": "7d8043ab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{eqnarray*}\n", + "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", + "\\\\\n", + "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", + "\\\\\n", + "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", + "% \\\\\n", + "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", + "\\\\\n", + "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", + "\\end{eqnarray*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a126ec7", + "metadata": { + "editable": true + }, + "source": [ + "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", + "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", + "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", + "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", + "variance of the estimate of the $j$-th regression coefficient:\n", + "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", + "construct a confidence interval for the estimates.\n", + "\n", + "In a similar way, we can obtain analytical expressions for say the\n", + "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", + "when we employ Ridge regression, allowing us again to define a confidence interval. \n", + "\n", + "It is rather straightforward to show that" + ] + }, + { + "cell_type": "markdown", + "id": "7287992a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2803f793", + "metadata": { + "editable": true + }, + "source": [ + "We see clearly that \n", + "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", + "\n", + "We can also compute the variance as" + ] + }, + { + "cell_type": "markdown", + "id": "d7443c20", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a0d32eb0", + "metadata": { + "editable": true + }, + "source": [ + "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", + "\n", + "With this, we can compute the difference" + ] + }, + { + "cell_type": "markdown", + "id": "cbd0143e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6de88a4", + "metadata": { + "editable": true + }, + "source": [ + "The difference is non-negative definite since each component of the\n", + "matrix product is non-negative definite. \n", + "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + ] + }, + { + "cell_type": "markdown", + "id": "55630c9f", + "metadata": { + "editable": true + }, + "source": [ + "## Deriving OLS from a probability distribution\n", + "\n", + "Our basic assumption when we derived the OLS equations was to assume\n", + "that our output is determined by a given continuous function\n", + "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", + "distribution with zero mean value and an undetermined variance\n", + "$\\sigma^2$.\n", + "\n", + "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", + "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", + "the design matrix are not stochastic variables, we can assume that the\n", + "probability distribution of our targets is also a normal distribution\n", + "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", + "single output $y_i$ is given by the Gaussian distribution" + ] + }, + { + "cell_type": "markdown", + "id": "745e6352", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ec8c116", + "metadata": { + "editable": true + }, + "source": [ + "## Independent and Identically Distrubuted (iid)\n", + "\n", + "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", + "We define this distribution as" + ] + }, + { + "cell_type": "markdown", + "id": "f92be14e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "edbdd556", + "metadata": { + "editable": true + }, + "source": [ + "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "\n", + "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "6a6903ca", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "57680896", + "metadata": { + "editable": true + }, + "source": [ + "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", + "in case we have a simple one-dimensional input and output case" + ] + }, + { + "cell_type": "markdown", + "id": "9309640f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67fb3df9", + "metadata": { + "editable": true + }, + "source": [ + "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", + "We can now rewrite the above probability as" + ] + }, + { + "cell_type": "markdown", + "id": "894f0fb7", "metadata": { "editable": true }, @@ -3148,7 +3473,508 @@ }, { "cell_type": "markdown", - "id": "aa5613ea", + "id": "13f3f705", + "metadata": { + "editable": true + }, + "source": [ + "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + ] + }, + { + "cell_type": "markdown", + "id": "6bf3edfa", + "metadata": { + "editable": true + }, + "source": [ + "## Maximum Likelihood Estimation (MLE)\n", + "\n", + "In statistics, maximum likelihood estimation (MLE) is a method of\n", + "estimating the parameters of an assumed probability distribution,\n", + "given some observed data. This is achieved by maximizing a likelihood\n", + "function so that, under the assumed statistical model, the observed\n", + "data is the most probable. \n", + "\n", + "We will assume here that our events are given by the above Gaussian\n", + "distribution and we will determine the optimal parameters $\\beta$ by\n", + "maximizing the above PDF. However, computing the derivatives of a\n", + "product function is cumbersome and can easily lead to overflow and/or\n", + "underflowproblems, with potentials for loss of numerical precision.\n", + "\n", + "In practice, it is more convenient to maximize the logarithm of the\n", + "PDF because it is a monotonically increasing function of the argument.\n", + "Alternatively, and this will be our option, we will minimize the\n", + "negative of the logarithm since this is a monotonically decreasing\n", + "function.\n", + "\n", + "Note also that maximization/minimization of the logarithm of the PDF\n", + "is equivalent to the maximization/minimization of the function itself." + ] + }, + { + "cell_type": "markdown", + "id": "be42ac0c", + "metadata": { + "editable": true + }, + "source": [ + "## A new Cost Function\n", + "\n", + "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" + ] + }, + { + "cell_type": "markdown", + "id": "b6fbab08", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "33ef1983", + "metadata": { + "editable": true + }, + "source": [ + "which becomes" + ] + }, + { + "cell_type": "markdown", + "id": "7d6504da", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "03d8775c", + "metadata": { + "editable": true + }, + "source": [ + "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + ] + }, + { + "cell_type": "markdown", + "id": "176d1dab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e20e33bf", + "metadata": { + "editable": true + }, + "source": [ + "which leads to the well-known OLS equation for the optimal paramters $\\beta$" + ] + }, + { + "cell_type": "markdown", + "id": "94f73eae", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f5d2cf20", + "metadata": { + "editable": true + }, + "source": [ + "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." + ] + }, + { + "cell_type": "markdown", + "id": "c32b8f92", + "metadata": { + "editable": true + }, + "source": [ + "## More basic Statistics and Bayes' theorem\n", + "\n", + "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", + "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "\n", + "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "\n", + "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", + "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", + "\n", + "**Union of events is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "a3779fd2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "37e13b18", + "metadata": { + "editable": true + }, + "source": [ + "**The product rule (aka joint probability) is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "bd8b41f0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "71b22e26", + "metadata": { + "editable": true + }, + "source": [ + "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "\n", + "If we have independent events then $p(X,Y)=p(X)p(Y)$." + ] + }, + { + "cell_type": "markdown", + "id": "5f6645d2", + "metadata": { + "editable": true + }, + "source": [ + "## Marginal Probability\n", + "\n", + "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" + ] + }, + { + "cell_type": "markdown", + "id": "7349fe65", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a6205b4f", + "metadata": { + "editable": true + }, + "source": [ + "## Conditional Probability\n", + "\n", + "The conditional probability, if $p(Y) > 0$, is" + ] + }, + { + "cell_type": "markdown", + "id": "43cb130d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d8f1813", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem\n", + "\n", + "If we combine the conditional probability with the marginal probability and the standard product rule, we have" + ] + }, + { + "cell_type": "markdown", + "id": "d2746770", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f3659d1d", + "metadata": { + "editable": true + }, + "source": [ + "which we can rewrite as" + ] + }, + { + "cell_type": "markdown", + "id": "8dfd9ec7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3f247ad1", + "metadata": { + "editable": true + }, + "source": [ + "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." + ] + }, + { + "cell_type": "markdown", + "id": "a2615306", + "metadata": { + "editable": true + }, + "source": [ + "## Interpretations of Bayes' Theorem\n", + "\n", + "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", + "evaluated for the observed data $Y$ and can be viewed as a function of\n", + "the parameter space represented by $X$. This function is not\n", + "necesseraly normalized and is normally called the likelihood function.\n", + "\n", + "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "\n", + "Let us try to illustrate Bayes' theorem through an example." + ] + }, + { + "cell_type": "markdown", + "id": "91eae9e8", + "metadata": { + "editable": true + }, + "source": [ + "## Example of Usage of Bayes' theorem\n", + "\n", + "Let us suppose that you are undergoing a series of mammography scans in\n", + "order to rule out possible breast cancer cases. We define the\n", + "sensitivity for a positive event by the variable $X$. It takes binary\n", + "values with $X=1$ representing a positive event and $X=0$ being a\n", + "negative event. We reserve $Y$ as a classification parameter for\n", + "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "\n", + "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "\n", + "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "76fe1c46", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=1) =0.8.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "397d9768", + "metadata": { + "editable": true + }, + "source": [ + "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", + "It is however not correct, as the following Bayesian analysis shows." + ] + }, + { + "cell_type": "markdown", + "id": "f2c701f5", + "metadata": { + "editable": true + }, + "source": [ + "## Doing it correctly\n", + "\n", + "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", + "Let us assume that the prior probability in the population as a whole is" + ] + }, + { + "cell_type": "markdown", + "id": "0fdb2639", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1) =0.004.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f9ae9f95", + "metadata": { + "editable": true + }, + "source": [ + "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" + ] + }, + { + "cell_type": "markdown", + "id": "a681f1dc", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=0) =0.1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4d2c6030", + "metadata": { + "editable": true + }, + "source": [ + "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" + ] + }, + { + "cell_type": "markdown", + "id": "67a1c582", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "969f368b", + "metadata": { + "editable": true + }, + "source": [ + "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" + ] + }, + { + "cell_type": "markdown", + "id": "356a70c0", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem and Ridge and Lasso Regression\n", + "\n", + "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "\n", + "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + ] + }, + { + "cell_type": "markdown", + "id": "fa653f05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5dd1081", + "metadata": { + "editable": true + }, + "source": [ + "is given by" + ] + }, + { + "cell_type": "markdown", + "id": "731b21d4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "362ae4e2", "metadata": { "editable": true }, @@ -3158,7 +3984,7 @@ }, { "cell_type": "markdown", - "id": "7dea4e11", + "id": "8ba56ac5", "metadata": { "editable": true }, @@ -3170,7 +3996,7 @@ }, { "cell_type": "markdown", - "id": "99ab1d57", + "id": "658e6364", "metadata": { "editable": true }, @@ -3180,7 +4006,7 @@ }, { "cell_type": "markdown", - "id": "170ee68b", + "id": "28d6553d", "metadata": { "editable": true }, @@ -3192,7 +4018,7 @@ }, { "cell_type": "markdown", - "id": "3388d6a4", + "id": "8f4ec5f0", "metadata": { "editable": true }, @@ -3202,7 +4028,7 @@ }, { "cell_type": "markdown", - "id": "a5bdaa16", + "id": "e72b34ca", "metadata": { "editable": true }, @@ -3218,7 +4044,7 @@ }, { "cell_type": "markdown", - "id": "cbf8e7e6", + "id": "0e656652", "metadata": { "editable": true }, @@ -3230,7 +4056,7 @@ }, { "cell_type": "markdown", - "id": "52c9e48f", + "id": "57606d08", "metadata": { "editable": true }, @@ -3240,7 +4066,7 @@ }, { "cell_type": "markdown", - "id": "1ae37a89", + "id": "cf40e3a9", "metadata": { "editable": true }, @@ -3252,7 +4078,7 @@ }, { "cell_type": "markdown", - "id": "dd8fee27", + "id": "65038f20", "metadata": { "editable": true }, @@ -3265,7 +4091,7 @@ }, { "cell_type": "markdown", - "id": "de603b44", + "id": "deb9890c", "metadata": { "editable": true }, @@ -3277,7 +4103,7 @@ }, { "cell_type": "markdown", - "id": "3f08e4a0", + "id": "1a844813", "metadata": { "editable": true }, @@ -3287,7 +4113,7 @@ }, { "cell_type": "markdown", - "id": "b65ee2f0", + "id": "a32e4148", "metadata": { "editable": true }, @@ -3299,7 +4125,7 @@ }, { "cell_type": "markdown", - "id": "6526b426", + "id": "4050a8ed", "metadata": { "editable": true }, @@ -3309,7 +4135,7 @@ }, { "cell_type": "markdown", - "id": "29190fe5", + "id": "ec31e4e2", "metadata": { "editable": true }, @@ -3321,7 +4147,7 @@ }, { "cell_type": "markdown", - "id": "71b1585c", + "id": "f872c397", "metadata": { "editable": true }, @@ -3333,7 +4159,7 @@ }, { "cell_type": "markdown", - "id": "e18baaa9", + "id": "e8eeb2e5", "metadata": { "editable": true }, @@ -3343,7 +4169,7 @@ }, { "cell_type": "markdown", - "id": "1c58b8db", + "id": "54d2f39d", "metadata": { "editable": true }, @@ -3355,7 +4181,7 @@ }, { "cell_type": "markdown", - "id": "238ee8c5", + "id": "54cbc3a8", "metadata": { "editable": true }, @@ -3367,7 +4193,7 @@ }, { "cell_type": "markdown", - "id": "46389b82", + "id": "7ee728b6", "metadata": { "editable": true }, @@ -3379,7 +4205,7 @@ }, { "cell_type": "markdown", - "id": "0ecbb2d2", + "id": "00e9a0eb", "metadata": { "editable": true }, @@ -3389,7 +4215,7 @@ }, { "cell_type": "markdown", - "id": "27368dfb", + "id": "50f72b22", "metadata": { "editable": true }, @@ -3401,7 +4227,7 @@ }, { "cell_type": "markdown", - "id": "27082f48", + "id": "d53fea2e", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/html/exercisesweek37.html b/doc/LectureNotes/_build/html/exercisesweek37.html index e9f863c97..de203096b 100644 --- a/doc/LectureNotes/_build/html/exercisesweek37.html +++ b/doc/LectureNotes/_build/html/exercisesweek37.html @@ -438,11 +438,20 @@ doconce format html exercisesweek37.do.txt -->

Date: Deadline is Sunday September 17 at midnight

Overarching aims of the exercises this week

-

This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer). The exercise is also a part of project 1 and can be reused in the theory part of the project.

-

For more discussions on Ridge regression and calculation of expectation values, Wessel van Wieringen’s article is highly recommended.

-

The assumption we have made is -that there exists a continuous function \(f(\boldsymbol{x})\) and a normal distributed error \(\boldsymbol{\varepsilon}\sim N(0, \sigma^2)\) -which describes our data

+

This exercise deals with various mean values and variances in linear +regression method (here it may be useful to look up chapter 3, +equation (3.8) of Trevor Hastie, Robert Tibshirani, Jerome +H. Friedman, The Elements of Statistical Learning, +Springer). The +exercise is also a part of project 1 and can be reused in the theory +part of the project.

+

For more discussions on Ridge regression and calculation of +expectation values, Wessel van +Wieringen’s article is highly +recommended.

+

The assumption we have made is that there exists a continuous function +\(f(\boldsymbol{x})\) and a normal distributed error \(\boldsymbol{\varepsilon}\sim N(0, +\sigma^2)\) which describes our data

\[ \boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon} @@ -488,7 +497,7 @@ A given parameter \(\beta_j\)

Show that

\[ -\mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\ +\mathbb{E} \big[ \hat{\boldsymbol{\beta}}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\ \boldsymbol{\beta}^{\mathrm{OLS}}. \]

We see clearly that @@ -496,7 +505,7 @@ A given parameter \(\beta_j\)

Show also that the variance is

\[ -\mbox{Var}[\boldsymbol{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +\mbox{Var}[\hat{\boldsymbol{\beta}}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, \]

and it is easy to see that if the parameter \(\lambda\) goes to infinity then the variance of Ridge parameters \(\boldsymbol{\beta}\) goes to zero.

diff --git a/doc/LectureNotes/_build/html/searchindex.js b/doc/LectureNotes/_build/html/searchindex.js index dbe10ecb7..4db0feb6b 100644 --- a/doc/LectureNotes/_build/html/searchindex.js +++ b/doc/LectureNotes/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["chapter1","chapter10","chapter11","chapter12","chapter13","chapter2","chapter3","chapter4","chapter5","chapter6","chapter7","chapter8","chapter9","chapteroptimization","clustering","exercisesweek34","exercisesweek35","exercisesweek36","exercisesweek37","intro","linalg","project1","schedule","statistics","teachers","textbooks","week34","week35","week36","week37"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["chapter1.ipynb","chapter10.ipynb","chapter11.ipynb","chapter12.ipynb","chapter13.ipynb","chapter2.ipynb","chapter3.ipynb","chapter4.ipynb","chapter5.ipynb","chapter6.ipynb","chapter7.ipynb","chapter8.ipynb","chapter9.ipynb","chapteroptimization.ipynb","clustering.ipynb","exercisesweek34.ipynb","exercisesweek35.ipynb","exercisesweek36.ipynb","exercisesweek37.ipynb","intro.md","linalg.ipynb","project1.ipynb","schedule.md","statistics.ipynb","teachers.md","textbooks.md","week34.ipynb","week35.ipynb","week36.ipynb","week37.ipynb"],objects:{},objnames:{},objtypes:{},terms:{"0":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,24,26,27,28,29],"00":[0,1,5,6,9,11,26,27],"000":[1,3],"0000":9,"00000":[9,11],"000000":[5,11,26,27],"00000000e":[5,27],"000054":26,"0000e":[],"0001":[1,17],"00010403373827253124":[],"00011551":27,"000148":[],"00017471":[],"00018425":[],"00019998":5,"00024087":5,"00025531":[],"00028369228101198006":[],"00029012":5,"0003043256065368937":[],"0003153514830957865":6,"00031535148309580783":6,"0003153514830958081":[],"0003153514830958126":[],"0003153514830958235":[],"00033779":[],"00034944":5,"00040953":[],"000417932":2,"00042089":5,"000464088":2,"00046714":[],"00050694":5,"00054144":[],"00054272":[],"00057106":[],"00058016":6,"000599":[],"00060705":6,"00060708":[],"00061058":5,"00062582":[],"00062595":6,"00063364":29,"00063862":29,"00066668":6,"00068088":29,"00068734":6,"00068941":[],"00068946":6,"00069103":[],"00071305":[],"00072037":27,"00073541":5,"00074389":[],"0007480407244119591":[],"00075315":[],"000755":[],"00075534":[],"00075597":29,"00076495":6,"00076905":6,"00076919":[],"00076925":[],"0007698473260556325":6,"0007698473260556334":[],"0007698473260556339":[],"0007698473260556344":6,"0007790285013223805":[],"00079123":[],"00079129":6,"00079130":[],"00079910":29,"00079968":5,"00083346":29,"00084705":6,"00084711":[],"000854409696581":[],"00085883":[],"00085884":[],"00085889":6,"00086553":[],"00087697":6,"00088573":5,"00089187":29,"00089459":[],"00092645":[],"00092646":[],"00092647":6,"00092904":29,"00096314":5,"001":[1,2,8,13,17],"00100519":6,"0010059":27,"00102864":[],"0010479245355337968":[],"0010479245887943952":[],"0010479245926411787":[6,29],"00105081":6,"001060":11,"00106025":11,"00106677":5,"00107405":6,"00111756":6,"00112431":[],"0011526":6,"00115999":5,"0011828302640124":[],"00118368":[],"00118504":[],"00118508":6,"00118527":[],"00119699":29,"001263":[],"00128479":5,"00128634":[],"00129543e":[],"00131939":[],"001323":6,"00137818":6,"00137823":[],"00137835":[],"00139198":[],"00139705":5,"00140849":29,"00143188":27,"00145838":[],"00147197":[],"00149311":6,"0014937":[],"00149956":6,"00152117":6,"00152188":[],"00154733":5,"00154860":29,"00156376":6,"00156379":[],"00156382":[],"00158723":[],"00162091":[],"00168141":[],"00168251":5,"00174276":6,"00174508":[],"00175331":6,"00183687e":[],"00186347":5,"00187154":27,"001880":5,"00191249":[],"001988":[],"00198806":[],"00200":8,"00201915":[],"00202464":[],"00202624":5,"00202756":6,"00207627":[],"00217499":6,"0021877":[],"00219194":29,"00220229":[],"00224413":5,"00226569":27,"00228741":[],"00228742":6,"00228986":27,"00229179":20,"00232255":27,"00233155":[],"00233441":[],"0023548":6,"002381316302584885":[],"002381316302584886":6,"0023813163025848865":6,"00242954":29,"00242999":6,"00243186":6,"0024401":5,"00249435":6,"0025074":[],"0025125":[],"00252344":[],"002526":[],"0026081":[],"00266858":2,"00267599":[],"00270244":5,"00274989":6,"00275135":[],"00281835":[],"00283636":[],"00287154":[],"00289406e":[],"00289724":6,"00293838":5,"002948":[],"0030828":[],"003083":[],"003100":26,"00310113":2,"00310568":11,"003106":11,"00312361":6,"00315593":6,"003215318065760509":[],"0032153180657605116":[6,29],"0032153180657605125":[],"00321694":27,"00323332":6,"0032542":5,"00325450":29,"00327082":[],"00329264":27,"003301":6,"00330907":27,"0033955154592040923":[6,29],"003395515459204093":[],"0033955154592040944":[],"00353655":27,"00353823":5,"0035417":[],"00355118":[],"00355175":[],"003620":26,"0036237":6,"0036367":6,"00363986":27,"00369758":6,"003704":[],"003717":[],"00373019":[],"003738":[],"003759":[],"003774":[],"003788":[],"003788445263115483":5,"0038335":6,"00388494":[],"00390021":[],"00390805":27,"003909404072811217":[],"003909404072811221":[6,29],"003909404072811231":[],"00391839":5,"00393128":[],"00393744":27,"00397836":[],"00399247":[],"0039987":6,"004":[5,28,29],"00403785":[],"00406231":[],"00407056":[],"004091940707753925":[6,29],"004091940707753948":[],"0040919407077539514":[],"00410387":6,"00410478":6,"004113634617443116":[],"0041136346174431284":[],"004113634617443131":[6,27],"004113634617443135":[],"004113634617443139":[6,27],"00411363461744314":[6,27],"004113634617443141":[],"004113634617443147":[6,27],"00415473":[],"0041559863458613296":[6,29],"004155986345861364":[],"004155986345861374":[],"00415763":[],"00416461":[],"00420072":29,"00424909":2,"00424967":6,"00426027":5,"00427039":[],"00431511":[],"00432302":[],"00433417":11,"004339135748752641":9,"00437363":[],"00440346":6,"00443743":6,"00444561":[],"00445655":11,"00449439":[],"00452742":[],"004579219539673834":[6,29],"004579219539673836":[],"00458014":[],"00458671":[],"00458878":6,"004610275230656182":[6,29],"004610275230656187":[],"004610275230656294":[],"00462287":6,"00465099":29,"00465403":[],"00471782":5,"00471966":27,"00472199":6,"00472303":[],"00472512":6,"00477722":[],"00478786":[],"004788":[],"0049544":6,"004999999999999984":[],"0049999999999999845":0,"00499999999999999":[],"004999999999999994":[],"004999999999999996":[],"004999999999999997":[],"005":[],"0050000000000000044":[],"005000000000000011":[],"00503218":27,"00506501":[],"00506765":27,"00509527":[],"00512927":5,"00514913":27,"00517074":[],"00517114":6,"00523269":[],"00526348":6,"0053018":6,"00534085":23,"00537912e":[],"00538287":27,"00546438":[],"00548686":[],"0055043":[],"00554475":[],"00554552":6,"00556826":6,"00563446":[],"00564956":[],"0056799":5,"00569317":[],"00577016":[],"00577441":[],"00578738":[],"00578776":[],"00579953":6,"0058263":23,"00588657":6,"00605451":[],"00607783":6,"00611932":[],"006158537606875468":[],"006162":6,"00617499":5,"00622755":[],"00625843":[],"00626339":27,"006275173922218319":[],"00628788":[],"00630331":6,"00634741e":[],"00642221":6,"00645183":[],"00654702":27,"00658717":[],"00659545":27,"00660427":6,"00660801":[],"00661104":[],"0066916":[],"00672607":6,"00673393":[],"00673407":6,"00676387":6,"0068011":6,"0068094":[],"00681644":[],"00683748":5,"00683964":6,"00684221":[],"00689391":[],"00695357":[],"00695738":[],"00700232":[],"007012403613997257":[],"007024126888936694":[],"007024126888938144":[6,29],"0070241268889382116":[],"0070381":27,"00704468":[],"00706459":[],"00708794":[],"007098683295819775":[],"00712899":27,"00715835":[],"00719176":6,"0072559952738542694":[],"00727646693":[0,26],"007315":[26,27],"00732273":[],"007334080205654388":[],"00733497":[],"00740173378493795":[],"0074331":5,"00749579":27,"00751823":[],"00754532":[],"00758025":[],"00759119":6,"00770648":[],"00771046":27,"0077750600806588575":27,"00779538":[],"00779804":[],"00780259":[],"00781035":[],"00781918":29,"007839875253055148":[],"00784393":6,"00791019":27,"00791291":[],"00801078":[],"00803064":6,"00805074":29,"00806407":[],"00808144":[],"0080866254785146":[],"008089688732934681":[],"00813803":6,"00817056":[],"00817631":6,"00819724":[],"00820764":[],"00823002":5,"00827728":6,"008278885543361304":[],"00831018":6,"008334493024006613":[],"00834567":6,"00839839":[],"008435888964550838":[],"0084603":[],"00848904":6,"00850988":[],"00851638e":[],"00852965":27,"008550594684081035":[],"00857688":27,"0086649156":[0,26],"008675369724975977":5,"008675369724976501":[],"00867698":[],"00868337":[],"008754614401844201":[],"00877989":[],"008818897251043893":[],"008823125370709272":[],"00882596":[],"00886682":[],"00894639":5,"008947823579448178":[],"00895315":27,"00905423":6,"00910241":27,"00911938":[],"009163":[],"00916336":[],"009163470508352211":[],"009163470508352218":5,"009164545680330616":[6,29],"00917248":6,"00918914":27,"00928016":27,"009290411029763584":[],"00934327e":27,"00934499":6,"00934728":[],"00943261":[],"009450756365829578":[],"00950269957492606":[],"00953728":27,"00954257":27,"0096208":6,"00963223":[],"00965032":[],"0097018":[],"009710035292693498":[],"0097577":[],"0098629":[],"00990475":5,"00992331":6,"00996754":6,"009981":[],"00998135":[],"00it":[],"01":[0,1,2,5,6,9,11,13,17,25,26,27],"010018312644139205":[],"010018312644139219":[6,29],"010018312644139347":[],"01003763":[],"01006893":[],"0100706":6,"01011376":[],"0101339":27,"010154":[],"01023891":[],"010239":[],"01028231":[],"01028255":27,"01028995427580139":[],"01029589":[],"010331721306655144":[],"01033172130665515":[],"010331721306655165":[6,29],"01037947":[],"01039549":27,"01049472":[],"010516485576646504":[6,29],"010516485576646513":[],"010516485576652856":[],"010530":[],"01053024":[],"01056549":[],"010596520060222926":27,"01066519":6,"010687707010849668":20,"01072866":[],"01073921":[],"01073938":[],"010751622516868333":[],"01076611":5,"010902":26,"01097227":27,"0110":23,"0110407067093945":[],"01104071":[],"011076219011339788":[],"01107621901137465":[],"01107621901137467":[6,29],"01121347":[],"011225":2,"01122997":[],"0113104":6,"01132947":27,"01133171":[],"01135107":[],"0113975":[],"01152107":[],"0116726":27,"01167264":[],"01176895":[],"01178253":27,"01179792":6,"01191824":5,"01194406":[],"01199624e":[],"01199845":[],"012":[],"01200342":[],"01205286":27,"0120561":23,"0120561022775605":23,"01206708":[],"012073649439965807":[],"012073649469946107":[6,29],"012073649472576395":[],"01207435":[],"01212754811385597":[],"01219292":6,"01222822":27,"01223198":6,"01224109":[],"01227458":[],"01231917":6,"0123679":[],"01242273":[],"01242803":[],"01244295e":[],"01247056":[],"01248731":[],"012501013881387798":[],"01256189":[],"01257962":[],"012580":[],"012633802944855959":[],"01265755":27,"012658":27,"01285424":[],"01290947":6,"01294617":[],"01295356":5,"013121573975499602":[],"013121573975499604":[],"01312157406137079":[],"013121574061370796":[],"013121574062587286":[6,29],"01317074":27,"01318643":6,"01322099":[],"01322543":[],"01323884":[],"01326608":[],"01347916":6,"01348565":6,"013537225644261203":[],"01354598394614281":[],"0136671":[],"01367553":6,"01374437":27,"01381723":[],"01397146":6,"01400329":[],"01405935":6,"01406736":[],"01416528":6,"014209325470380275":[],"01422494":[],"01422853":[],"0143088":[],"01433809":5,"014436800088896274":[],"014436800088896381":[6,29],"014436800088969727":[],"01448144":[],"01449782":6,"01458337":6,"014586":27,"01458611":27,"014599964106338128":[],"0146081":6,"01463049":6,"01471878":[],"01472766":27,"01479194":[],"01487742":[],"01489692":27,"01491299":27,"01492855":27,"01494533":[],"01497781":[],"015072388895177088":[],"0150723888951771":6,"015072388895177109":[],"015072388895177157":6,"015072388895177239":[],"01509543":[],"01510929":[],"01515966":27,"01525206":[],"01530715721129232":[],"01531845":6,"01538461538462":[],"01539":[],"01542944":[],"01547992":27,"01549377":6,"01550656":[],"01551946":[],"01558197":5,"01558461":[],"0156548":[],"01572164":[],"01574091":[],"01574375":[],"01586048":[],"01588614":[],"01588619":[],"01591355407242949":[],"01594452":29,"01614046":[],"01619802":[],"016285782696017055":[],"016285782696017142":[6,29],"016285782696075054":[],"01628823":27,"01631138":[],"01633913":6,"01637147":27,"01640891":6,"0164286":[],"01644452":27,"01646892":[],"01654471":[],"01655318":6,"01655455":27,"0165866":[],"016587414993037307":[],"016587414993045335":[6,29],"016587414993045405":[],"01667873":27,"01678286":[],"01691985":6,"01693":[],"01693278":[],"0169643":5,"01708781":6,"01708852":6,"01710429":20,"01713366":6,"0171347":[],"01724499":5,"01725808":[],"01735584819559184":[],"017355848195591845":[],"01735584819559331":[6,29],"017355848195593312":[6,29],"017355848195593354":[],"01742183":[],"01751529":[],"01752962e":[],"01756972":[],"01761413":[],"017665":5,"01770163":[],"0177181":27,"01780471":[],"017805":[],"01783414e":27,"01788423":[],"01799917e":[],"01801774":[],"01816821":[],"01817152":[],"01817919":[],"018232":[26,27],"01826743":27,"01831036e":[],"01831130e":[],"01831200e":[],"01831207e":6,"01831251e":6,"0183718":[],"01840062":[],"01844381":[],"01844727":23,"01852793":[],"01854824":27,"01855624":27,"01866537":6,"01869785e":[],"01871582":[],"01873344":11,"01873409":[],"01873869":5,"01885351":[],"01889099091713886":27,"0189082":[],"01898855":6,"01904583":27,"01905883":6,"01907538":[],"01908936":6,"01923991":[],"01927709":[],"01929643":27,"01933058":[],"0193453":[],"01943082":[],"01944669":[],"01950574":27,"01957153":[],"019587":[],"01960258":[],"01963611":6,"01969004537138309":[],"01969145":6,"01975416527168255":[6,29],"019754165271682844":[],"01975416527179247":[],"01975848":6,"01990155":[],"02":[0,4,6,7,12,26],"02002781":[],"02008059":[],"0202458":[],"020246":[],"02024962":6,"0202731":[],"02035804":[],"02054837e":6,"02055673":20,"02058701":[],"020634":[],"02063403":[],"02068067":6,"02073509":5,"02082761":[],"02086933":[],"02090061":[],"0209518407597062":[],"020964":[],"02098261":6,"0211496":[],"02123176":6,"021250026482402":[],"02126934":[],"02129803":[],"02143332":[],"0214539":[],"02151219":27,"02158709":[],"02159270458799264":[],"021592704587992645":[],"021592704588021164":[],"021592704588021167":[],"021592704588021174":[6,29],"021592704588021178":[6,29],"02160095e":[],"02166269":[],"02185737":20,"02193035":27,"02195273":[],"02198702e":6,"02198703e":6,"022":[],"02200044":[],"022022882954618364":[],"02206138":[],"022075":[],"02207532":[],"02208512":6,"02212776":[],"02212893":[],"02226654":[],"02228115":6,"02229529":6,"02232779":[],"02233576":[],"02237669":[],"02251524":27,"02252765":5,"02255062":[],"022556":[],"02255619":[],"02259051":[],"02263868":[],"02283536":[],"02291121":27,"02298589":[],"02299949826036602":[],"022999498260366198":[6,29],"0229994982603662":[],"02335522":[],"02341603":[],"02344111":[],"02344605":27,"0234505":[],"02348765":6,"02354476":[],"023545":[],"02365049":6,"02376036":[],"02377668":[],"02383034":[],"02401344":[],"024023115996453476":[],"02408959":[],"0242106":[],"02421521":[],"02422503":[],"02424794":29,"02430520e":[],"02442398":[],"0245528":6,"02480935":[],"02492265":5,"02493054":[],"02498832":6,"02502101":27,"02503753":6,"02511518":6,"02521356":27,"02521597":[],"02522069":6,"025325212488352":20,"02533184":27,"02539993":[],"02541796":[],"02546675":29,"02551463":[],"02566258":[],"02568505":[],"025709":[11,27],"02586427":6,"02596838":[],"0260906":6,"02609885":27,"026121042099059435":23,"02612167":[],"02625193":8,"02625928":[],"026408391362671896":[],"02642055e":[],"02644748":27,"02649453":[],"026605727637176654":[],"026605727637184554":[6,29],"026605727637184558":[6,29],"02660572763718461":[],"026605727637184613":[],"02670606":27,"02697521163514974":[],"026986":[],"02698605":[],"02699523":[],"02701924":[],"02705154":27,"02707227":5,"02708158":[],"02712723":[],"02723445":6,"02724253":[],"0272773":23,"02747522":[],"027609773491022314":[],"02760977349102238":[6,29],"027609773491022387":[],"027609773491022394":[6,29],"027609773491022407":[],"02762405215108776":[],"02766489e":[],"02766981":[],"02768957":[],"02775293":27,"02783644":[],"02786488":[],"027865":[],"02803041":[],"02808201":[],"02818883":[],"0283589":27,"02840224":[],"02840884":[],"02857":4,"02862254":[],"02868501":[],"02897168":27,"02900131":[],"02903947":[],"02917662":[],"029177":[],"02917931":[],"02925853":[],"02926396":[],"02928941":[],"02944194":[],"029483":5,"02951554e":[],"029574060388349064":[],"029613363972682966":[],"02970592":[],"02971158":[],"029733":[26,27],"02976145":6,"02979908":[],"02994311":5,"02f":[6,21],"02it":[],"03":[1,6,27],"03012368":[],"030169":[],"03016916":[],"03026672":[],"03028379":27,"03032441e":6,"030407120354722424":[],"03054765":[],"03056589":[],"030566":[],"030693":[],"03069324":[],"03071040e":[],"03076923076924":[],"03077640549":4,"03078597":[],"03079528":[],"03080664":[],"03099776":5,"031":[5,28,29],"03123832":[],"03168642":[],"0317159":[],"03173163":[],"031846":[],"03184647":[],"03187054":[],"03187339273559067":[],"03192893":[],"03196357":6,"032":[],"03206303":[],"03224523":27,"03232013":[],"03232719":[],"03235599":[],"03251863":5,"03255258":[],"03256632e":1,"03260201":[],"03265139":[],"03267527":6,"03278964":[],"032790":[],"03279636":6,"03286166":5,"032862":5,"033":[],"0330308045180234":[],"0330308045183163":[],"0330308045183219":6,"0330308045188872":6,"0330308045190182":[],"03308408":5,"03324117":[],"03326365":[],"033264":[],"03330335":27,"03352185":[],"03352725":[],"033657685071527485":[],"033657685071527624":[],"03365768507152769":[6,29],"03371334":[],"03382304823545749":[],"03393764":11,"033938":11,"03399521":27,"034047":26,"03413302":27,"03416188":20,"0341899":27,"03423556":[],"03437582":[],"03447512":6,"034557":27,"0348171":[],"03483677":[],"03499693245669077":[],"03500198":[],"03507211":[],"03509912":[],"03510756":[],"03523311":[],"03533092":[],"03536062":27,"03536673":[],"03546779":[],"03562355":6,"03568439":6,"03570747":[],"03583342":[],"0358909447132981":[],"035909":26,"0359565":5,"03608726":20,"03610331":[],"03613949":[],"03617972":[],"03630548":6,"03634298":[],"0365416391166":23,"03654164":23,"03707133":11,"03718926":[],"03724642":[],"03739357":[],"03739417":[],"037503978471253":[],"037559":[],"03755944":[],"03761519":[],"03781367141738885":[],"03781367141738886":[],"037813671417388985":[],"03781367141738899":[],"03781367141738902":[6,29],"03791967":[],"03794112":[],"03807267":[],"038073":[],"03814292":6,"03815288":6,"03822007730337899":11,"038300":[11,27],"03850557":[],"038506":[],"03859372":[],"03893106":[],"039039":5,"0391902":[],"03932541e":[],"03952093":27,"03981057":[],"03995478":[],"0399676689527966":6,"0399676689527968":[],"0399676689527975":6,"04":[1,6,11],"04008203e":[],"040102":5,"04010697":6,"0402861":20,"04028979":[],"04039087":[],"04046799":[],"04063602":6,"04075841e":[],"040863":[],"04086326":[],"04087592":[],"041":9,"04100738":[],"04101587":[],"041129694365319464":20,"041148729430502":[],"041148729430523":6,"0411487294305246":[],"0411487294305746":[],"041148729430595":6,"04117184":27,"04120281":[],"04179719":[],"041797190646905":[],"04184636":[],"04184782":[],"04193203":27,"04208001":[],"04208274":27,"04220758":6,"04234339":[],"042708":[],"04273116":[],"04279651270127165":[],"04284519":[],"04295757":29,"043":9,"04315108":5,"04316854":26,"04331431":[],"0433816":[],"04346721":5,"04355837":6,"04362":9,"04368707":[],"0437499":2,"04389027":6,"043927769551648":[],"04413933503955871":[],"04419696":[],"04423486":6,"04426744e":27,"04426816":[],"044334":[26,27],"0443659":[],"0444119":[],"044613":6,"04481443e":[],"04483457":[],"04495095":[],"04496448":[],"04516608":[],"04531375":[],"045314":[],"04537385":6,"04543942":6,"04554454":[],"045603":[],"0456042":[],"04566103":27,"04566964":6,"04573057":[],"04573105":[],"0457893":26,"0458":9,"0459682":[],"04615384615386":[],"04621521":[],"04637311":[],"04648335":5,"046491":[],"04649105":[],"0466":[],"04670006":[],"04673082":[],"046731":[],"04674926":26,"046785461905835435":[],"04678736":[],"04683565":5,"04691549":[],"04735806":[],"04771105":27,"047737":[],"04775904":[],"04784395":6,"04785727":[],"04787099":[],"047953":[],"04800573":26,"04818727730430286":[6,29],"04818727730430296":[],"048187277304303056":[],"04825655":5,"048257":5,"048547423739546604":[],"04860436":[],"048653428302840175":[],"048762":[],"048818":[],"048920":26,"04892055":6,"04909093":6,"04912436":6,"04926746":29,"04930820e":[],"049318":[],"049462":26,"0495569966278238":[],"0495569966278269":6,"0495569966278295":6,"0495569966278315":[],"04965397e":[],"049747":[],"04983109":20,"049858":[],"0499586":[],"049960":[],"04996008":[],"04it":[],"05":[1,4,6,9,13,21,27],"05004617":[],"05009826":6,"05023564":[],"05029894":27,"05069808":[],"05087958":[],"050910":[],"05100875":6,"051043":[],"05110883":[],"0511284278960114":11,"0512238":[],"051391":[],"051418":5,"051451":[],"05150928":[],"051649":[11,27],"051662":[],"0517473":5,"05177021":27,"05177092":[],"05177527":[],"05177695":[],"052206":[],"052227":[],"05227921801205679":[6,29],"05227921801205691":[],"05227921801205692":[],"052279218012057004":[],"052305":27,"052330":[],"05236797e":[],"052372":[],"0525063":[],"0525221":[],"05255759":[],"052581":[],"05263":9,"05268304":[],"0527437":[],"05279147850537411":[],"05288681":27,"05290417684691035":[],"05302":9,"053417":27,"05358851":[],"0536097":[],"05363779":[],"05364854":8,"05367466":[],"053678":[],"05383795":6,"053849":5,"053994":27,"05402365":[],"054182":[],"05421908":[],"054244842835462305":[],"054305":[],"054335":[],"054340":[],"05434571":[],"0543653729690416":23,"054411":[],"05447415":6,"054491":[],"05454172":[],"054582":[],"054585":[],"054617":27,"054650":[],"054655":27,"054774":[],"054785":[],"05480098":[],"05483267":[],"05491301":[],"05491655":[],"054963":[],"05505310046362":[],"05505310046363":2,"055137":[],"055153":[],"055302":[],"055320":[],"05533":9,"055337":[],"055344":[],"055369":[],"05554042":[],"055676":[26,27],"055684":[],"055734":27,"055859":[],"05587489":[],"055875":[],"055910":27,"055958":[],"055987":27,"056127":[],"056131":[],"05614483":5,"056165":[],"056169":[],"05622775":[],"05623":9,"056233":[],"056235":[],"05635863":[],"056468":[],"05648":9,"05651951":6,"056528":[],"05667":9,"056683":27,"05698425":[],"057088709963182":[],"057163681553428394":[],"05716368155342902":[6,29],"05716368155351588":[],"0572":[],"057300":27,"057393":27,"057418":27,"057457":27,"057463":5,"057469":[],"05748797":[],"05751737":27,"05756733":[],"057587":[],"057588":[],"05763394":[],"057648":[],"057686":[],"057722":27,"057835":27,"05785343":6,"057877":[],"05789007":6,"057899":[],"05796251":6,"058033":[],"058035":[],"058038":[],"058040":[],"058058":[],"05807125":6,"058088":[],"05833369":[],"058343":[],"05834332":[],"05835037":23,"058416":[],"05850532":[],"05851756":20,"058552":27,"05855607":[],"058585":[],"058596":[],"058645":27,"058663":[],"058715":[],"058732":[],"058740":[],"058816":[],"05881845":20,"05883":9,"05884":9,"058854":27,"058890":[],"05892262":[],"058948":[],"058952":27,"058955":[],"058957":[],"058974":[],"058996":27,"059013":[],"059031":27,"0590702":27,"05911793":[],"059118":[],"059180":[],"059245":5,"059272":[],"059278":[],"059294":[],"059378":[],"059437":[],"059453":[],"059601":[],"05971574":[],"059783":[],"059795":[],"059807":27,"059824":[],"059933":[],"059951":27,"059952":[],"05999":9,"059993":27,"06":[6,27],"060016":[],"060048":[],"0600658":27,"060080":[],"060083":27,"06008452":[],"060146":[],"060183":[],"06020587":6,"06021285":[],"060213":[],"060228":[],"060254":27,"060288":[],"060291":[],"060309":[],"06031681":[],"060349":26,"060369":[],"06038053":[],"060381":[],"060394":[],"060401":[],"060425":5,"06043581":6,"060441":[],"060502":[],"060507":[],"060534":[],"060550":[],"060603":[],"060649":[],"060655":[],"060711":[],"060716":27,"060756":27,"060781":[],"060822":11,"060841":[],"060863":[],"060872":27,"060875":[],"060924":[],"060963":[],"061025":[],"061028":[],"061037":[],"061048":[],"061054":[],"0610602":[],"061113":5,"061133":[],"061144":[],"061149":[],"061163":27,"061188":[],"061209":[],"06121384":[],"061272":[],"061297":[],"061308":[],"061321":[],"061359":27,"06136773":[],"061390":[],"061443":27,"061452":27,"061468":[],"06150169":[],"061502":[],"061505":[],"061509":[],"061529":11,"061538":[],"061604":[],"06160438":[],"061614":27,"061646":[],"061679":26,"061685":[],"06168748":[],"061701":[],"0617129312534124":23,"061716":[],"061725":[],"061745":[],"061765":[],"061821":[],"0618352118150005":9,"061898":[],"061917":[],"061923":[],"061966":[],"06200174":5,"062016":27,"062061":[],"062062":[],"062064":[],"06208238634231944":[],"062082386342319454":[6,29],"06208238634231953":[],"062089":[],"062097":[],"062136":[],"062142":[],"062195":[],"062197":5,"0622197":[],"062220":[],"062292565":4,"062294":[],"062295":11,"062298":[],"06229862e":[],"062305":[],"062333":[],"062337":27,"062370":[],"062391":[],"062409":[],"062411":[],"062424":[],"062435":[],"06243748":23,"062451":[],"062498":[],"062517":[],"062534":[],"062551":[],"062563":[],"062599":27,"0626202708457115":[],"062624":27,"062640":[],"062665":[],"062676":[],"062706":[],"062719":[],"062721":[],"06273487":[],"062734874360835":[],"062742":[],"062773":[],"062777":[],"06282378894371":5,"062827":[],"062834":[],"062874":27,"06287717":[],"062881":[],"062884":[],"062888":[],"062920":[],"06292011":[],"06293056":20,"062983":[],"063014":[],"063019":27,"063024":[],"063052":[],"063061":27,"063066":5,"063070":[],"063078":[],"063080":27,"063084":[],"063115":11,"063125":[],"063155":[],"063173":[],"063192":[],"06319374":[],"063241":[],"063250":[],"063260":27,"063287":[],"063343":[],"063353":[],"063364":[],"063371":[],"063375":[],"063401":[],"063443":26,"063493":[],"063513":[],"063524":[],"06353697":[],"063600":[],"063612":[],"063619":[],"063631":[],"063653":27,"063657":[],"063661":[],"063724":26,"063750":[],"063752":[],"063766":[],"063781":[],"063851":[],"063857":[],"063872":[],"063874":[],"063876":5,"06388888888888888":[],"063894":27,"063977":11,"063979":[],"064041":[],"064048":5,"064052":[],"064057":[],"064062":[],"064074":27,"064081":[],"064108":[],"064110":[],"064176":[],"064184":[],"064232":[],"064274":[],"064284":[],"06432223":[],"064329":[],"064384":[],"064388":[],"064390":[],"064431":[],"06444":9,"064442":[],"064459":[],"064461":[],"064469":[],"064483":[],"064501":27,"064526":[],"06453579006728315":[],"06453579006728322":[6,29],"064538":[],"064573":[],"064577":[],"064604":[],"064606":27,"064609":27,"064618":[],"064623":[],"064634":[],"064637":[],"064648":[],"064658":[],"064681":[],"064684":[],"064699":27,"064701":[],"064753":[],"064759":[],"064787":[],"064793":27,"064823":[],"064838":[],"064859":[],"06486061":[],"064880":[],"064896":[],"06491736":6,"064918":[],"064931":[],"064937":[],"064964":[],"06497046":[],"064974":[],"065020":[],"065026":26,"065030":[],"065032":[],"065077":27,"065095":[],"065101":5,"065106":5,"065110":[],"065118":[],"065165":[],"065168":[],"065175":[],"065183":[],"065228":[],"065237":[],"065249":[],"065252":[],"065268":5,"065282":[],"065314":[],"065348":[],"065349":[],"065350":[],"065378":27,"065389":[],"06538944":[],"065390":27,"065410":27,"06547790180152352":[6,29],"06547790180152353":[],"06547790180152355":[6,29],"06547790180152357":[],"06547790180152363":[],"065514":[],"065537":[],"065557":[],"065575":[],"065641":[],"065657":[],"065680":[],"065730":[],"065761":[],"065796":[],"065826":[],"065846":[],"065869":[],"065892":[],"065942":[],"065952":[],"065973":[],"065974":5,"065983":[],"066006":[],"06602663":[],"066051":[],"066056":[],"066064":[],"066066":27,"066093":[],"066110":[],"066189":[],"066194":[],"066209":[],"066220":[],"066223":[],"066228":[],"066241":[],"066270":[],"066284":[],"066309":[],"066317":[],"066340":[],"066348":[],"066356":[],"06637":9,"066388":[],"066449":[],"066474":27,"066500":27,"066519":[],"066528":[],"066553":[],"066556":[],"066558":[],"066569":[],"066580":[],"066590":[],"066609":[],"066612":27,"06662062":[],"066647":27,"066651":[],"066654":[],"066656":[],"066670":[],"066672":[],"066675":[],"0666807":2,"066700":5,"066714":[],"066728":[],"066729":[],"066741":[],"066742":5,"066753":[],"066787":27,"066789":[],"066808":[],"066828":[],"066837":[11,27],"066850":[],"066854":27,"066870":[],"066873":[],"066919":[],"066934":[],"06695337":[],"066954":[],"066969":[],"066999":27,"067011":[],"067079":[],"067089":[],"067110":[],"067141":[],"067162":[],"067183":5,"067191":[],"067213":[],"067224":[],"067233":[],"067236":[],"067240":[],"06724062":5,"067282":[],"067288":[],"067392":[],"067400":[],"067402":[],"067432":[],"067437":[],"067443":[],"067449":[],"067484":[],"067489":[],"067499":[],"067506":[],"067515":5,"067525":[],"067527":[],"067544":[],"067547":[],"067560":[],"067585":[],"067593":[],"067597":[],"067599":[],"067612":[],"067615":[],"067620":[],"067622":[],"067628":[],"067637":27,"067675":[],"067693":[],"067707":27,"067709":[],"067710":[],"067712":[],"067714":[],"067719":[],"067720":[],"067723":[],"067767":[],"067769":[],"067775":[],"067787":[],"067854":[],"067856":[],"067864":[],"06786925114666595":[],"067873":[],"067905":[],"067907":[],"067908":[],"067934":[],"067956":[],"067967":5,"067984":11,"067990":[],"068007":[],"068019":[],"068059":[],"068075":[],"068087":[],"068093":[],"068094":[],"068098":[],"068103":[],"068104":[],"068110":[],"068120":[],"068136":[],"068163":[],"068168":[],"068170":5,"068197":[],"068201":[],"068206":[],"068210":[],"068215":[],"068239":[],"06826023e":[],"068261":5,"068264":27,"068297":[],"068307":27,"068318":[],"068339":[],"068345":[],"068358":[],"068363":[],"068365":[],"068375":[],"068376":[],"068391":[],"068417":[],"068418":[],"068441":27,"06844519414009438":[],"06844519414009442":[],"06844519414009444":[6,29],"06844519414009445":[6,29],"068461":[],"068500":[],"068505":[],"068517":[],"068521":11,"068524":[],"068525":[],"068538":[],"068541":[],"068546":5,"068573":[],"06858699":[],"068593":[],"068608":[],"06861085":[],"068619":[],"068622":[],"068629":27,"068642":[],"068643":[],"068654":[],"068659":[],"068664":[],"068676":5,"068689":[],"068701":11,"068724":[],"068727":27,"068735":[],"06875458":26,"068776":[],"068782":[],"068792":[],"068799":[],"068806":[],"068809":[],"068831":[],"068832":[],"068841":[],"068852":[],"068867":[],"068876":[],"068906":27,"068912":5,"068920":[],"068937":[],"068953":[],"068965":[],"068991":[],"068992":[],"068999":[],"069010":[],"069013":[],"069015":[],"069028":27,"069036":[],"069042":[],"069044":[],"069065":[],"069072":11,"069080":[],"069085":[],"069091":[],"069125":[],"069129":5,"069181":[],"069187":[],"069190":[],"069202":[],"069203":[],"069231":[],"069233":11,"069242":[],"069265":[],"069268":[],"069275":[],"069276":[],"069281":[],"069283":[],"069285":[],"069291":[],"069312":11,"069315":5,"069320":[],"069343":[],"069355":[],"069386":[],"069408":[],"069409":[],"069421":[],"06942103":[],"069436":[],"069441":[],"069452":27,"069454":[],"069455":[],"069465":[],"069475":[],"069476":[],"069494":[],"069506":[],"069522":[],"069534":[],"0695405":[],"069578":[],"069583":5,"069584":[26,27],"069612":[],"069617":11,"069621":11,"069626":[],"069640":[],"069651":[],"069654":[],"069667":[],"069669":5,"069672":[],"069681":[],"069685":[],"06969872":[],"069738":[],"069748":[],"069754":[],"069782":[],"069796":[],"069801":[],"069816":[],"069823":5,"069838":[],"069845":[],"069847":[],"069855":[],"069873":5,"069877":[],"069884":[],"069886":11,"069888":[],"069903":[],"069907":[],"069911":11,"069912":[],"069977":[],"06it":6,"07":[6,27],"070018":[],"070028":[],"070034":[],"07003926":[],"070043":[26,27],"070056":[],"070071":[],"070074":[],"070092":[],"070096":[],"070105":[],"070119":11,"070127":[],"070139":11,"070141":[],"070146":27,"070150":[],"07016":9,"070161":5,"070163":[],"07017":9,"07019610e":[],"070213":27,"070216":[],"070222":[],"070241":[],"070245":[],"070276":[],"070288":[],"070302":11,"070305":5,"070333":[],"070339":[],"070343":[],"070364":[],"070370":[],"070379":[],"070381":11,"07039":9,"070419":[],"070428":[],"070429":[],"070443":[],"070450":[],"070453":[],"070457":27,"070466":[],"070486":5,"070491":[],"070492":[],"070494":[],"070518":[],"070562":[],"070576":[],"070577":5,"070582":27,"070585":[],"070601":[],"070612":[],"070618":[],"07062318":6,"070625":5,"070630":[],"070635":[],"070671":[],"070700":[],"070737":27,"070769":11,"070775":[],"070794":[],"070797":5,"070810":[],"070835":[],"070860":[],"070871":[],"07087395e":[],"070874":[],"070887":[],"070889":[11,27],"070911":[],"070936":[],"07093644":[],"070945":[],"070950":[],"070959":[],"070961":[],"070972":[],"070975":11,"070986":27,"07099747918547346":[],"071003":[],"071006":[],"071009":[],"071022":[],"071044":[],"071051":[],"071071":[],"071084":[],"071086":11,"071105":[],"071106":[],"071112":[],"071115":[],"071120":[],"071122":[],"071136":[],"07115":9,"071164":[],"071185":[],"071239":[],"071243":[],"071284":[],"07128434":[],"07129539":[],"0712953943627344":[],"0713":[0,26],"071314":5,"071321":11,"071326":[],"071329":[],"071387":[],"071388":[],"071392":[],"071394":[],"071420":5,"071421":[],"071437":[],"071438":[],"07143876":[],"071441":[],"071445":[],"07145103":11,"071453":11,"071455":5,"071460":[],"071467":[],"071476":[],"071480":[],"0714956":[],"071501":5,"071505":[],"071510":[],"071515":[],"071523":[],"071529":[],"071534":5,"071547":[],"071552":[],"071567":[],"071569":[],"071585":[],"071600":[],"07160048164232467":[],"07160048164232538":[6,29],"0716004816423254":[6,29],"07160048164248561":[],"071604":[],"071614":[],"071619":11,"071632":[],"071641":[],"071659":5,"071666":[],"071698":[],"071705":[],"071741":[],"071751":[],"071767":[],"071804":[],"071836":[],"071852":[],"071853":11,"071897":[],"071901":[],"071906":[],"071908":[],"071920625289855":[],"071921":[],"071922":[],"071935":[],"071942":[],"071946":[],"071949":[],"07195822":[],"071982":11,"071991":[],"072000":27,"072008":[],"072011":[],"072016":11,"07201957":[],"072021":[],"072041":[],"072084":[],"072091":[],"072111":27,"072122":5,"07212695":[],"072127":[],"072132":27,"072145":[],"072147":[],"072150":[],"072176":11,"072183":5,"072184":[],"072186":[],"072198":[],"072216":[],"072222":[],"072240":[],"072242":[],"072254":[],"072290":[],"072334":11,"072342":[],"072344":11,"072348":[],"072352":[],"072361":[],"072364":[],"072365":5,"072370":[],"072384":11,"072398":5,"072435":[],"072442":[],"072452":[],"072468":11,"072471":[],"072474":11,"072476":27,"072483":[],"072492":11,"072493":11,"072515":5,"072521":[],"072522":5,"072523":[],"072529":[],"072530":[],"072554":[],"072558":[],"072569":[],"072575":[],"072597":[],"072598":[],"072611":[],"072617":[],"072618":[],"072620":[],"072621":27,"072638":5,"07264063e":[],"072647":[],"072651":[],"072652":[],"072682":11,"072705":[],"072750":[],"072751":[],"072783":[],"072798":[],"072813":[],"072824":[],"072828":[],"072839":[],"07285":3,"0728785":[],"072898":[],"07291729":[],"072918":[],"072928":[],"072955":[],"072958":[],"072959":11,"072970":[],"072974":[],"07297557":[],"072994":[],"073013":[],"073016":[],"073038":[],"073044":[],"073045":11,"073052":[],"073062":[],"073080":[],"073105":[],"073120":[],"073127":5,"073134":[],"073136":[],"073159":[],"073163":[],"073183":[],"07319349":[],"073206":[],"073225":[],"073230":[],"073236":5,"073238":[],"07325301":[],"073280":[],"073298":[],"073299":[],"073300":5,"073303":[],"073320":[],"073325":[],"073337":11,"07333967":26,"073344":[],"073362":[],"073363":[],"073368":[],"073369":[],"07337358":[],"073374":[],"073380":[],"073388":[],"073391":[],"07339251":[],"073398":[],"073400":[],"073423":[],"073433":[],"073436":[],"073449":[],"07345504":[],"073465":[],"073477":[],"073489":[],"073497":5,"073506":[],"073520":[],"073531":[],"073548":[],"073567":[],"073575":[],"073583":[],"073585":5,"073597":[],"073598":[11,27],"073601":[],"073624":[],"073629":[],"073630":[],"073634":[],"073644":27,"073647":[],"073657":[],"073668":[],"073684":[],"073695":[],"073699":[],"073716":[],"073717":11,"073719":[],"073750":[],"073761":[],"073764":[],"073768":[],"073777":[],"073810":[],"073811":[],"073816":[],"073827":[],"073831":[],"073904":[],"073907":[],"073925":[],"073938":[],"073949":[],"073955":11,"073959":[],"073985":[],"0739975":[],"074004":[],"074006":[],"074007":5,"074019":[],"074034":[],"074046":[],"074050":[],"074061":[],"074067":[11,27],"074084":27,"074102":11,"074111":[],"07413172":[],"074144":[],"074149":[],"074161":11,"074180":[],"074183":[],"074187":[],"074193":11,"074203":5,"074210":27,"07421084":5,"074230":5,"074237":[],"074258":[],"074276":5,"074293":5,"07430684":[],"074310":5,"074323":27,"074336":[],"074346":[],"074375":[],"074401":[],"074418":[],"074420":[],"074453":[],"074480":[],"074488":[],"074495":[],"074513":[],"074530":[],"074552":[],"07456491":5,"074569":[],"074573":[],"074592":[],"074613":[],"074653":[],"074666":[],"074671":5,"074684":[],"074692":11,"074693":[],"074708":[],"074715":[],"074729":[],"074732":[],"074743":[],"074752":11,"074760":[],"074763":[],"074773":[],"074818":[],"074819":[],"074833":[],"074838":[],"074839":[],"074845":[],"074868":[],"074881":[],"074905":[],"07490892":6,"074922":[],"074945":[],"074951":[],"074969":[],"074971":[],"074980":[],"074983":5,"074984":[],"074986":[],"074992":[],"075012":[],"075014":[],"075017":[],"075020":[],"075028":[],"075035":5,"075043":[],"075045":11,"075047":[],"075050":[],"075058":[],"075061":[],"075064":11,"075079":[],"075102":[],"075115":5,"075121":[],"075136":[],"075138":[],"075145":[],"075149":[],"075181":[],"075188":[],"075192":[],"075195":[],"075201":[],"075222":[],"075224":[],"075225":[],"075227":[],"075228":[],"075241":[],"075262":[],"075263":11,"075277":[],"075302":[],"07532297":[],"075330":11,"075335":[],"075356":[],"07535606":[],"075389":[],"075400":[],"075418":11,"075422":5,"075433":[],"075455":11,"075498":[],"075529":11,"075569":5,"07557024":[],"075573":[],"075586":[],"075587":5,"075590":11,"075594":[],"075612":[],"075618":[],"075620":11,"075629":[],"07565581":[],"075727":[],"075728":[],"075760":[],"075770":[],"075779":[],"075782":[],"075797":[],"075816":27,"075820":[],"075829":[],"075840":[],"075863":[],"075887":5,"075889":27,"075899":[],"075911":11,"075952":[],"075958":5,"075959":[],"075982":[],"075987":[],"075996":[],"076007":5,"076022":[],"07602623":[],"076055":[],"076066":[],"076079":[],"07608141":[],"076096":[],"076097":[],"076112":5,"076121":[],"076130":[],"076150":27,"076173":[],"076204":[],"076216":[],"076248":[],"076259":[],"076264":[],"076270":[],"076282":[],"076288":11,"076306":[],"076313":5,"076315":[],"076324":[],"076329":[],"076338":[],"076349":[],"076371":[],"076389":[],"076403":[],"076408":11,"07641937":29,"076454":[],"076491":5,"0764924":6,"076518":[],"076519":5,"076525":[],"076532":[],"07656896":[],"076570":[],"076573":[],"07657669":[],"076588":11,"076589":[],"076604":[],"076612":5,"076619":[],"076628":[],"076632":[],"076637":[],"076665":[],"076697":[],"076726":[],"076729":[],"076760":[],"076775":[],"07678":9,"076780":[],"076783":[],"076795":[],"076804":[],"076818":[],"076819":[],"076821":[],"076843":[],"076862":[],"076866":[],"076867":[],"07687896":26,"076886":[],"076895":[],"076905":[],"07692307692307693":9,"076927":[],"076935":[],"076938":[11,27],"076942":[],"076955":[],"076958":[],"076990":[],"077003":[],"077004":[],"077009":[],"077012":11,"077013":[],"077015":[],"077022":[],"077033":[],"077050":5,"077062":[],"0770789":[],"077079":[],"077083":[],"077091":[],"077107":[],"0771079":[],"077108":[],"077131":[],"077140":[],"077144":[],"077158":[],"077164":[],"077168":27,"077182":11,"077200":[],"077203":[],"077210":[],"077211":[],"077219":[],"077228":[],"077280":5,"077291":[],"077338":[],"077349":[],"07735703":[],"077359":[],"077372":[],"077375":[],"077387":5,"077397":[],"077402":[],"077411":5,"077419":[],"077425":[],"077428":[],"077452":[],"077460":27,"077467":[],"077491":11,"077499":[],"077511":[],"077521":11,"07752967":26,"077536":5,"077604":[],"077606":[],"077607":[],"077611":11,"077620":[],"077644":[],"077663":[],"077666":5,"077676":11,"077688":[],"077690":11,"077706":[],"077711":5,"077713":[],"077724":11,"07777777777777778":1,"077785":[],"077793":[],"077822":[],"077825":5,"07783152589039466":5,"077846":[],"077849":11,"07789772e":[],"077910":11,"077921":[],"077930":[],"077948":[],"077949":[],"077954":[],"077970":5,"077978":[],"077986":[],"078033":[],"078040":[],"078042":[],"078082":[],"078101":[],"078110":27,"078133":[],"078143":[],"078157":[],"078158":[],"078165":[],"078167":[],"0782":[],"07820":9,"078211":[],"078253":[],"078254":[],"07826171e":[],"078265":[],"078269":[],"078293":[],"07829878":[],"078299":[],"078300":[],"078314":[],"078323":[],"078326":[],"078333":11,"078336":27,"078426":[],"078432":[],"078436":[],"07844310540708652":[],"078484":[],"078508":[],"078521":[],"078540":27,"078542":[],"078548":[],"078555":11,"07858099596662704":[],"078582":[],"078587":[],"078609":[],"07864":9,"07864140367329474":11,"078646":[],"078651":[],"078652":[],"078667":[],"078672":[],"078697":[],"078704":[],"078707":[],"07871":9,"078710":[],"078715":[],"078719":[],"078725":5,"078749":[],"078759":[],"078768":[],"078771":5,"078776":[],"07878641":[],"078795":[],"078812":[],"078845":[],"078860":[],"078880":[],"078899":[],"078901":11,"078908":[],"078911":[],"078926":[],"078942":[],"078962":[],"078974":27,"07897647347778382":[],"07898165660100093":[],"078986":27,"078987":11,"078992":[],"078995":5,"078999":[],"079001":[],"079021":[],"07903849":[],"079043":[],"079056":[],"079110":[],"079142":[],"079152":[],"079157":[],"079158":[],"079179":[],"079214":[],"079218":[],"079224":[],"079225":[],"079242":[],"079260":[],"079264":[],"079268":[],"079273":[],"079278":[],"079281":[],"079298":[],"079306":[],"079319":11,"079333":[],"079347":[],"079368":5,"079379":[],"079383":[],"079389":[],"079405":[],"079406":[],"079424":[],"07943621":[],"079437":[],"079438":[],"07944154":[20,26],"079443":[],"079447":[],"079448":[],"079476":[],"079483":[],"079486":[],"079489":[],"079493":[],"079495":[],"079499":[],"0795449":[],"079546":11,"079553":[],"079558":[],"079580":[],"079609":[],"079615":5,"079618":[],"079622":[],"079631":[],"079649":11,"079671":5,"079676":[],"079680":[],"07968918676726029":[],"0796891867672603":[6,29],"079704":[],"079719":[],"079729":5,"079741":[],"079745":[],"07978179":[],"079786":5,"079814":5,"079819":[],"079823":[],"079827":[],"079839":[],"079847":[],"079856":[],"079858":11,"079860":11,"079882":[],"079893":[],"07989327":[],"079901":[],"079944":[],"079946":27,"079948":[],"07995375":[],"079958":11,"079991":[],"07999999999998":[],"07e":[],"08":[9,23,27],"080026":[],"080028":11,"080048":[],"080056":[],"080060":[],"080061":[],"080070":[],"080098":[],"080106":[],"080120":[],"080137":[],"080168":5,"080200":[],"080249":[],"080256":[],"080288":[],"080295":[],"080297":[],"080305":[],"080312":[],"080319":[],"080332":[],"080334":[],"080343":[],"080347":[],"080370":[],"080374":[],"080384":11,"080395":[],"080406":[],"080407":[],"080410":[],"08041015":[],"080434":[],"08043851":5,"080455":[],"080479":[],"080502":[],"080517":[],"080518":[],"080548":[],"080555":[],"080571":[],"080582":[],"080588":[],"080593":[],"080616":27,"080630":[],"080635":[],"080641":[],"080642":[],"080647":[],"08066746e":[],"080699":[],"080706":[],"080738":[],"080764":27,"080780":[],"080801":[],"080821":[],"080839":[],"08084392":[],"080847":[],"08085812":[],"08086066":[],"080863":11,"080867":5,"080914":[],"080916":[],"080922":[],"080965":11,"080973":[],"080979":5,"080984":[],"080998":[],"081007":[],"081022":[],"081046":[],"081072":[],"081100":[],"081140":[],"081144":[],"081163":[],"081194":[],"081198":[],"081211":[],"081216":[],"081223":[],"081233":[],"081253":[],"081258":[],"081262":[],"081276":27,"08131003":6,"081311":[],"081315":5,"081321":[],"081334":[],"081343":[],"081357":[],"081388":[],"081395":[],"081396":[],"081431":[],"081468":[],"081471":[],"0814721":[],"081481":[],"081483":[],"081503":[],"081520":[],"081533":[],"081540":[],"081547":[],"081559":[],"08156108":6,"081576":[],"081613":[],"081620":[],"081637":[],"08165104":27,"081655":27,"081699":11,"081707":[],"081728":5,"081773":[],"0818000034712947":[],"081827":[],"081832":[],"081837":[],"081848":[],"08188077":[],"081881":[],"081932":[],"081953":[],"081970":11,"081977":5,"081988":[],"081989":[],"081990":[],"082014":[],"082027":[],"082040":[],"082089":[],"082105":[],"082122":[],"082139":[],"082140":[],"082145":[],"082158":[],"082166":[],"082174":[],"082183":[],"082189":[],"082204":[],"082232":11,"082267":[],"082269":[],"082282":[],"082285":[],"082286":[],"082288":[],"082295":[],"082315":[],"0823185":[],"082322":[],"082339":11,"0823594":26,"08237141":27,"082385":11,"082424":[],"082426":[],"082431":[],"082449":11,"082451":[],"082487":5,"082501":[],"082503":[],"082509":11,"08251519":6,"082525":[],"082527":[],"082536":[],"082553":[],"082569":[],"0825971":[],"082604":[],"082620":[],"082642":[],"082653":27,"082683":[],"08271198519070039":[],"082716":[],"082756":[],"082779":[],"082786":[],"082791":[],"08281969e":[],"082839":[],"082846":[],"082852":[],"082874":[],"082875":26,"082877":[],"082896":[],"082900":[],"082921":[],"082939":[],"082990":[],"082991":5,"08299273e":6,"083000":[],"083015":[],"083021":[],"083026":[],"083051":[],"083053":[],"083087":[],"083102":[],"083111":[],"083151":[],"083152":[],"083159":[],"08318298e":1,"083217":[],"083220":[],"08322642264994606":[],"083227":[],"083242":[],"083251":[],"083272":[],"083276":[],"08328216846752691":[],"083300":[],"083319":11,"083320":[],"083321":[],"08333333333333333":[1,9],"083348":[],"083361":[],"08336233266":4,"083380":[],"083399":[],"083404":[],"083417":[],"08343519179767796":[],"083441":[],"083446":[],"083485":[],"083495":[],"083506":[],"083517":5,"083527":[26,27],"08352721390288316":26,"083555":[],"08356774001062162":[],"083573":[],"083579":[],"083600":[],"083627":[],"083629":[],"08363087":[],"083648":[],"083681":[],"083685":[],"083691":[],"083692":[],"083694":[],"08370888":[],"08373167e":[],"08374032367704139":[],"08376632":[6,27],"083766322923899":[6,27],"0837663229239016":[],"0837663229239025":[],"0837663229239043":[6,27],"083772":[],"083779":[],"083799":[],"083829":[],"083832":[],"083834":5,"083849":[],"083859":[],"083899":[],"083913":[],"083935":[],"083977":[],"084000":[],"084006":[],"084008":[],"084017":[],"084018":[],"084019":[],"084024":[],"084028":[],"084032":[],"084076":27,"084079":11,"084086":[],"084092":[],"084096":[],"084101":[],"084107":[],"084124":[],"084147":[],"084151":[],"084157":[],"084184":[],"084187":[],"084199":5,"084223":[],"084224":[],"084226":[],"084230":[],"08426840630693411":[],"08426840630693412":[6,29],"08426840630693413":[6,29],"084269":[],"084277":[],"084278":[],"08428156":[],"084282":5,"084286":[],"084364":[],"084391":[],"084400":[],"084408":[],"084414":[],"084422":[],"084426":[],"084434":[],"084438":[],"084444":[],"084476":[],"084477":[],"084484":[],"08448544e":[],"084487":11,"084523":[],"084536":[],"084541":[],"084549":[],"08455":9,"084570":[],"084604":[],"084624":[],"084629":[],"084636":11,"084645":[],"08464758160254343":[],"084657":[],"084663":[],"084666":11,"084670":[],"084682":[],"084683":[],"084685":[],"084702":[],"084728":[],"08474":9,"084777":[],"084809":27,"084846":[],"084862":[],"084904":[],"084906":[],"084909":[],"084912":[],"084920":[],"084927":[],"084936":[],"084965":[],"084977":[],"084979":[],"085010":[],"085018":[],"08502083":[],"08502185":23,"085027":[],"085044":[],"085104":11,"085105":[],"085108":[],"085113":[],"085154":11,"085163":[],"085165":[],"085167":[],"085184":[],"085185":[],"085210":[],"085224":[],"085242":[],"085249":[],"085252":[],"08526352":27,"085264":[],"085272":[],"085290":11,"08529655":23,"085298":11,"085345":[],"085361":[],"085368":[],"085384":[],"085405":[],"085410":[],"085416":[],"085447":5,"085456":[],"085484":[],"08549564e":[],"08549632935144091":[],"08551306":6,"085571":[],"085603":[],"085655":[],"085716":[],"085764":[],"08576932":6,"085776":[],"085778":[],"085876":[],"085879":[],"08593216":6,"085936":[],"086021":[],"086074":[],"086076":[],"086109":[],"08611111111111111":1,"086112":[],"08612280083325631":[],"086154":[],"086174":[],"086178":[],"086203":[],"086249":[],"086250":[],"086257":[],"086262":[],"086290":11,"086303":[],"08630331":[],"086322":[],"086358":[],"086409":[],"086420":[],"086440":[],"086460":[],"086518":[],"086524":[],"086567":[],"08657894597048958":[],"086608":[],"086610":[],"086624":[],"086647":[],"086665":[],"086679":[],"086690":11,"086705":[],"08673755293381497":[],"086802":[],"086807":[],"086809":[],"086841":[],"08684675":[],"086872":[],"086877":[],"086888":[],"086890":[],"08690":9,"086904":[],"086925":[],"086942":5,"086951":[],"086956":[],"086994":[],"086997":[],"086999":[],"087081":[],"087208":[],"087224":[],"087235":[],"087315":[],"08732305":[],"08734015":[],"087372":[],"087377":[],"087386":[],"087387":[],"08741274":[],"087419":[],"08743287e":[],"087470":[],"087501":[],"087520":[],"087573":[],"08758":9,"08759":[],"087615":[],"087689":[],"087746":[],"087833":[],"087834":[],"087848":[],"087860":[],"087878":[],"087887":26,"087891":[],"087894":[],"087910":[],"08791994":20,"088026":11,"088029":[],"088037":[],"088074":[],"088104":[],"088119":[],"088164":[],"08816688":[],"088176":[],"0881981":5,"088201":[],"088212":[],"08821204":[],"088238":[],"088261":[],"088271":[],"088275":[],"088317":[],"088348":[],"088361":[],"088425":[],"088460":[],"088476":[],"088488":[],"088502":[],"088506":[],"088519":[],"088526":[],"088560":[],"088606":[],"088611":[],"088631":[],"08871404":5,"08873443359350565":[],"08874264":[],"088754":[],"088758":[],"088760":[],"088765":[],"088778":[],"088809":[],"08881497884574564":27,"088816":[],"088825":[],"088853":[],"08887708":[],"08888888888888889":1,"08889302":[],"08891672":5,"088917":5,"088926":[],"088946":[],"088974":[],"088982":[],"088995":11,"08902":9,"089040":[],"089041":[],"089059":[],"089063":[],"089082":[],"08910476e":[],"08917679":[],"089177":[],"08918584":[],"089206":[],"089212":[],"08923158":[],"089233":[],"089240":[],"08924325e":[],"089246":[],"089306":[],"089310":[],"089329":[],"089355":[],"0893679":[],"089369":[],"089374":[],"089425":[],"089441":[],"08945041":[],"089501":[],"089505":[],"089524":[],"0895387":27,"089539":27,"089551":[],"089614":[],"089641":[],"08966600e":[],"089683":[],"0896981":[],"089744":[],"089758":[],"089775":[],"089857":[],"089924":[],"08996":9,"089982":[],"08it":6,"09":[1,27],"090014704675496":[],"090028":[],"090039":[],"090128":[],"090174":[],"090229":[],"09023660662586945":[],"090241":[],"090268":[],"090270":[],"09030825":[],"090337":[],"090349":[],"09034902":[],"0903549":[],"090356":[],"090358":[],"090424":11,"090451":[],"090503":[],"090564":[],"090620":[],"090630":[],"090649":[],"090694":[],"090730":[],"090755":[],"090768":[],"0907788":[],"0907831":[],"090828":[],"090830":[],"09083636328656121":[],"090849":[],"09085624":[],"090888":[],"090923":[],"090929":[],"091031":[],"091044":[],"091051":[],"091060":[],"091113":[],"09117221":[],"091209":[],"091224":[],"091236":[],"091266":[],"091311":[],"091315":[],"091340":[],"091349":[],"091363":[],"091369":[],"091376":[],"091414":[],"091416":[],"091418":[],"091426":[],"091440":[],"091445":[],"091477":[],"09149881":[],"091579":[],"091620":[],"091630":[],"091647":[],"091659":[],"09166666666666666":1,"091696":[],"0917":9,"09170751":[],"09172408":[],"09172409":6,"091772":[],"091891":[],"091896":[],"09189927":[],"091996":[],"092007":[],"092066":[],"092126":[],"09216046":[],"09218919":20,"092249":[],"092290":[],"0923":[],"092409":[],"092478":[],"09251":9,"09252364":[],"092524":[],"092621":[],"092724":[],"092769":[],"092879":[],"09289204":[],"09297039":[],"092998":[],"093078":[],"09309965420717024":[],"09318696260700278":[],"093247":[],"0934597075922044":[],"093497":[],"093556":[],"093593":[],"093624":[],"0938":[],"093844":[],"093930":[],"093979":[],"09404851e":[],"094082198961999e":6,"0940821989624095e":[],"0940821989643363e":6,"094082198966615e":[],"0940821989673748e":[],"094139":[],"094227":[],"094252":[],"094362":[],"09440475":[],"094404754965417":[],"09444444444444444":1,"094472507965532":[],"094477":[],"094610":[],"094619":[],"094657":[],"094722":[],"09480900e":[],"094812":[],"094848":[],"094867":[],"094925":[],"09497154":26,"095171":[],"095184":[],"095313":[],"095315":[],"09533574":[],"095510":[],"095522":[],"095617":[],"095702":[],"0957156":[],"09575221":[],"095871":[],"095902":[],"095935":[],"09609807":5,"096173":[],"09631112":[],"09634375e":[],"096472":[],"096516":[],"09652672":[],"096623":[],"09662862":[],"09672929714683368":[],"09677394":[],"096833":[],"096845":[],"096872":[],"096893":[],"096894":[],"097":[],"097111":[],"097360":[],"09739330e":[],"097396":[],"097435":[],"09744":9,"09744272":[],"09759873":[],"097602":[],"097634":[],"097710":[],"09772449":[],"09780":9,"09781386":[],"09789363e":[],"09791":9,"098069":[],"098135":[],"09846636e":[],"09849763":[],"098498":[],"09856879":[],"09861229":[20,26],"09862064":20,"09873132":[],"098802859381565":[],"099":[],"09903804":8,"0991919894927399":[],"09919198949274803":[6,29],"099191989493334":[],"09951287404314545":1,"09963157":13,"099648":[],"09964817":[],"099777":[],"09989861":[],"0n":[0,26],"0s":[],"0x11da42d90":[],"0x11db14850":[],"0x11e031e50":[],"0x11e09b370":[],"0x11e77b070":13,"0x11ecf9850":13,"0x128ee8850":[],"0x12946f2e0":[],"0x1336a6040":[],"0x133c2e1c0":[],"0x138098070":[],"0x1387541c0":[],"0x13b240850":[],"0x13b8c72e0":[],"0x7fad10f9a280":[],"0x7fad20f69be0":[],"0x7fd098df3280":[],"0x7fd0a9063be0":[],"1":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,22,23,24,25,28,29],"10":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,22,23,24,26,27,28,29],"100":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,20,23,24,26,27,28,29],"1000":[0,1,2,4,5,8,11,13,14,19,23,26,27],"10000":[2,5,6,10,11,13,23,28,29],"100000":8,"10001":10,"1001":[9,23],"1002":23,"1003":23,"10030":9,"100303":[],"100358":[],"10044225464078282":[],"1005":23,"10056192":20,"100670":[],"1007":28,"10077114273548984":[6,29],"1009":23,"10097956":[],"1011":23,"101292":6,"1013":23,"1013904243":23,"10141413e":6,"10148978":20,"1015":23,"10188442":[],"102":27,"10207647":[],"1022233262115424":[],"10222333":[],"10222868":[],"1023":23,"10230":9,"1023111":[],"10235180e":[],"1023858":[],"1024":3,"102401":[],"102449":[],"10254846":20,"1026":23,"1027":23,"1028":[],"103":1,"1030":23,"10302062":[],"10308387":[],"103257":[],"10326551":20,"10340":9,"1037":23,"10378326e":1,"1038":23,"103822":[],"10391807":6,"10398646080125035":[],"10398646080125036":[6,29],"10398646080125037":[6,29],"1040":23,"10405456":11,"10427270e":[],"10430":9,"104411":[26,27],"10441496e":[],"10455924":[],"1047":23,"10486183":[],"104862":[],"10505137":[],"105161":[],"105169":[],"10516924":[],"10520":9,"10555555555555556":1,"1056":[],"10572":29,"10589577":5,"106":[],"106095":[11,27],"10620135":[],"10635754":26,"106431":26,"10659682":[],"10686259":[],"106971":[],"107":[],"10712245e":[],"1078":29,"10790125813226321":27,"108":[6,9],"108359":[],"10835935":[],"10880921":[],"109":[],"10913":6,"10931453":6,"10950766":[],"10951119":26,"10960":9,"1096767776832326":[],"10967678":[],"1098906":[],"10it":[],"10th":9,"10x":[0,26],"11":[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,20,21,22,23,24,25,26,27,29],"110":[],"1100":23,"1101":23,"11022302e":[],"11029323":[],"11033412":26,"11046771":[],"111":[1,7,12],"11100":9,"11167579":[],"11202337":29,"1122558214":[],"112319":[],"11231934":[],"112383":[11,27],"1124":9,"1125":[],"11276576e":[],"11283168":[],"11297834":[],"1133":[],"11341291":26,"11352543":[],"1136":[],"11362930e":[],"11366595":[],"11388888888888889":1,"1139":[],"11390":9,"114":[3,4,14],"11402309":[],"114437":[],"114550":29,"11462415":5,"11470382":20,"11481199":[],"11482289e":6,"11499517":[],"115":[3,4,14],"11507992e":1,"11520281":[],"11547777218875695":[],"11547777218875696":[],"11547777218876518":[6,29],"11547777218940905":[],"11547777218940906":[],"11551515":20,"115822":6,"11582635":23,"11590":9,"11598862":[],"11598862273198":[],"116":[3,4,14],"11604844":[],"116048442683864":[],"11660":9,"11666666666666667":1,"11674401539815678":[],"117":[3,4,8,14],"117430":26,"11744554e":6,"11780":9,"11790868":[],"11793498e":[],"117986":[],"118":2,"1180677":[],"118068":[],"1182":29,"1183":[],"118318":[26,27],"1184":[],"11840":9,"1185":[],"11850274":[],"1186":[],"11868087":[],"118681":[],"11890":9,"119":[2,3,4,14],"11909685e":[],"119625":[],"11962537":[],"1199":[],"119936":2,"119999999999976":[],"11it":[],"12":[0,1,2,3,4,5,6,8,9,11,12,13,20,23,25,26,27,29],"120":[2,3,4,14],"1203":9,"1203284":8,"12044974":27,"120450":27,"1206":8,"12069773":[],"12086283":[],"121":[8,9,10],"12107137e":[],"12152272":[],"1215pm":[24,26],"12160173":[],"12167821":[],"12178678":[],"121787":[],"12182967":6,"12196674":[],"122":[2,8,9,10],"12211324e":[],"12222222222222222":1,"122282":26,"122439":[],"1224392":[],"12288563":[],"122886":[],"12289498e":[],"123":2,"12302855":[],"12318726e":6,"12330033":27,"12330280229368":[],"12333649":6,"12345260617257443":[],"123459876":[],"123711":6,"12380":9,"12387751":[],"12394241e":[],"124":[0,26],"12400":9,"12427889":[],"12434807":[],"12446443":20,"1245596497124075":[],"12455965":[],"125":[],"1250":[],"125000":[],"12539459":[],"12552073e":6,"12591227":27,"126":9,"1261":9,"12611267":20,"12618549":5,"12630042":[],"1265":9,"127":4,"1270":[],"127043":[],"1271":6,"1277":6,"127773":[],"12777777777777777":1,"127812":26,"12788968":[],"12790":9,"128":[3,4,13],"12803365e":[],"1280756":26,"128664":6,"12871842":27,"12885048":[],"12890007e":[],"12898627064868978":[],"129":2,"129099":[],"12909917":[],"12933164":[],"129332":[],"12934015":[],"12937662":[],"12945452":[],"1297":9,"1297300314822336":[],"1298":9,"129963":29,"12adb44b1c20":[],"12it":[],"12m":[],"12pm":[24,26],"13":[0,2,5,6,9,11,12,13,20,23,26,27,29],"130":9,"13003291":6,"13006658e":[],"13019959":[],"130200":[],"13030182":[],"13055555555555556":1,"130694":27,"13069442":27,"13083543199018746":[],"131":9,"13117061":[],"13120126":20,"13131825":[],"1314":[],"132":9,"13208672e":[],"13220608e":6,"13228567":[],"13229545716505003":[],"132360":[],"1323603":[],"1326":9,"13260331":[],"13261136":[],"1326197715":26,"13280":9,"13281267":[],"13288373":[],"133":7,"13314597157635316":26,"13320535":[],"13333333333333333":[],"13375152e":[],"13392097":[],"13393141":[],"1345":29,"1346":29,"135":9,"13519106":[],"13535942":6,"135365":[],"13536532":[],"136":[],"1360":[],"13609760e":[],"1361":[],"13618516e":[],"1362":[],"1363":[],"1364":[],"13646574":5,"13655438":[],"13661243e":6,"13679863":6,"13687085e":[],"1371":6,"13734823":[],"13740":9,"137400784702911":27,"137652":[11,27],"1378":[],"1382":[],"1383":[],"1384":[],"13847924":[],"1385":[],"1386":[],"13865173":5,"13876586436927824":[],"138775":[11,27],"1388888888888889":1,"13890":9,"1392559581788775e":[],"1392559584983597e":[],"1392559585048734e":6,"1392559591206444e":6,"13925595925919e":[],"13it":[],"14":[0,2,3,4,5,6,8,9,10,11,12,13,14,20,23,25,26,27],"140":[2,9],"14010994":26,"14021063":6,"1404":[],"14042769":[],"140428":[],"14045566":[],"140525":[],"14052537":[],"14061742":[],"14072037":[],"141":2,"14100":9,"1416398":6,"14174062":[],"14174745":6,"1418":9,"142":9,"14231548":[],"14248744":27,"14250":9,"14252712":[],"14266332e":[],"14277718e":27,"142857":[],"14298603":[],"143":[2,3,4,7,14],"1435666":[],"14360598":[],"1437":1,"144":[3,4,14],"14400":9,"1440501043841336":1,"14432948":[],"14440":9,"14451625":[],"1446729567":4,"14484695":[],"14491591":20,"144993":26,"145":[2,3,4,14],"14561980e":[],"1457774":[],"146":2,"14617985":[],"14623399":26,"14629156":29,"14648477":[],"146489":5,"14648903":5,"147":2,"14710":9,"1472032":[],"14722222222222223":1,"14724135e":[],"147400":[11,27],"14741468":[],"147420":[11,27],"14783702":[],"1479":9,"147934":6,"14793475":26,"148":2,"14812206":6,"14818":[],"14822021080294":[],"1484256":[],"148463":[],"14846308":[],"148564":[],"14859":6,"14868784":5,"148688":5,"148734":[],"148768":[],"148771":6,"148854":[],"148895":29,"149":[],"14900336":[],"149213":[],"14932651":[],"149452":[],"149655":[],"149864":[],"14988578":[],"149886":[],"14995486":[],"149955":[],"149991":[],"14g":[6,29],"14it":[],"15":[0,2,3,4,6,7,8,9,12,13,14,15,16,17,18,23,26,29],"150":[4,8,9],"15005476":5,"150184":[],"15060311":[],"15084741":[],"15089627":[],"150920":[],"15092012":[],"1509778":[],"15098090e":6,"151":[],"15130074e":6,"151348":[],"15172236":13,"151986":[],"1519922":23,"152":9,"15200":9,"15254784":20,"15258907":[],"152696":[],"15269628":[],"15273101":23,"1527777777777778":1,"153036":[26,27],"153106":[],"15313054":29,"15320771e":[],"15321987":[],"1533795":[],"153760":[],"15384615384615385":9,"15384615384616":[],"154":[],"15410688":[],"154107":[],"15442554":[],"15454301":[],"15456415e":[],"154720":[],"15483121":[],"15484848":26,"15485374225770068":[],"154911":[],"1549258803307933":26,"15499979":5,"155":9,"155000":5,"15503141":[],"15508406":[],"155491":[],"15550086":[],"155687":[],"155883":[],"156":[],"1562":[],"15626984":[],"156270":[],"15629539":[],"15680777":[],"156956":5,"15696119":26,"157":[],"15714797":[],"15717291":[],"15746031e":[],"1575":9,"15752533e":[],"158":9,"1582408":[],"158241":[],"1583767":[],"15843769515580663":[],"1586300629904382":[],"1587":9,"15891336":[],"15899253":[],"159":[],"1590":9,"15939618":20,"15990":9,"15g":[6,29],"15it":[],"15pm":26,"16":[1,2,3,4,5,6,8,9,10,14,23,26,27,28,29],"160":[],"16006961":[],"16026316":20,"1603":3,"1604":[],"16049171e":[],"1605":[],"160539":[],"16054190e":[],"1606":[],"16061448":[],"1607":[],"16105443":[],"16111111111111112":1,"1612":[],"1614891":[],"16167387":[],"16171326":26,"16211139":5,"16220":9,"162246":5,"16231451":4,"16232481":[],"1625":9,"1628":9,"162999":26,"163":[],"16304863":27,"163049":27,"1630775253":1,"16342407":5,"16343471":6,"16350368":[],"163504":[],"16356503":29,"16384":3,"16397496":[],"16499193":26,"16500":9,"1655226":[],"16575256":[],"166":9,"16660817":[],"166667":[],"167":[],"16740002":[],"16754123":26,"167787":5,"168":9,"168044":[],"16804444":[],"16805821e":6,"16807":[],"16827044":[],"16832385":[],"16840081":[],"16961682":[],"16981204":[],"16b8e3cda33a":[],"17":[1,2,3,4,5,6,8,9,14,18,23,26,27,29],"17001895":[],"170019":[],"17009925":26,"17012048":[],"17074761":26,"17084902":[],"1709":9,"171525":27,"1715252":27,"17174962e":1,"17222222222222222":1,"172405":[],"17240522":[],"1726":9,"17297886491529482":[],"17300":9,"1731":9,"17339342":[],"173439546289586":[],"17428293":[],"17446471":6,"1752":9,"175300":[26,27],"175652":11,"17565241":11,"17606773":[],"17641709":6,"17647619":6,"17676228":[],"176880142835407":[],"17709473":[],"17764878e":[],"17774184e":[],"17777777777777778":1,"17796101e":[],"17801022":5,"17826349":[],"17861098":6,"17897671":[],"17917768":5,"17928538":[],"17930649":[],"17938393":[],"17942122":[],"17949575":5,"17953942":11,"1797":[1,3],"17983246":23,"18":[2,3,4,6,7,8,9,10,13,14,23,29],"180":[],"18029127":5,"18036103":[],"1807":4,"1809":9,"181":9,"1810":[],"1812":9,"18156717":[],"18166474":[],"1821":9,"18217743e":[],"18221254":[],"18264218":[],"18266178":[],"182662":[],"18276924":[],"18306165":[],"183062":[],"18327677":[],"18333333333333332":1,"1836":29,"18383522":27,"18393678":[],"184":9,"18409479":23,"184094793957168":23,"184519":[26,27],"184895":[],"185":[],"1851":[],"1852":[],"18526":[],"185278747229417":[],"1853":[],"1854":[],"1855":[],"185584177293881":11,"18573652":20,"186":[],"1860":9,"18603611e":[],"1861":[],"18611111111111112":1,"18613217e":6,"18632646":20,"18655084":5,"186551":5,"18657863":[],"18660":9,"18670072e":27,"18673098":11,"18682538":[],"18695705":[],"187268182169147":[],"18780801":26,"18790439176058887":[],"188":[],"18826299":[],"188263":[],"188499":[],"18849928":[],"1887":6,"18906854":[],"18912963":[],"18931988":[],"189367":26,"189377":[],"1893772":[],"189496":[26,27],"189622":[26,27],"1897755":20,"18998208":[],"18it":[],"19":[2,3,4,6,9,13,14,23,29],"19003":6,"19023414e":[],"19032152e":[],"19096968":[],"190970":[],"19105386":[],"19121744":20,"191262820314401":[],"19144544":[],"19158446":[],"19166666666666668":1,"191963":26,"19207979":5,"19213479":[],"19220":9,"19237495":[],"19264104e":[],"19284552e":[],"193":[],"19322111":26,"19354258":[],"19379506":[],"19394283":[],"194":[],"1940":[0,27],"19404282648955e":[],"194042826653172e":[],"194042826815498e":[],"1940428268204826e":6,"194042827197027e":6,"19425617":[],"1943":12,"194325":[],"19432526":[],"1944":[],"19444444444444445":[],"19466812":[],"19468989":20,"1950915150":[],"19528375":[],"1954":[],"19542686939553625":[],"1956":9,"19569961":[6,27],"1961":[],"1962":[],"19627430151896047":[],"1963":[],"1964":[],"1965":[],"19650622e":[],"19677483e":[],"1970":[20,26],"197104":[],"19710439":[],"1973":9,"197370":[11,27],"19740":9,"197738983259782":[],"1979":[6,29],"19800":9,"19853775e":[],"19866124":[],"1989":[],"19890453":[],"199":[],"19910037e":[],"19937":[],"19983530":6,"1999":29,"19994371":[],"1999902359008225":23,"1_1":12,"1_2":12,"1_3":12,"1cm":[0,8,10,23,26],"1d":[1,2,3],"1e":[1,2,4,13,14],"1e10":14,"1e4":6,"1f":1,"1k":20,"1n":[0,26],"1s":[],"1x":[0,26],"2":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,22,23,25,28,29],"20":[0,1,2,3,4,6,7,8,9,13,14,20,23,24,26,27,29],"200":[0,2,3,4,8,9,10],"2000":[0,27],"200000":[],"20015436":6,"20038136":[],"2004":13,"2006":25,"2008":26,"20098424e":[],"2010":1,"2011":1,"2014":4,"2015":1,"2016":[0,26],"20174121":[],"2018":[0,3,4,6,14,27,29],"2018906331":[],"2019":[8,9],"2020":[3,4,14],"2021":[6,14,27],"2022":26,"2023":[15,16,17,18,26,28,29],"20255857":[],"2027":9,"20277777777777778":1,"20299677":[],"203":[],"20313883":[],"20316225":[],"20320502":[],"2034747":23,"20355156":29,"20375361":[],"203753611274545":[],"20436207e":[],"20461136":[],"20500":9,"20509391":[],"20517254e":[],"205231":26,"20561803":[],"206":[],"2060":9,"20609082":[],"206091":[],"20661216":[],"206640":[],"20664007":[],"20669787e":[],"2069":9,"20708879":[],"207089":[],"207545":[11,27],"20756638":[],"207888":26,"20793824":[],"20833333333333334":1,"2086376":[],"20867052175003364":[6,29],"20867052175006306":[],"20867052175109335":[],"20893188":[],"20893514":[],"20916295":27,"20965182":[],"209789":26,"20980":9,"20999563":[],"21":[0,1,2,3,4,5,6,7,9,12,13,14,15,20,26,27,29],"210151":[],"2101511":[],"210340":[11,27],"21037404":26,"21058097":5,"21089449":20,"21110005":29,"21130":9,"2116753732":4,"21169159e":6,"21171751e":[],"21186069":26,"21265216":[],"21266166e":[],"21296315":[],"213":[],"213103":[11,27],"21328866":[],"21350883":[],"213509":[],"21369083e":[],"213743":[11,27],"214241":11,"21424133":11,"21435514e":[],"21467644":[],"21472683":[],"2147483647":[],"21493572":[],"21503438e":[],"21519063":[],"21561586":20,"21576737e":[],"21582995e":[],"21596432":6,"21601417":26,"216290":[11,27],"21639245":13,"21654926":[],"216683":[11,27],"21692581":23,"21786964":[],"2188":[],"21909039":[],"21913656":[],"21919813":[],"21922061":[],"219222942590059":[],"219249":[],"21924917":[],"21927988":[],"21944444444444444":[],"2195":[],"21985165":[],"21997099":[],"22":[0,1,2,3,4,5,6,9,12,13,14,20,26,27,29],"22014758":[],"220148":[],"22016672":[],"22044605e":[5,27],"220506557673408":[],"22090974":[],"22094791":[],"220968":[],"221":8,"221180":[26,27],"22134069":[],"2216":9,"2218":9,"221805":2,"221921":5,"2223808":[],"222400":[26,27],"2225":[],"2232822462117919":[],"22335823":[],"22403397554517984":[],"22442417":[],"2246674023625205":[],"22467274":29,"225":4,"22524236":26,"22566121":[],"22567203":[],"2259440937":[],"226124657522696":[],"22612466":[],"226856":[],"22685646":[],"22690428":5,"22722021":[],"22772039":[],"22821344":[],"22828694":[],"2284246870217162":[6,29],"2284246870217459":[],"22842468702288576":[],"22842468702288582":[],"22847924":5,"22887861":[],"22893751":[],"22929905e":[],"2299":[],"22it":[],"23":[1,2,3,4,6,7,9,12,13,14,20,26,29],"23002365e":6,"23014274e":[],"23056012":[],"23076923076923078":9,"23095558e":[],"231":29,"23103285":[],"23106527":26,"231224729143838":[],"2315033":[],"23167717":5,"23168292":[],"231683":[],"232":29,"2321528":[],"232153":[],"232435":[26,27],"23297056":[],"23309661":26,"23319332":20,"23333333333333334":1,"233528":[],"23365145e":[],"2338675":27,"233868":27,"234":6,"23405814":20,"23408962e":5,"23470374":[],"234723":[],"23472346":[],"23477043":[],"23483916":[],"23496851":[],"234969":[],"23522201":[],"23528337":29,"2357089093":[],"23593659":23,"2360682191515046":[],"2361111111111111":[],"2364":9,"23643365":29,"23659936":[],"2369833":26,"237038":[],"23703839":[],"23732849":[],"2379":6,"23792491":[],"237925":[],"238":29,"23816792":[],"238168":[],"238574":[],"23857423":[],"23875722":[],"23962594":[],"2397":9,"23971032":27,"24":[0,1,2,3,4,6,9,13,14,20,23,26,29],"240670854503034":[],"24085321":29,"241252":[],"24125217":[],"24140":9,"24159785":27,"2416":9,"24175744e":6,"24181044e":[],"2419":9,"24276315":[],"24279008":[],"2430":9,"24390":9,"24444444444444444":1,"24479698":26,"244858":[],"24485843":[],"24501973e":[],"24512498":[],"24533566":[],"24566623":20,"246":2,"2460037":[],"24611674":[],"24629107e":[],"24672263":23,"24688732":[],"2470":[],"24712854":[],"2476536":[],"247654":[],"24770094":[],"24782786":[],"24784064":[],"24801014":[],"24829908":5,"24844778e":[],"24848907":[],"24906604e":6,"24908246":[],"24924624":[],"24929083":[],"249302":[],"2493023":[],"2495":[],"25":[2,3,4,5,6,7,8,9,11,13,14,15,21,26,27,29],"250":[2,4,7,9],"2500":[],"25000":[0,27],"250000":[],"25006757":[],"250154":[],"25028968":[],"25033968":26,"250636":26,"25070907":[],"25115233":[],"25116851":[],"25137033":26,"2517560119":[],"25186426":26,"251879":[26,27],"25201286e":[],"25216021":[],"252436":[26,27],"25253892":[],"25259666":[],"2526":[],"252866":[],"25286618":[],"25328924":[],"25366775":[],"253668":[],"253775":[],"25385387":[],"253854":[],"2544356":11,"254436":11,"25455904":[],"25485457":[],"255":3,"255001":[26,27],"25522012":[],"256":[2,4],"25617654e":6,"25617657e":[],"25617658e":6,"256962":[],"257":[],"2572":9,"25724702e":[],"2572495066":[],"2572e3a4b38d":[],"2575":9,"25781559":[],"257816":[],"25789255":[],"25792767":[],"25845e8df859":[],"25881631":[],"25902112":[],"259153":[11,27],"2597":9,"25976336":[],"26":[2,3,4,6,9,13,14,23,29],"26015755":[],"260225":[],"26022531":[],"26079358":[],"26115367":[],"261154":[],"26153846153846":[],"26181254":[],"261813":[],"26185107":[],"26186844":[],"2619":27,"26193455e":[],"26223899":[],"26265627":[],"26290036":[],"26291585":[],"262916":[],"26292364":29,"26294938":[],"2629493813057833":[],"26297455":27,"26301436":5,"26306244":[],"26350283":23,"264":[],"26409315307910025":6,"2640931530791003":[],"26409315307910036":6,"2640931530791005":[],"26409315307910053":[],"264421":26,"264540221699101":[],"264666":[],"26466619":[],"265":[],"2650":9,"26506981e":[],"265109911":4,"26512016":0,"26521761e":[],"26531223e":[],"2654":9,"26549135":[],"266":[],"26610075":[],"26662339374864535":[],"26666667":13,"26678445":[],"26710969":5,"26780278":5,"268":9,"26804287":[],"268043":[],"268227":[],"26822717":[],"268484":[],"26848435":[],"26896710e":[],"269":[],"26995402":27,"27":[0,1,2,3,4,6,13,14,27,29],"270":[],"27011455":[],"2703":[],"27050214":[],"27090196":[],"27091656":[],"270917":[],"27092897":[],"27092910":6,"271082":5,"27108228":5,"273817":[],"27381733":[],"27404277":[],"27412395":[],"2745333":[],"27498198":[],"2750":9,"27524966e":[],"27525917":[],"275341":[],"27547877":[],"27552531e":[],"27562809e":[],"27580794":[],"27621662e":[],"27625974":[],"276263":[11,27],"27660123":[],"27674224":20,"27692307692308":[],"27700":9,"2772":[],"2775623201":[],"27760":9,"27801122":[],"278036":[],"27803645":[],"27809912":[],"27865175":23,"27898999":20,"27924636":5,"27932015e":[],"27961873e":[],"27n_":23,"28":[1,2,3,4,6,9,13,14,16,27,29],"28001319":[],"280179":26,"28047021":[],"280573":5,"280647":[11,27],"28065343":[],"28067036":[],"28097861":[],"280979":[],"28102554":[],"2812":[],"28186376":[],"281930":26,"28205578e":27,"28213741":[],"28219089":[],"282191":[],"282259":26,"28230186":26,"28266792e":[],"282727":[11,27],"28284417e":[],"283":[],"2830637392":4,"2831603":[],"28336218e":6,"2836":[],"28390":9,"28391978":27,"28443039":29,"28457442":[],"28475098":8,"28500583e":[],"28506507":[],"28587856":20,"28590743":[],"2861":23,"28634473":[],"286345":[],"2871":9,"2873":9,"28744729":[],"2882":23,"28836053":[],"288455813429108":[],"2886":23,"2886847885377843":[],"28868479":[],"28875373":[],"2890":[0,26],"2891298":[],"2892":23,"28948521":[],"2895238":[],"28967287":[],"28it":[],"29":[3,4,6,7,9,14,29],"29082851":[],"290829":[],"29083183":[],"290832":[],"29115152e":[],"29148545e":[],"29149329":[],"2915":23,"29167186":5,"29229741":[],"2923076923077":[],"29238177":26,"29276615":[],"29294618e":[],"2931":26,"294277":[],"2942772":[],"29512284":[],"295123":[],"2954":[],"29546690e":[],"2955":[],"2956":[],"2957":[],"2958":[],"29588901":[],"29592539":[],"296247":[],"296414":26,"29650918":[],"29661191":[],"29662593e":[],"29672381":[],"2968":26,"297":[],"2971492148":[],"2972":[],"297248":[],"29724806":[],"2980":26,"29822833":6,"298273":[],"29830651":13,"298375":[],"298836":[],"29883607":[],"299":[3,4,14],"2990":26,"299267190588216":[],"29938171e":[],"29972182":[],"299748":[26,27],"29it":[],"2_":12,"2_1":12,"2_2":12,"2_3":12,"2_i":12,"2_m":[6,23,29],"2_t":13,"2_x":23,"2b":23,"2cm":8,"2d":[1,3,11,12,19,26],"2e":[6,29],"2f":[0,7,9,10,11,12,26,27],"2ff97f4bf03b":[],"2g":2,"2g_i":2,"2k":3,"2m":[6,29],"2n":[0,2,3,26,27],"2nd":9,"2p":23,"2pm":[24,26],"2pt":4,"2s":[],"2x":[0,3,8,13,26],"2x_ix_jy_iy_j":8,"2x_j":8,"2y_i":10,"2y_j":8,"3":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,19,20,21,22,23,24,28,29],"30":[0,1,3,4,6,7,9,10,13,14,20,23,24,26,27,29],"300":[3,4,14],"30000":[0,26],"30010":9,"30035718":[],"3005849":[],"301":[3,4,14],"30119421":8,"301475":[],"3014751":[],"3017":[],"3018":[],"3019":[],"301927":[],"302":29,"3020":[],"3021":[],"30232456e":[],"30258509":[20,26],"302m":[],"303":6,"30313128":[],"30337713":[],"3037":[],"3038":[],"30384239":[],"3039":[],"303m":[],"3040":[],"3041":[],"30442964":27,"30466214e":6,"304m":[],"305":[0,26],"30535506":[],"30552077":[],"30559547":[],"30577095":26,"305m":[],"306":[0,3,4,14,26],"3063":[],"30638512e":[],"306m":[],"307":[0,3,4,14,26],"30700873e":[],"30702075":[],"3072":3,"3073":29,"30761405e":[],"3076923076923077":9,"30787294":6,"3079975":[],"307m":[],"308":[0,3,4,14,26],"30813073":[],"30839676":[],"30844205":[],"308763":[],"30891693":[],"308917":[],"308m":[],"309":[0,3,4,14,26],"30936179":[],"309362":[],"30940":9,"30984244e":[],"30993886e":[],"30b53504a633":[],"30it":[],"31":[3,4,6,12,14,20,23],"310":[0,3,4,14,26,28],"310579":27,"3105791":27,"311":[0,26,28],"31120247":[],"31138734e":[],"31193395e":[],"311m":[],"312":[],"3120271598582915":[],"31211671":[],"31228042":[],"31229747":[],"3123314713548606":[6,29],"31276579e":6,"312m":[],"313":28,"31311243":[],"31318084":5,"31331481":[],"313315":[],"31347523":[],"31377492":[],"314":28,"3141149":[],"31424359":[],"314244":[],"31447174":[],"31457796":5,"315":6,"31507591":[],"31519573":[],"31531613":[],"3155":[0,5,6,28,29],"315746":[],"31574634":[],"315977":[],"31597731":[],"316":[],"31605061":[],"31614188e":[],"31644071":[],"31650694":6,"31658671e":[],"31681097":[],"316811":[],"317":[],"31718909":11,"31719317":[],"317367":11,"31768693":[],"31782593":[],"317825933186352":[],"3178411477108273":[],"317m":[],"31816439":26,"31835835":[],"31866499":[],"3189":29,"31896852":8,"319803":[],"31980301":[],"31988705":[],"319m":[],"31it":[],"32":[3,4,6,12,13,14,20,23],"3200":1,"32039227e":[],"32047562":[],"32066545e":5,"32068012":[],"3208":2,"320m":[],"3211480515121596":23,"32149601703519115":[6,29],"3214960170351912":[6,29],"3215":9,"32158804":[],"32179365":[],"32183101":[],"32183405":[],"32190363":[],"32305813":[],"32309075":[],"32320052":[],"32360769":[],"323608":[],"324":2,"32450054":27,"32458459":[],"32459186":[],"32474290e":[],"324m":[],"3250":[1,6],"32512":[],"32541271":[],"32542732":26,"32566092":[],"32609560e":[],"32611807":[],"326238":[26,27],"3263505":[],"3267061":[],"326m":[],"32714903e":5,"32721178":[],"327212":[],"32750531e":[],"32766274":[],"32784259e":[],"327883798133277":[],"32816737":[],"32903042":[],"329492":26,"32968167":23,"3297":[],"32992274":[],"329923":[],"32it":[],"33":[2,3,4,9,12,14,20,24],"330":[3,4,14],"3303":[],"33066907e":[5,27],"33078983":[],"33079801":[],"331":[3,4,14],"3310":[],"33113182":26,"3313155":[],"33135331":[],"33166055e":[],"3318361":[],"331939":[26,27],"33196524":13,"33199489":26,"331m":[],"332":[3,4,14],"332018":[],"33201844":[],"3322326":[],"332331":[],"333":[3,4,7,14],"3331":[],"33327369":[],"333274":[],"33333333":13,"3337":[],"33370315e":[],"334":[3,4,14],"335":[3,4,14],"33589475":[],"336":[3,4,14],"33611111111111114":[],"33645986e":[],"33646268e":[],"336801":[],"33680101":[],"33697405":[],"337":[3,4,14],"33711888":[],"33736485":[],"338":[3,4,14],"33819480e":[],"33823452e":[],"33829607":26,"3384":[],"33874172":20,"33900713":[],"33939645e":[],"33941551":[],"33956555":[],"33it":6,"33m":[],"34":[2,3,4,9,14,20],"340071371496255":27,"3403":9,"34037032e":[],"34040204e":[],"340583":26,"340782":[11,27],"34114547":5,"34158665":[],"341m":[],"34212496":[],"342680":[26,27],"34270689":23,"34305928":[],"3431023":26,"3435384":26,"3436":[0,26],"3437":[0,26],"34378387e":[],"34412923":27,"34498451":[],"3456":[],"34561469":[],"345687771875474":[],"34568872":[],"34569596":5,"3457":[],"34578026":[],"3458":[],"34585355":[],"3459":[],"34591885":[],"345919":[],"3460":[],"34626037":[],"34652523":[],"34678929":[],"346810":[],"34695383":[],"346954":[],"3469819128513336":[],"3473884":[],"34740615":[],"34852463e":[],"34914442e":[],"3498":[],"34989509":[],"34it":6,"35":[0,2,3,4,6,9,14,17,21,24,26],"35030572":[],"35074881":[],"350749":[],"35123041":[],"35128114":20,"3514":[],"35140":9,"35147135":[],"35147532":[],"351636":[11,27],"35182854":5,"35203688":[],"35207264":[],"352073":[],"3521478":26,"3522":[],"3525":[],"3528556":[],"352856":[],"3529":[],"3530606977":[],"35309734":[],"35319678":[],"35346808":[],"3538":[],"35396404e":[],"35401056e":[],"35412147":29,"3543":[],"35438626":[],"3544313922":6,"3546":[],"35470445e":[5,27],"35473061":[],"3548":[],"355":[3,4,14],"3551":[],"35533773":6,"35533774":[],"35559283":20,"3556":[],"35562617e":[],"35568525":[],"3558":[],"355906":[],"35590603":[],"356":[3,4,14],"3560":[],"356399":[26,27],"3567":[],"357":[3,4,14],"35700089":26,"35704388":[],"357044":[],"3572":[],"35724288e":[],"3575":[],"357508":[26,27],"35759569e":[],"3576":[],"3577":[],"35770742":[],"35771826":6,"35771842":[],"35776087e":[],"35793003441520066":[],"358":[3,4,14],"3581341341":4,"3581571":[],"35819312":[],"35821426":[],"35846425":29,"3585":[],"3587":[],"35892474":[],"35894575":[],"359":[3,4,5,14,28],"3593":[],"35947075":[],"3594821":[],"3595":[],"3598":[],"359999999999985":[],"36":[0,2,3,4,5,6,14,21,23],"360":1,"3604":[],"3605":[],"3606":[],"360688":[],"36074377":26,"3609":[],"36102113":[],"36117602":[],"3613":9,"3615":[],"361556":[26,27],"3617":[],"3621311":5,"36227629e":[],"3624":[],"3627":[],"3628":[],"363295916323784e":6,"363295916414895e":[],"3632959170548605e":[],"3632959215700067e":6,"363295924430451e":[],"36336962":20,"363834":[],"36383443":[],"363936":[],"36393643":[],"364244035362507":[],"36424404":[],"3643":[],"3644017":[],"364402":[],"364418e97433":[],"3645":[],"36452684":26,"3646":[],"36462687e":[],"3647":[],"365350":[],"36535019":[],"3655":[],"3655222":5,"36570403":[],"3659":[],"36593124":13,"3665263":[],"3668":[],"36689784":[],"366898":[],"3669":[],"36692356":[],"367":2,"3672":[],"3673":[],"3674":[],"3679":[],"36809474":[],"3684":[],"368537":[],"36853738":[],"3687":[],"3688":[],"368m":[],"36904006e":[],"369139":[11,27],"36it":[],"37":[3,4,6,9,14,21,26],"3702755":26,"3704":[],"370782966":4,"3711":[],"37114298":[],"37126183":23,"3713":[],"37157712":[],"3718":[],"3721":[],"3722":[],"37239927e":27,"3725":[],"3729":[],"37307168":[],"3739":[],"37396662":6,"37415316":[],"37419664":[],"374197":[],"374291":[],"37429133":[],"3744637":5,"374464":5,"3748":[],"3749":[],"3753":[],"375351":[],"37535105":[],"375694":26,"37585988":[],"3759":[],"3760":[],"37622356":26,"37624303":[],"3765":[],"376547":26,"376559":[],"37655936":[],"376834":[],"37683438":[],"37692363":[],"37703055":[],"37713991":[],"3772":[],"3773":[],"37732":[],"377372":27,"37737221":27,"3776":[],"3777801602":6,"3779":[],"37816901":[],"37835429e":[],"3784":[],"378664":[],"37866422":[],"37885034":[],"37900111":6,"3791":[],"379203":[],"3794":[],"379647":[],"37964744":[],"37992857":[],"37it":[],"38":[3,4,9,14,23],"380":9,"38019139":[],"3802":[],"3803":[],"3804":[],"3805":[],"380739":[],"38073947":[],"38098948":26,"3811":[],"38135733e":6,"3815":[],"381627865854956":[],"38162787":[],"38168549":[],"3817475779":[6,29],"3818":[],"3819":[],"3820":[],"382187":26,"38222896":[],"3823886":[],"382389":[],"38246359":[],"382672":[],"38267217":[],"3827":[],"382951":[],"38295101":[],"38299798":[],"3830":[],"38302314":[],"3833":[],"38336316":[],"3834":[],"3837":[],"3838":[],"383878":[],"3838783":[],"3839":[],"383942":[],"38394242":[],"3842":[],"38432672e":[],"38461538461538464":9,"38469441":[],"3850":[],"3851":[],"38533184":[],"38533185":6,"3854":[],"3858":[],"386":9,"38605872":[],"386059":[],"3861":[],"3862":[],"386294":[],"38629412":[],"38629436":[20,26],"38629844":[],"38636467":[],"38637915":[],"3864":[],"38644889":[],"3867":[],"3869":[],"387":28,"3871":[],"3872":[],"38730917e":[],"387482":[],"38748219":[],"3876":[],"38763091":[],"38788697":[],"388":[],"3881":[],"38816656":[],"388167":[],"3882":[],"3885":[],"3886":29,"3886211842840971":[],"38868469":[],"3888":[],"3889":[],"389":[],"38901478":[],"38906684":[],"389067":[],"3891":[],"38916861e":6,"38962192e":6,"3898":[],"39":[0,3,4,9,14,24,26],"390":[],"3906":[],"39074222e":[],"39095416":[],"39116042":[],"3914":[],"3915":[],"3916":[],"391602":[],"3917":[],"39197698":27,"391977":27,"39214397":27,"392144":27,"3922":[],"3925515884752442":[],"39272691":[],"3928":[],"3929303193461995":[],"3931":[],"39311435":[],"39340081":[],"39384255":[],"393843":[],"39417707e":[],"3943":[],"3944":[],"39444293":[],"39456996":[],"39493843":[],"3950":[],"39548035e":[],"3955":[],"39550083":[],"395501":[],"39560937":[],"3957482":[],"39579407":5,"3958":[],"39595799":[],"3960":[],"39600607":[],"39614347e":[],"3962":[],"39626502":[],"3970":[],"39706038":5,"39707147":[],"39724390e":[],"39730396":21,"3975":[],"39769983":26,"397700":[11,27],"39789527":[20,26],"3979":[],"39794864e":[],"3980313467":6,"3983":[],"39864527":[],"39892216e":[],"39920034":[],"39924562":[],"399246":[],"39931051e":27,"3994":[],"39943328":[],"3996":[],"39969826":23,"399836":[26,27],"39it":[],"3bdkrb":[],"3d":[2,3,4,6,13,21,29],"3f":[1,3,9],"3n":20,"3x":[2,8],"3x_i":2,"3y":8,"3yk470mj5p931p9dtkk0y6jw0000gn":[1,6,13,21,26,29],"4":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,23,26,28,29],"40":[1,3,4,6,9,14,21,24,26,29],"400":4,"4000":[9,26],"40026896":13,"4005":[],"40062254e":[],"4011":[],"40116777":[],"401168":[],"4012":[],"4017":[],"4017308":[],"40183706":[],"401842":[11,27],"40214433":[],"4022":[],"40236659e":[],"40236901e":[],"40264255":[],"4027":[],"4030":[],"40340043":[],"4034668048":[],"40355028":23,"4038":[],"40389922":[],"4039":[],"404":27,"4045":[],"4048":[],"4050":[25,26],"40506137":[],"4055":[],"405890":[11,27],"4059":[],"40599799":[],"40620066":27,"406201":27,"40672425":[],"4068":[],"4069":[],"4075":[],"40754621":[],"40768645e":[],"4077":[],"40789038":23,"408":2,"40805354":26,"4082":6,"40828861e":[],"40829683":[],"408297":[],"40837841":26,"4084":[],"4087":9,"4087793":5,"4088":[],"4089":[],"409":2,"40927184e":6,"40947888":20,"40961677":[],"40980089":[],"409801":[],"40984037":20,"40a38ad763f1":[],"41":[0,2,3,4,9,14,20],"410":2,"4100":[],"41050604":20,"4107":9,"410815":[],"41081513":[],"41083418e":[],"41091395e":[],"411":2,"4115":[],"41165766":13,"4117":[],"41170104":[],"412":2,"41211579":[],"412116":[],"4128":[],"4131":[],"41335011":[],"41340248":[],"41351287":[],"413513":[],"41357064":[],"41369795e":[],"41386599":[],"4143":[],"41433969":5,"4144":[],"41446721":[],"4146":[],"41486312e":[],"41511965e":1,"415201":26,"41542567":[],"4155":[2,15],"415534701258823":[],"41566661":[],"4162706317":6,"41646135":[],"4166666666666667":9,"4167":[],"41718305":20,"4176":[],"4177":[],"41816468e":[],"418506":[11,27],"4185877":26,"4186":[],"41876267":[],"418763":[],"41882036e":[],"41882037e":6,"41923956":[],"4192423635":[],"41958102e":[],"4198":[],"41990268":[],"41it":[],"42":[1,3,4,8,9,10,14,20],"420":2,"42028578":[],"42037468e":[],"420543":[],"42054337":[],"42078103":[],"42090072":26,"421":2,"4212":[],"42130182":[],"42190899e":[],"422":2,"422275":[],"4224":[],"422480":[],"4224801":[],"4229538":11,"422954":11,"423":2,"4230769230769231":9,"42340403e":[],"42352791":[],"42394972":[],"424":[],"42441033":5,"42449643":[],"42450":9,"42451192":[],"425":[],"4255":[],"4258989918":[],"426":[6,7],"42615374e":[],"42619859":[],"42631342":[],"42694395":[],"427017":[],"42729886e":[],"42732954":[],"42745728579187":26,"4275":[],"42790075e":[],"42794448":[],"4284066":[],"42882114e":[],"429":29,"42902155":[],"42915006":[],"42976761":[],"43":[0,1,2,3,4,7,9,14,20],"43001767":[],"43007002e":[],"4300745":[],"43012535e":[],"43036842":[],"4304167":[],"43054282":5,"4310":26,"43113284e":[],"43135183":[],"43185254":[],"43294696e":[],"433":29,"43330971e":6,"43333886":[],"43379387":23,"43471448":20,"43482628":[],"43490889":23,"434932":[],"43493232":[],"435163":[26,27],"43533824":20,"43552433":[],"435567":[],"43556723":[],"43560678":[],"43579948e":6,"43639284e":27,"436462435":4,"43656628":[],"43693489":[],"43761347":[],"43766686":11,"437731":11,"43773101":11,"4378299":20,"4379":[],"43794243e":[],"43795846":[],"438060758":6,"438111004052204":[],"438136":[26,27],"43876695":[],"438767":[],"439230":6,"43975616":26,"4399427":[],"439943":[],"44":[0,1,2,3,4,14,20],"44021223":[],"44065609":[],"44089210e":[5,27],"4410":[],"4411":[],"441264":26,"44152248":[],"44190304":[],"44198566":[],"4421034379901165":[],"44210344":[],"44214614":26,"442600":[11,27],"442701":[],"44270138":[],"443217":[26,27],"443538568902246":[],"44353857":[],"444":9,"44463741":[],"446033":29,"44624525e":27,"446453":26,"44651551":[],"44688507e":[],"44792888":20,"44830642":[],"44838453e":[],"44879019":23,"44894073":[],"44906236e":[],"44970586e":1,"44993812":26,"44997121":[],"44it":[],"45":[2,3,4,9,14,24,26],"450":9,"45020332":23,"450257":[11,27],"4504":9,"45112054":[],"45112450e":[],"45143018":[],"45146007e":[],"45220770e":[],"45241645e":[],"4524458":23,"452553":26,"45268835e":[],"45290829":[],"4533029":5,"453303":5,"45359479":26,"45372697":[],"453727":[],"45393214e":[],"454027":[],"45402701":[],"4543859":[],"45464765":[],"45494697":20,"455173":29,"45571233e":[],"4557763":11,"455947":[26,27],"456":9,"456418966187335":[],"45643708e":[],"457":2,"45720968e":[],"45739087":[],"45741697":[],"457770268480242":[],"45797073":[],"458027":[26,27],"458078":[11,27],"45840226e":[],"45865699e":[],"45915671e":27,"45917334":20,"45951629e":[],"45960079":5,"45979894":[],"45985488":[],"459855":[],"46":[2,3,4,9,14,24,26],"4601":9,"46023985":[],"461":[],"461175":[],"46153846153846156":9,"46164895":26,"462":7,"46284227":[],"46312791":[],"46369869":[],"46383925e":6,"463861":26,"46400080e":[],"464424":[],"4644244":[],"46508305":[],"46567887":[],"46573851":[],"46575843e":[],"466":[],"46602982":[],"46650488":[],"466505":[],"46696223":[],"4676059":27,"467606":27,"46794929":[],"469":[],"46932688":[],"4694":[],"46984697e":6,"469868":[],"46986815":[],"47":[2,3,4,9,14,24,26],"47008799e":[],"4703":[],"47042744":5,"47057708":23,"470714":[26,27],"47075725":6,"47082726e":[],"47099765e":[],"47116868e":6,"47125748":5,"47132891":5,"47158029e":[],"47202234":20,"47202442":29,"47245463":[],"472455":[],"47248297":[],"47313680":29,"47355725e":[],"47356208":26,"47387858":[],"47400238":[],"47432993":[],"4744":[],"4744219":[],"47478057":[],"47482507":[],"47485224":[],"47513129":20,"475311":[],"47531107":[],"4756968":20,"47610036":6,"47649841":[],"47726407e":[],"47732343e":[],"47756853":20,"47815203":11,"47920156":[],"47942814":[],"479465113":4,"47977394e":[],"48":[3,4,9,14,29],"480":26,"48005485":11,"480055":11,"48089797":[],"48133064":[],"48179077":23,"481979":6,"48212873":[],"48217977":[],"48241947":[],"48257387":[24,26],"48259891e":[],"48316523e":[],"483257001":13,"4837":[],"48420165":[],"48442336":[],"48442771":[],"48476997":11,"48574149":[],"486873":[],"48687342":[],"487000566129411":23,"48739546":[],"48783291":11,"487833":11,"48783370e":[],"48802709":26,"48871288":[],"488713":[],"48877744e":[],"489":[],"489522":[],"48952201":[],"48971452":[],"48974963":27,"48994188":5,"48it":[],"49":[3,4,5,6,9,11,14,21,27],"490":[],"49001938e":[],"49052317e":[],"491":[],"49152":3,"49197115":20,"492":[],"49242688":[],"49282737":[],"493":[],"49352496":[],"493525":[],"49372893":[],"49385454e":[],"4940954":[0,26],"49468927e":[],"49545491":[],"495708":[],"49570819":[],"4959161509356135e":[],"495916150936645e":6,"495916150936654e":[],"4959161509377256e":6,"495916150938325e":[],"49614357":[],"496337":[],"49633743":[],"49636583":[],"49666527e":[],"49672291":[],"49680475019226":[],"497":[],"49730046e":[],"498":[],"49838167e":[],"49862065e":[],"49887949e":[],"499":[],"4990":23,"499012":[],"49901244":[],"4992":23,"49932184":26,"49932427e":5,"4997":23,"49974854":[],"49992743e":[],"4b5d3mupqma":[],"4c4c7f":[9,10],"4d":3,"4f":6,"4pm":[24,26],"4y":8,"4y_i":10,"5":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,20,21,23,26,27,28,29],"50":[1,2,3,4,6,7,8,9,10,13,14,26,27,29],"500":[1,3,4,6,9,10,13,29],"500000":[],"50000455":5,"50000553":5,"50000718":5,"50000855":5,"50000969":5,"50001063":5,"50001142":5,"50001207":5,"50001261":5,"50001306":5,"50001343":5,"50001374":5,"500014":5,"50001414":5,"50001422":5,"50001439":5,"50001454":5,"50001466":5,"50001476":5,"50001485":5,"50001492":5,"50001498":5,"50001502":5,"50001506":5,"5000151":5,"50001512":5,"50001515":5,"50001517":5,"50001518":5,"50001519":5,"50001521":5,"50001522":5,"50001523":5,"50001524":5,"50001525":5,"501":[],"501049":[],"50104946":[],"50105159":[],"50173752e":[],"5018":23,"50219982":[],"50227564e":6,"5031474174499113":[],"50314742":[],"50321091":5,"504167":[],"50416731":[],"50427787":29,"50453066":26,"50456256":5,"504563":5,"504881":[],"50488131":[],"50504743":[],"50584526":[],"506":[0,27],"506553":27,"50655336":27,"50680321e":[],"50727059":[],"50733669e":[],"50754416":[],"50769230769231":[],"50799086":[],"507d50":[9,10],"50812169":[],"50835355":[],"508354":[],"50846111e":[],"50846112e":6,"50894343":[],"50931378":[],"50it":[],"50j":13,"50x10":1,"51":[3,4,10,14],"510":1,"51004249":[],"51013506":26,"51054961e":[],"51075117":[],"51108791":[],"51126895e":[],"51131471":[],"511315":[],"51146442":[],"51150176":[],"51174395":[],"51178587":23,"511888":5,"51191552":6,"511977":[],"51197747":[],"512132":[],"51214899":[],"51220693":26,"51265232":[],"51289697":[],"512897":[],"51374050":29,"5138":[],"51389553":[],"51400374e":[],"514219":[26,27],"5146617":26,"51507839":[],"51508374":[],"51561271":[],"51578522e":[],"51656406":[],"51664729":[],"51719273":5,"517193":5,"51732028":[],"517350858882083":[],"5174581":[],"51749473":[],"5177783846":4,"517823":[],"51782322":[],"517931":5,"51793145":5,"518030":[],"51803019":[],"51850390e":[],"5188328":[],"518833":[],"518895":26,"51900727e":[],"51905011":[],"51933924":[],"519750":5,"51975002":5,"519842":[],"52":[3,4,14],"520931":[],"52093124":[],"52113415":[],"5216048821598704":[],"521719":[],"52171908":[],"52191849":[],"521986":[],"5222222222222223":1,"522758":[],"522836":26,"52307692307693":[],"52362157e":27,"52428467":[],"52512084":[],"52580392":[],"525804":[],"5260627":[],"52606351e":[],"52606392":11,"526744":[11,27],"52678301":[],"52713812":[],"52723079":[],"52744746":[],"52773051":[],"52773891":[],"52775466":29,"52799362":[],"528115":[],"52811546":[],"52812155":23,"52819229":[],"52839842":[],"52872817":20,"52874252":5,"529":[],"52902847e":[],"52911941":[],"52945798e":[],"53":[3,4,9,14],"5303329":11,"5305555555555556":1,"5312":[],"531280":[26,27],"5316382":26,"53189647":[],"53219937":[],"53254035":[],"532643":[],"53264317":[],"53312754":[],"533142":20,"5333734":[],"533941":[],"53394148":[],"53423784":[],"5342745":26,"534362":26,"53462883":26,"53476918":20,"53479276":[],"534793":[],"53502096e":[],"53603432":[],"53611562":[],"53617977":11,"536180":11,"53623614":[],"53697476":[],"53703498":6,"53738247":29,"53752634":[],"5378811":11,"53794784":[],"537948":[],"53815559":[],"53822787":[],"5384615384615384":9,"53846153846155":[],"539261":[11,27],"5393681":20,"53938383":[],"539384":[],"53952479":[],"539525":[],"53952664e":[],"539683":[],"53968309":[],"53971260e":[],"53it":[],"54":[3,4,6,9,14,23],"540":9,"54016188":[],"540162":[],"54039921":5,"54041041e":[],"54041662":[],"54078623":[],"541605":[26,27],"54163136":[],"54219245":[],"5424657":13,"54302039":[],"543939":27,"54393936":27,"544439":[26,27],"54445121":[],"54526769":[],"54537329":[],"546166676":[],"54632096":26,"54637219":29,"54644646e":[],"546712":[],"54671213":[],"5472246386316972":[],"54722464":[],"54780216":[],"548110376991839":[],"54845458":[],"548455":[],"54861905":20,"54886137":[],"54894095":[],"54956992":[],"54957609e":[],"54969168":[],"54it":6,"55":[1,3,4,9,14],"5501":[3,4,14],"55026099":[],"550321":[],"55051175":[],"55070893e":[],"55141095":[],"55202922":[],"55220861":[],"55229041":23,"552731":[],"55273102":[],"5538822":20,"5539":[],"55505907":[],"555187":[],"55518724":[],"555496":[],"5554964":[],"55553537":[],"5555555555555556":1,"55571665":[],"55578041":[],"55626454":[],"55640013":[],"55640387e":[],"5566":[],"55687658":13,"55765644e":[],"557795":[11,27],"55805262":26,"558080":[],"55808001":[],"558241":[],"55824107":[],"55854694":11,"55907387e":[],"55914617e":[],"55936475e":[],"5594":6,"55965719":[],"56":[1,9],"56033697":5,"56071752e":[],"561":[],"5611":[],"5615":[],"56183518":[],"56198284":5,"5625":[],"56302854":[],"563167":26,"56343926":[],"56383695":23,"5639":[],"564":9,"564374":[11,27],"56437897":[],"564379":[],"56447742e":[],"56481272":[],"565":9,"565006":[],"56500639":[],"56536":[0,26],"56542321e":[],"56556315":[],"565651":[],"56565106":[],"56589683":29,"566":9,"566074":[],"56607416":[],"56636616e":6,"567":9,"568":[9,29],"56807589e":[],"56821272":20,"56848886":[],"569":[1,9],"56912044e":6,"56939714":5,"56953559":26,"56965674":29,"56992937":[],"57":[0,8,9,24,26],"570":9,"5700":[],"571":[5,28],"571092":[],"57109235":[],"571105947979336e":[],"571105947979352e":6,"571105947979394e":6,"571105947979395e":[],"571105947979439e":[],"57113133":23,"57154252":[],"57174058":[],"57201944e":6,"572069":26,"57266138":[],"57286194":[],"57350724":[],"57361898":[],"57403561":[],"57445493":[],"574465":[11,27],"57488231":23,"57505712":[],"576":29,"576431":[],"57643113":[],"57673618":[],"5769230769230769":9,"57776818e":[],"57807896e":[],"57810065":[],"57810898e":[],"5783575":[],"57842073e":[],"57867898":[],"578889":[],"57888946":[],"57947706":[],"57it":[],"58":[2,9,10,24,26],"58039072":[],"58042237":[],"58084359":[],"58104393":[],"581044":[],"58112299":[],"581123":[],"58171189":[],"58187211":23,"58187347":[],"58312967":[],"5833333333333334":9,"583595":[26,27],"58395707":[],"58428804":[],"58474054":[],"584804":[],"58492636e":27,"58574143":[],"58590795":[],"58666142":[],"58671946":[],"58738148":[],"5888888888888889":1,"589":[],"58908017":26,"58912508e":[],"58986647":[],"59":[2,9],"590":[],"59095724":20,"591":[],"59118929":[],"591317992":4,"591594":[],"59159438":[],"59164202":26,"5916751":20,"59182949":[],"59190877":[],"592":[],"59203799":20,"59277468":[],"593":[],"593040":[],"59304008":[],"59323905":[],"59366475":20,"59367954":[],"5944444444444444":1,"59445979":[],"594460":[],"59468796":[],"59567145e":[],"59591979":[],"595920":[],"59592669":[],"59609892":[],"59616454":[],"596165":[],"59654302":[],"597084":11,"5970842":11,"59766":[],"597660":[],"59813773":[],"59833875":[],"598392":[],"59839245":[],"59914165":[],"599142":[],"59950055":[],"59955801":[],"59981487e":[],"59990347":23,"59it":6,"5cm":23,"5dd54edf2138":[],"5f":8,"5x":8,"5y":8,"6":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,23,24,26,27,28,29],"60":[1,2,3,6,9],"60000":4,"60056803e":[],"60081665":[],"60083846e":[],"60133562":[],"60158334":[],"601583341274718":[],"60164827e":[],"60176806":[],"6019067271":4,"601942":[],"60194224":[],"60236782":[],"60236938":[],"60268567e":[],"60293962":5,"603494":[],"60349429":[],"603636":26,"60378214":26,"60420593":5,"60515083":[],"6052136":[],"60522226":[],"60538875":[],"60543038":27,"60619654":[],"6063219":[],"606439":[],"60651016":[],"60673226":11,"606760":5,"60685864":[],"60707801":[],"60715594":[],"607384":11,"60738434":11,"60791699":[],"6079169911277265":[],"608028":[],"60802823":[],"60815105":6,"60826013":[],"6084611325305795":[],"60938882":[],"609389":[],"60943791":[20,26],"60975366":[],"60980325":[],"60it":6,"61":[2,7],"61031939":[],"61033524":[],"6111111111111112":1,"61123608":[],"612939":[26,27],"613579":[26,27],"61399851":[],"613999":[],"61406871":[],"614133":[],"61413333":[],"61463451":[],"614808":[11,27],"6149274949460215":[],"61520194e":[],"61536670e":[],"6153846153846154":9,"61576236e":[],"61605739":[],"61634314":[],"61702282":6,"617572":[],"61757228":[],"61775176":29,"618":[],"61808351":[],"61885405":23,"61893735":[],"618982":[26,27],"61899355e":[],"61970354":20,"6199381169260247":[],"62":[],"62083489":20,"621102":26,"62123034":[],"62168613":[],"62170669":[],"621707":[],"62223681":5,"622237":5,"62249103":[],"62364974":[],"62373464":11,"62389761":[],"62405149":[],"62420578":20,"624220":[],"62422045":[],"62432074":[],"62451626e":[],"62470815e":[],"625":7,"62571673e":[],"6257439":26,"62620724":[],"626635268":[6,29],"62683307":[],"62686358":26,"62759057":[],"62761922":[],"62767384":[],"62769765":[],"62806444":[],"62819534":20,"62894215":5,"62896882":[],"62914124":[],"629961":[],"63":[0,1,6,7,27,29],"6300745149331701":27,"63012429":[],"630224":[],"63025821e":6,"63112036":[],"63115792":[],"63162342":27,"63180447":29,"63207808":[],"63249532e":6,"63258368e":[],"63262164":[],"63264564":[],"63270833":[],"63281351":[],"63332882e":[],"63367582":[],"633676":[],"633949":26,"63395187":[],"63430285":[],"6343356":[],"63498144":5,"63550707":[],"6367311":[],"63675140":[],"637104":[],"63710438":[],"637129335071195":27,"6371293350711955":[],"63734811e":[],"637545":[],"63754524":[],"63860687":[],"63862189":[],"63901111":[],"63933182":[],"63it":[],"64":[1,2,3,4,7,13,20,26],"64008343":5,"64012627":5,"64033527":[],"64064128":[],"64113381e":[],"64141716e":[],"642380":[],"64238022":[],"64293754":[],"64298419":[],"6431453":[],"64316192":29,"64316482":29,"64425009":[],"64452882":[],"645":[],"64528459":[],"645285":[],"64615384615385":[],"646283":[11,27],"647":6,"64736858":[],"64741194":23,"64742912e":6,"647473":[11,27],"64858537":[],"6489862":[],"649339":[],"64933923":[],"649382":[11,27],"64972151":[],"64x50":1,"65":[1,2,7,8,9],"65011691":20,"65045925":26,"65136857":[],"65246006":[],"65280103":[],"6530742540053943":[],"65335632":[],"653578701904388":[],"65359027e":[],"6536392":27,"65442354":27,"654424":27,"65444431":[],"6546329613199204":[],"65522261":[],"65534935e":[],"65554133":[],"65571174e":27,"65578316":[],"65599456":27,"6562":[],"65626043":[],"65628853":[],"65643756e":[],"65693278e":[],"657041":26,"65743689":27,"65746064":[],"6577093":[],"65795279":[],"658":[],"65833132":27,"65885453":5,"65891389":27,"658914":27,"659":[],"65907898":[],"659306":[],"65it":[],"66":2,"660":[],"66001778":[],"66054752":[],"66064822":27,"66080313":27,"66087937":[],"66111673":[],"66124684":5,"66204648":6,"66219404":6,"66226149":[],"66247212":[],"66250644":[],"662854":[],"66285417":[],"66285434":[],"6628996975186953":27,"66292841":[],"66294408":27,"66295776":[],"662958":[],"66310422":[],"66351927":[],"6638":9,"66410989":[],"66443521":[],"6652177":27,"66560":9,"66599246":[],"66614025":[],"66619972":[],"666200":[],"66620847":[],"66638078":[],"666597":26,"66665241":[],"6669838269597004":[],"667":9,"667239":26,"66726905":[],"66758934":[],"668172":[26,27],"668186":[],"66818635":[],"66934291":[],"66950317e":[],"66973261":[],"66981186":[],"66it":[],"67":[2,9],"67047975e":6,"67054028":[],"671089":[],"67142502":20,"67160204":[],"67177934":[],"67236038e":[],"67258699":[],"672721":[26,27],"67314874e":[],"673457751724773":[],"67354673":[],"67392051":5,"673921":5,"67442363":[],"67554302":[],"67588315":[],"67591616":[],"67655635":[],"67697934":[],"67743788":23,"67783138":[],"67838309":[],"67882608":[],"67970864":[],"679709":[],"67973263":[],"68":2,"68029036":[],"68040392e":[],"68075712":[],"6808538439837775":[],"68127341":[],"68176047e":[],"68192193":5,"68232235":23,"68246089":[],"682464":[],"68246434":[],"68279358":[],"68282072":11,"68282653":[],"68286725":27,"68322599":[],"6834195":[],"68347072":[],"68372271":[],"68485505":[],"68534263e":6,"68542204":5,"68545647e":[],"68592431":29,"68626148e":[],"68650491":[],"68664664":[],"6869":9,"68719389":[],"687194":[],"68731669e":[],"6887363571":4,"68886089e":[],"689230294669661":[],"68929213e":6,"689345":26,"68944595":[],"689519":[11,27],"68965135":[],"68996179":[],"68it":[],"69":[2,7,9,23],"690":9,"69023787":[],"69051742":20,"690569639355314":[],"69061276":[],"690617":[26,27],"69069n_":23,"69115646":[],"6914791":[],"69154964":[],"692":[1,29],"6923076923076923":9,"6924788":5,"692479":5,"69295955":[],"69313663e":[],"69356940e":[],"69363975e":[],"6943316601792833":[],"69481746":[],"69484813e":[],"69487891e":[],"69493539":[],"69504801":6,"6960326":[],"69634577e":6,"69695259":5,"69764852":[],"69826476e":[],"6984511994530214":[],"69896355":[],"698964":[],"69908626":6,"69915639":[],"69965466":26,"69981939e":[],"69985355":[],"69987647":23,"6999536":11,"69it":[],"6e75736fdab1":[],"6f7a6bd7d79f":[],"6m":[],"6n_":23,"6pm":26,"7":[0,1,2,3,4,5,6,7,8,9,11,12,13,14,15,20,21,22,23,25,26,27,29],"70":[1,6,7,9],"70037324e":27,"70127680":[],"701370":5,"70179437":[],"70205195":[],"70208087e":[],"70233292":[],"70244042":[],"70311568":[],"70354373":[],"703716d317a7":[],"70421972e":[],"70434005":[],"70465373e":[],"70473799e":[],"70482138":[],"7050":[],"70501534":[],"705698":[],"70569833":[],"70598996":[],"70653767":4,"70710678":[5,27],"70721787":[],"707218":[],"70727837":[],"70761641":20,"70769586":[],"70827971e":[],"70831425":[],"70832814":5,"70850263":20,"70856363":26,"7086067479626619":27,"70911476":26,"709698":26,"70980493":[],"70it":[],"71":1,"71075825e":[],"71131626":[],"71135895":20,"71142161":27,"711422":27,"7119":9,"712018":[11,27],"71287025":[],"713163":26,"7133574":20,"71335815":23,"7134":[],"713487":[],"71348713":[],"71350226":[],"71441535":[],"7151":[],"71721168":[],"71737253":27,"71760245":[],"71761101":27,"718165":5,"71822416":[],"71846708":26,"718697":[],"71869727":[],"71921767e":[],"72":[],"72057967":[],"72097126":[],"72101205":[],"72117282":11,"721173":11,"72174172":11,"72199579e":[],"72201591":[],"722047011333792":[],"7221662":[],"72228205":26,"7225597":[],"72261919":[],"72271878e":6,"72299619":26,"72347283":[],"7236674":5,"72373129":[],"72394787":[],"724":3,"72509099":[],"72522848":[],"72567956":[],"725680":[],"72594302":[],"7265799":[],"72671218":[],"72855434":[],"72859758":5,"72904541e":[],"7293182":[],"72981762":8,"72986500e":[],"73":[6,29],"7300372":[],"73005463":[],"7306310662842432":[],"731000":[26,27],"73133982":26,"7316348":[],"73174557":[],"73185363e":[],"73287103e":[],"733096":[26,27],"73406033e":[],"734540":[],"73454039":[],"73484667":[],"73509675":20,"73565195":[],"735738558766299":[],"73573856":[],"736115211426478":[],"73628359e":[],"73667253":11,"73704762":20,"737386":[],"73738602":[],"73752910":[],"73772319":26,"7386068":[],"73947737":[],"73it":[],"74":[2,6,29],"740":9,"74081822":8,"741264":[],"74126425":[],"74199799":20,"74260356":26,"742975":[],"74297504":[],"74368436":[],"74384949":27,"743u":[],"74409239":26,"74433499":[],"7448615806559786":[],"74577306":20,"74592114e":[],"74607851":27,"7469898175164704":[],"747224":[],"74722409":[],"74829661":27,"74840212":5,"7484669886413582":[],"74935095":[],"74943449":[],"749765":[26,27],"74997869e":[],"75":[2,5,6,8,9,11,21,26,27,29],"7501749450963715":[],"750445":[26,27],"75050135":27,"7506246":26,"750u":[],"75106135":[],"75118364":[],"751275":[],"75127504":[],"751699":[11,27],"75170092":5,"7522":[],"7525":[],"75269037":27,"75320666e":[],"75350216":[],"75352505":23,"75357263":[],"753573":[],"75357959e":[],"75361646":[],"753846153846155":[],"75390337":[],"75408649":[],"75453081":[],"75460639":[],"7546383166870465":[],"75492292":[],"75505597":[],"75517445":[],"75576555":27,"75621895":[],"756232":[],"75623206":[],"75628425":[],"756352":[26,27],"75670484":[],"756705":[],"75703965":[],"757040":[],"75747652":20,"75770568":27,"758193":[],"75819326":[],"75838233":[],"75853556e":[],"75it":6,"76":[2,9,24,26],"7600498":20,"7601030735620569":[],"76060096":[],"76063234":[],"76066069":[],"763u":[],"76400155":[],"7643536":[],"765":7,"7651899":[],"76529528":[],"76598941e":[],"76674796":[],"76707201":[],"76731400e":[],"76802186":[],"76834212e":[],"768813":[],"76881337":[],"7692307692307693":9,"76923076923077":[],"76936315":5,"7694444444444445":1,"7696835316227453":[],"7697":29,"77":[2,9,24,26],"77034458":[],"770345":[],"770356":20,"7705527590466072":[],"77070756":[],"770708":[],"77085375":[],"77144798e":[],"77152076":5,"77176703":23,"7718":9,"77288474":20,"773329728649545":[],"77332973":[],"77333117e":[],"774300":[],"77494715":[],"775":[],"77533038e":[],"7754499790100834":[],"77553346e":[],"77559332":[],"77618224":[],"77627886":[],"77632220e":[],"77636e":13,"77640601":[],"77662945":[],"7767978193240488":[],"77711437":[],"77714169":8,"77754132e":[],"77756468":20,"7780841853755783":11,"77816763e":[],"7782028952":4,"77836133e":[],"77859437":[],"77865169":[],"77954956e":5,"77it":[],"78":2,"78009660e":27,"78011544":[],"78018374e":[],"78027207":20,"7803076":26,"78035464":[],"78156479e":[],"78184120e":6,"78263078":26,"78272242e":[],"78286771":[],"78329097":[],"7843182645426894":[],"78459267":[],"7846153846154":[],"785061":26,"7851533175757255":[],"78515702":23,"785157020098348":23,"78561443":[],"78596961":[],"7860324":[],"7866024":[],"78674758":[],"787349":[],"7873493":[],"78752269":[],"78941903":5,"78972773":[],"78980112":[],"78988962e":[],"78990036":[],"79":2,"79000942e":[],"79009329":[],"790818":[],"79081838":[],"79093776":[],"79106527e":[],"79111643":5,"791123":[],"79135075":[],"79145214e":[],"79190435e":[],"79224555":27,"7923086060375724":[],"79230861":[],"79285683e":[],"79313278":26,"793167":[26,27],"79345488":[],"79394867":[],"7939646":[],"79419162":[],"794282":[11,27],"79441434":[],"79449156":[],"79512696":[],"79544237":26,"79552752":[],"79559972":[],"79577619":[],"79589065":13,"79602861":[],"79700882":26,"79725436":26,"79754246":[],"797e":6,"79827506e":[],"79856831e":[],"798761":[],"79876149":[],"79934594":23,"79974359":[],"79994469":26,"79it":6,"7bsq":[],"7c394b1e8b71":[],"7d7d58":[9,10],"7f2b3a6174c2":[],"7jexgh5soo":[],"7m":[],"7odqolophta":[],"8":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,20,21,23,26,27,28,29],"80":[0,1,2,5,6,8,9,27,28,29],"800":[4,7],"800615":[],"80088524":20,"80108096":[],"80162359":[],"802796":[],"80279641":[],"80280228":[],"80315282":[],"803153":[],"80344720e":[],"80354994":6,"803804":[],"80380438":[],"80422999":[],"80460179":29,"80462258":[],"80469739":5,"8049181":[],"80515205":20,"8055555555555556":1,"80561191":[],"80609615e":6,"80609616e":6,"80638926e":[],"8064":[],"80659468":11,"80661365":[],"806614":[],"80728853":[],"80734875":[],"807349":[],"80738685":[],"8074242":[],"80747253":[],"80763641":23,"80768431":[],"80788856":26,"80842254":[],"808423":[],"80847477e":6,"80947943":[],"80961969e":[],"80994952":[],"80it":[],"81":1,"81007682":[],"8104687":[],"810479":11,"8104791":11,"81048318e":6,"81051173":[],"81065392":[],"8108619":[],"81160425":5,"81168086":[],"81187794":[],"81333804":6,"81333805":[],"81388724e":[],"814":[7,11],"81409445":20,"8143782":5,"81447915":[],"814860456269878":[],"81486046":[],"815563":[],"81592628":[],"81597834":29,"815am":[24,26],"81633628":11,"816454":[26,27],"81656508":[],"81671821":[],"81681019":[],"816847":[26,27],"81709481":20,"8175":[],"8177581":23,"81784072":20,"818059":[],"81805914":[],"81874188e":[],"81912272":[],"8192798":[],"81955926":[],"81974332":[],"82":2,"82027884":[],"82045496":[],"82073684":[],"82092612e":[],"82113506":20,"82187419":26,"8219235992494145":[],"8219236":[],"82198978":5,"82202765":27,"82250097e":[],"82253259":[],"8225325933960339":[],"82358522":[],"823585220707946":[],"82365318":20,"82404007e":[],"82454365":[],"82484316":[],"82592":[],"8260149":20,"82650876":[],"826509":[],"8265786":5,"82747653":[],"827477":[],"82750687":[],"82771329e":[],"82773778":[],"828190347382744":[],"82819035":[],"828940":[],"82894017":[],"82949718e":[],"829648415811072":[],"82964842":[],"82975422e":[],"82985482":13,"82997013e":5,"83":[2,26],"8305555555555556":1,"83146596":29,"83170827":23,"83303886":26,"83320794":[],"833208":[],"8338786":[],"834027":[],"83402742":[],"83512277":5,"83516719e":[],"83632853":[],"836874":[],"83687444":[],"83698677":[],"83723539":[],"83724637":[],"83757843":[],"83760781":[],"83858458":23,"838584583553216":23,"839818":26,"83982725":[],"83999999999999":[],"83it":6,"84":2,"84031027e":[],"84062065":[],"84150451":[],"84203538":[],"84228957e":[],"84232163":[],"842436":[26,27],"84292394":[],"842u":[],"84310736":[],"84317815":[],"84355903e":1,"8437169762110144":[],"84371698":[],"84443254e":1,"84569271":[],"84614066":[],"8461538461538461":9,"84643086":11,"846431":11,"846471":20,"84653115":[],"84696315e":[],"84708672":[],"84723978":20,"84780262":6,"84854738":[],"84858":28,"848618":[],"84861838":[],"84886051":[],"848861":[],"84923989e":6,"84924359":20,"84924834":[],"84942942":[],"84955363":11,"849554":11,"84977962":[],"84994524":5,"85":[1,2,9],"850164":5,"8503720991789538":[],"85044503":11,"85051211":[],"85065653":[],"85071429e":[],"85086629":[],"85091337":[],"85097049":[],"85102768":[],"85104809":[],"85120833":[],"85212606":[],"85263220":6,"8527777777777777":[],"85278920e":[],"85299536":[],"853241":[],"85324115":[],"853835":26,"853u":[],"8548082":[],"8557822":[],"85581869":[],"85595734":[],"85601992":5,"85630009":[],"85688243e":[],"85694362":[],"85754328":26,"85758696":[],"85785878e":[],"858":9,"858185":26,"8583333333333333":1,"85834701":[],"85843684":[],"85895314":20,"85902603":[],"85953586":[],"85959007":[],"859628021353642":5,"85979474e":[],"85982436":[],"85it":[],"86":2,"86014003e":[],"86058560e":[],"8608479":[],"860848":[],"861":[],"86117291":5,"86134827":5,"86144422":[],"86145244":11,"861676":26,"86171505":[],"8617319":20,"86181269":[],"8620652":[],"86208133":[],"86252988":6,"86265211":23,"8634474":20,"86361743":[],"86387759e":[],"8638888888888889":1,"86416637e":[],"86425056":[],"864251":[],"86478158":[],"865158":[],"86515815":[],"86546962":[],"86550074":[],"86572911":[],"86597540e":[],"86623151":[],"86630":9,"86635201e":[],"8666666666666667":1,"86810":9,"86811137":[],"86858926":[],"86879198":[],"868792":[],"86893619":[],"869":[0,26],"86905621":[],"86925797":[],"869258":[],"87":[0,9,27],"870":[0,26],"8702784034":4,"8705211":[],"87072815e":27,"87099757":[],"871":[0,26],"87131961e":[],"87136737":[],"87191952":[],"871920":[],"872":[],"8722222222222222":1,"87243817":[],"873":[0,6,26],"87327414":[],"87327422":[],"87379129":[],"87381451":5,"874":[0,26],"874951":[],"87495119":[],"875":1,"87532289e":[],"87556429":26,"8759":13,"876":6,"87602519":20,"87618586":[],"87625493":[],"87627342":29,"87647541":[],"876664":[],"87666423":[],"87667593906086":[],"87667594":[],"8768":[],"878123":26,"878297":[],"87836018":[],"87849647":[],"878891":11,"87889126":11,"87890188e":[],"87902877":[],"8792323":[],"879271":[],"87927126":[],"87927601":[],"87940752":[],"87it":[],"88":[2,13],"88001352":[],"88017651":[],"880177":[],"88031314":[],"88046261":5,"8805555555555555":1,"88063413":[],"88118231":[],"88137798":[],"88168312e":6,"881788":[],"88182591":27,"8826033":[],"88274486":[],"883":[],"88323026":27,"88335933":[],"88336879":5,"88352503e":[],"884":[],"88450476":[],"884669":27,"88477619":[],"885":[],"8850944":[],"88512489":[],"88529063e":6,"8855417382991412":[],"88554174":[],"88560514":[],"88595926":[],"886":[],"88613493":27,"8866190885623907":[],"88661909":[],"88679947":[],"887":[],"887366":[],"88736621":[],"88739247":[],"887392472315955":[],"88746795":[],"88778045":20,"88778091":[],"8879":[],"88871662":[],"8888888888888888":1,"889080":[],"88908038":[],"8894":[],"889686":[],"88981061":20,"88it":[],"89":2,"89045167":[],"89126914e":27,"8916666666666667":[],"89249454":[],"89261458":[],"89274639":[],"89288636":11,"89298533":[],"89335182":[],"893352":[],"893489":[],"89348922":[],"89396164e":[],"89410423":5,"89421873":[],"8944444444444445":1,"89482188":26,"89543736":[],"89545727":[],"89550839":[],"89563529":0,"89565264":[],"89582298":[],"89591553e":[],"8962476":[],"896248":[],"89658408":13,"8970168":20,"89778737":23,"89779302":20,"89842298":[],"89869600e":[],"89873772":[],"89876748":[],"89881026":26,"8990231":26,"8991514":[],"89944994":[],"899450":[],"899734":[],"8f":[6,29],"8g":[6,29],"8n":20,"8x8":1,"9":[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,22,23,24,26,27,28,29],"90":[1,2,6,9],"90034533":[],"90044211":11,"900449":[],"9005135872087155":[],"90054363":[],"90066122":[],"9007164":[],"90075537":5,"90092179":26,"9011":6,"90189742":[],"90220243":5,"90252116":[],"90266948":5,"90268858":[],"9027777777777778":1,"9033":[],"90342093e":[],"903824":[],"90382431":[],"90383424":[],"904":[],"9040":9,"904221":[],"90475506e":6,"90491568e":[],"9054":29,"90540529":[],"9055555555555556":1,"90559087":[],"905591":[],"90595152":[],"906298":[],"90629812":[],"90643349":[],"9067362":[],"906747":5,"90694878":[],"90699099":[],"90793026":20,"90813514":[],"9083333333333333":[],"90884627":[],"90895045":[],"90940378":[],"9096305360355206":[],"90999452e":[],"91":[24,26],"910":9,"91094169e":[],"9111111111111111":1,"91124889":[],"91128596":5,"91236018":26,"912u":[],"913":[],"91346044":[],"91379157":[],"913791573406831":[],"91383439":[],"91396388":[],"914":[],"91410101":[],"91416375":[],"91479093":[],"91492986e":6,"915":[],"91511388":[],"91549644":[],"915549":[],"916":[],"91616374":[],"916164":[],"9166666666666666":9,"91682433":[],"9169":6,"917":[],"91760278":5,"918":[],"91811160e":[],"91812702":5,"918992":[26,27],"919":[],"91936525":[],"9196398947631166":[],"91it":[],"92":[6,9,24,26],"92087142":[],"9212905":[],"921291":[],"92143477":[],"921435":[],"92169784e":[],"92173899":26,"92208477":[],"922085":[],"92250132":26,"92272314":[],"92275042":[],"922u":[],"9230769230769231":9,"92322465e":[],"92337385e":[],"92343595":[],"923602":26,"92373266":[],"924018":[],"92461411":[],"92463154e":[],"92472578":11,"924726":11,"92477093":[],"924e":6,"925":1,"92507116e":1,"9252772":[],"92578916":5,"92603747":[],"92626212":[],"92631966":[],"926320":[],"92651068":29,"926583":[],"92658312":[],"92738529e":[],"92754397":[],"927544":[],"9277777777777778":1,"92811987":[],"92817032":[],"92819235":[],"92857143":7,"92863147":[],"92888885":[],"92916438e":[],"9292":[],"92921648":[],"929612":5,"92968793":[],"92995253e":[],"92it":[],"93":[],"93023603":[],"93034176e":[],"93037171":[],"930372":[],"9305555555555556":1,"930583":[],"930683":[],"93082503":[],"931":[0,26],"93155188":5,"93158979":5,"93166184":26,"93188452":[],"931885":[],"93229640e":[],"93248252":[],"932483":[],"93265898":20,"932u":[],"933":[5,28],"93325497":[],"93371943e":[],"93386901":[],"93414191":[],"93492130e":6,"93503":11,"93518534":[],"93522293":[],"93579127":[],"93586895":[],"935u":[],"93601008e":27,"9361111111111111":[],"936762":[],"93676229":[],"936856":[],"93685631":[],"937":23,"937082":[26,27],"93725291":5,"93792215e":[],"93799826":5,"938":23,"9383327":[],"9387":9,"939":[0,23,26],"93969474":20,"93it":6,"94":[7,23],"9400":[],"94015598":13,"94054854":[],"940549":[],"94073014":20,"94076632":[],"940776":[],"94077605":[],"94107596":[],"9410965":23,"94117118":[],"941866404575299":[],"94212937":[],"94226022e":6,"94227931":[],"9424":[],"9426":29,"94271589":[],"94273542":27,"94284104":5,"94320205":5,"94321297":[],"94390673":23,"94404545":20,"9440887":[],"94413158":[],"9444444444444444":1,"945":[],"94506946":23,"94548496":[],"94591015":[20,26],"946":[],"94639099":11,"946393":[],"94640767":[],"94659383":[],"946957":5,"947":[],"9472222222222222":1,"94735055":[],"9474044318317307":[],"947903":[],"94790323":[],"948":[],"94822514":6,"9482527":5,"94861749e":[],"948641":[],"94867516":[],"948675162607389":[],"949":[],"94905663":[],"949087":[],"949107":11,"94910711":11,"94915262":[],"94917559e":[],"94925462":13,"9492866":[],"94986593":[],"95":[1,7,9,11,29],"95008046":6,"95011994e":[],"95054046":[],"95079224":[],"95079764":[],"95136998":[],"9518":6,"95231424":5,"952387":27,"9524":[],"9527777777777777":1,"95284275":5,"95302":[],"953065564":1,"95324615":[],"95351665":5,"9538239":[],"954":23,"9543538503067644e":9,"95446837":[],"9549351910143222":[],"954988":[],"954u":[],"95511792":27,"955118":27,"95539206":26,"9555555555555556":1,"95569422":[],"955820c21e8b":4,"95594445e":[],"95604667":20,"95645365":[],"956454":[],"956563":[11,27],"95684892":5,"95703":13,"95714723":[],"957147232685324":[],"957421":[],"95742107":[],"9577775794806141":[],"958194":[],"95819409":[],"958228616652075":[],"9582286166520774":5,"95824108":[],"958476":[],"95911004":[],"95927495":[],"9594":[],"959533":[],"95953339":[],"95976303":27,"95994091":[],"95it":[],"96":[6,7,11,29],"960":23,"9601304850018328e":[],"960130485007504e":6,"960130485007934e":[],"9601304850213484e":[],"96013048502692e":6,"96024953":5,"96032148":[],"96081861":[],"96084663":5,"961":23,"9618":6,"962":23,"96251494":[],"96257580e":[],"96261297":[],"9626883":[],"962990":26,"9631064":[],"96317459":[],"9635449873404844":[],"9637117593816477":6,"96390357":[],"9640435":5,"964268":[],"96426825":[],"96431281":[],"964735":[],"96473528":[],"96489434":[],"9649652536":4,"96496641":[],"96504592":[],"96514849":[],"96527903":[],"96534137":[],"96542994e":[],"96551427e":[],"96552318":[],"965548":[26,27],"96562567":20,"965838":11,"96583833":11,"96601782":[],"96606158":[],"96620033":[],"96631321":27,"96653373":[],"96667129":[],"96670977":[],"96686324e":[],"96688672":5,"966899":26,"96735081e":[],"9674916":5,"967536":[],"9675364":[],"96769675":[],"9678":[],"967809":[11,27],"96804878":23,"96810393":[],"96812218":[],"96863851":[],"96941962":[],"9698377974429513":[],"96987657":[],"96992454":[],"96995249":[],"96it":[],"97":7,"97005689":5,"970057":[],"970477":[],"97047717":[],"97065296":[],"97087408":13,"97108e":13,"97117751":[],"971688":[],"97186457":11,"971865":11,"97200305e":[],"97202":[],"9722222222222222":1,"972286":[],"97228638":[],"97230501":[],"97243128":5,"972643":5,"97264311":5,"97300836":5,"97305528e":[],"97409408":23,"9740940821517547":23,"97421755":5,"974218":5,"97434186":[],"97467734e":[],"97477238":[],"97497404e":6,"975":1,"97507735":5,"97509292":[],"97533203":[],"97547354":[],"97547354476579":[],"975510299261579":[],"97558913":26,"9760832":[],"97656972":[],"97665553e":[],"97690235":29,"97705827":5,"97708897":[],"977089":[],"97713364":[],"97731700e":[],"977418":[],"97758848":5,"9777777777777777":1,"977880":[],"97788031":[],"978":28,"9780387310732":25,"9780387848570":25,"9781492032632":25,"97840267e":[],"97840468":[],"978553":5,"97898392":[6,27],"97906022e":27,"97926491":5,"97926642":[],"97936584":20,"97952271":[],"97955570e":[],"97980332":[],"97it":[],"98":[0,1,7,9],"980":9,"98005862e":[],"98017611":[],"98018358":[],"98036405":[],"9805555555555555":1,"98060869":[],"98091621":5,"98129668":[],"981321":[26,27],"98139097":5,"98153145":[],"98248312":[],"98248681":5,"982487":5,"98249059":[],"982491":[],"98266587":[],"98275501":5,"982758":[],"98275836":[],"98316352":[],"983164":[],"98328759":[],"983310":[26,27],"98346748":[],"9835443722554817":[],"98399675":[],"98400474":[],"98413059":5,"984525":[],"98453971":[],"98454786":5,"98467494":[],"98496212":26,"985":23,"98566191":5,"98581925":[],"9859105":26,"986":23,"98604688e":[],"9861111111111112":1,"986699":5,"98680716":5,"98699753":[],"98716878":5,"98740124":[],"98756882":[],"98794823":[],"9879924":[],"98808176":5,"98822371":6,"98842948e":[],"98880065e":[],"9888888888888889":1,"989":23,"9890348":5,"98919197":13,"9893149172528393":[],"9893405":[],"9893447":5,"989393825676568":11,"98947894":27,"98950411":[],"989645675273948":27,"9898254753574576":[],"98982548":[],"9898ff":[9,10],"99":[6,7,9,11,13,29],"990":9,"99009525":5,"9902552771282336":[],"99035178":26,"99049330":[],"99051150":6,"9906601091977738":[],"99088801":5,"9909606":[],"991":23,"99115119":5,"99157584":[],"99176998":5,"99180856e":[],"992":23,"9920084":26,"9920099299023244":[],"99218609":20,"99242921":5,"992504":[],"99265097":5,"99274513":[],"993":23,"99316252":5,"99353454":[],"993535":[],"993537":[],"99353748":[],"99371056":5,"99374388":[],"993865":[],"99389612":5,"99397049":[],"993972":[],"99397245":[],"9940672992288855":[],"9943201":5,"99435648":[],"9947613223847728":27,"9947756":5,"99492986":5,"9949347794612255":[],"99498108":[],"99519225":[],"9952495547161103":[],"99528218":5,"99539415":5,"99541140030306":[],"9954205504035075":[],"9954911579937146":[],"9955282554647219":[],"9956582":[],"99566069":5,"9957763445154583":[],"99578809":5,"9959033816551833":[],"9959293935368551":[],"99594988":[],"995950":[],"9959898232423614":[],"996":[5,28,29],"9960039497353536":[],"99608161":5,"9960913291755783":[],"9962074631757628":[],"9963202597920238":[],"9963525271588923":[],"9963864072152893":[],"9963961":5,"99650061":5,"9965048209369255":[],"996535469511469":27,"9966080206050006":[],"996694948530336":[],"9967458":5,"9967912002709148":[],"99700706":5,"99709215":5,"99724883":[],"99729756":5,"9973102337301051":[],"99751458":5,"99758326":5,"99760425":13,"99775587":5,"99775949":[],"997830":11,"99783004":11,"99793613":5,"99799099":5,"99813653":5,"99820454":[],"99828624":5,"99829953":[],"9983295":5,"99845267":5,"99854557":27,"998577":5,"99861053":5,"99862019":27,"99869482":27,"99871521":5,"99876945":27,"99881845":5,"99883628":27,"99884384":5,"99884409":27,"99891093":27,"99891873":27,"99893323":5,"99898558":27,"999":[9,23],"99901131e":[],"99901896":5,"99903755":5,"99906023":27,"99911427":5,"99912709":27,"99913489":27,"99918546":5,"99919837":5,"99920175":27,"99926459":5,"99927642":27,"9993237":5,"99933188":5,"9993511":27,"99938942":5,"99941797":27,"9994385":5,"99944272":5,"99948754e":[],"99949266":27,"99949306":5,"99952195":23,"99953381":5,"99953475":5,"99956735":27,"99957911":5,"99961294":5,"99965056":5,"99967865":5,"99970894":27,"99970988":5,"9997332":5,"99975913":5,"99977416":[],"99977849":5,"99978365":27,"99980002":5,"9998161":5,"99984732":5,"99987324":5,"99989476":5,"99991263":5,"99992698":[],"999927":[],"99992746":5,"99993978":5,"99995":5,"9999555851685968":[],"999955585168597":6,"9999840939906267":[],"9999858320366368":[],"9qcu8vcxxru":[],"9x":[6,21],"9y":[6,21],"\u00f8yvind":[6,27],"abstract":1,"boolean":4,"break":[0,4,6,11,14,26],"byte":[20,26],"case":[0,1,2,3,4,5,6,7,11,12,13,14,16,19,20,21,26,29],"catch":[0,26],"char":[],"class":[0,1,3,4,6,7,8,9,11,12,13,23,26,29],"const":[],"default":[0,1,2,4,6,7,13,16,20,21,26,27],"do":[0,2,3,4,5,6,8,9,10,11,12,13,14,15,16,20,21,27],"ekstr\u00f8m":4,"eng\u00f8i":[],"enum":[3,4,14],"export":9,"f\u00f8470":[24,26],"final":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,16,18,21,22,23,24,26,29],"float":[0,3,4,5,9,11,13,14,20,26,27],"function":[2,3,4,5,9,14,15,16,17,18,19,20],"import":[0,1,2,3,4,6,7,8,9,10,11,12,13,14,16,17,21,23,28,29],"int":[0,1,2,3,4,5,6,11,13,14,20,23,27,29],"long":[0,1,3,4,12,13,26],"m\u00f8svatn":[6,21],"new":[0,1,2,3,5,6,7,8,9,10,11,13,14,17,20,21,26,27],"null":26,"public":[0,3,4,14,19,26],"return":[0,1,2,3,4,5,6,7,8,9,11,13,14,20,21,23,26,27,28,29],"s\u00f8rli":[],"s\u00f8rlie":[],"sch\u00f8yen":[6,27],"short":[4,5,21,28,29],"steinsv\u00e5g":[],"super":[5,27,28],"switch":0,"throw":[3,6,23,29],"true":[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,20,21,23,26,27,29],"try":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,19,20,21,23,26,27,28,29],"var":[1,5,6,10,11,13,18,21,23,26,27,28,29],"while":[0,1,3,4,5,6,7,8,9,11,12,13,23,26,27,28,29],A:[2,3,5,6,7,10,11,12,13,16,18,19,20,21,22,23,24,25,27],AND:2,And:[0,3,4,5,6,9,13,19,21,23],As:[0,1,2,3,4,5,6,8,10,12,13,16,20,21,23,26,27,28,29],At:[0,4,6,13,21,26],BE:[0,26],Be:[2,19,26],Being:13,But:[0,1,2,3,5,6,9,10,23,27,28,29],By:[0,3,5,6,12,13,17,20,26,27,28,29],For:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,25,26,27,28,29],IF:6,IN:25,If:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,15,16,19,20,21,23,26,27,28,29],In:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,25,26,27,28,29],Is:11,Ising:[5,12,27],It:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,23,26,27,28,29],Its:[1,2,4,11],NO:[7,11],No:[6,9,26],Not:[0,1,5,6,26,27,29],OR:23,Of:23,On:[0,3,15,22,23,24,25,26],One:[0,1,3,4,5,6,7,8,11,12,13,17,21,23,27,29],Or:[0,1,6,21,26],Such:[0,6,12,16,23,29],That:[0,5,7,10,11,12,14,21,23,26,28,29],The:[4,10,13,14,15,16,18,20,21,22,23,24,25],Then:[0,1,6,8,9,10,11,12,13,14,20,21,26,27,29],There:[0,3,4,5,6,8,9,11,12,14,20,21,22,23,24,26,27,28],These:[0,3,4,5,8,9,10,11,12,13,14,15,16,20,21,23,24,26,27,28],To:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,23,28,29],With:[0,5,6,8,9,10,11,12,14,18,20,21,23,26,27,29],_0:[5,8,10,11,13,27],_1:[2,5,6,8,10,11,12,13,14,20,27],_2:[2,5,8,11,12,13,20,27],_3:20,_4:20,_9:13,_:[0,1,2,4,5,6,7,8,9,10,11,12,13,17,18,20,21,26,27,28,29],_________________________________________________________________:[],__call__:[],__class__:10,__doc__:[6,29],__future__:[8,9],__getattr__:28,__getitem__:[],__init__:[1,2,3,4,14,28],__main__:2,__mosek:[],__mul__:[],__name__:[2,10,28],__rmul__:[],__traceback__:[],__version__:[3,4,14],_api:[3,4,14],_asarrai:[],_auto10:[6,12],_auto11:6,_auto12:6,_auto1:[2,3,4,5,6,7,12,13,20,23,27],_auto2:[2,3,4,5,6,12,13,20,23],_auto3:[3,4,5,6,12,13,20],_auto4:[4,6,12,13,20],_auto5:[4,6,12,13,20],_auto6:[4,6,12,20],_auto7:[4,6,12,20],_auto8:[6,12],_auto9:[6,12],_ax:[],_base:8,_broadcasting_shape_rul:[3,4,14],_build:[0,17,19,21,25,26],_build_call_output:[],_c:1,_call:[],_call_flat:[],_check_1d:[],_check_optimize_result:[7,11],_compon:11,_config_pb2:[],_coordinate_desc:6,_copy_docstring_and_deprec:[],_cpgg0jyh8m:[],_decor:[0,26],_delta:[3,4,14],_depth:9,_dilate_shap:[3,4,14],_ey:[3,4,14],_fraction:9,_get_lin:[],_getitem_multilevel:[],_handl:[],_i:[0,1,2,5,6,7,8,11,12,13,15,16,17,21,26,27,28,29],_inference_funct:[],_interpolatefunctionerror:[],_is_primit:[],_j:[0,1,2,3,5,6,8,13,17,21,27,28,29],_jit_compil:[],_k:13,_l:12,_lambda:[6,26],_leaf:9,_logist:[7,11],_m:10,_make_vjp:[2,13],_maybe_define_funct:[],_multilayer_perceptron:1,_n:[2,5,8,11,13,27],_node:[2,9,13],_not_impl:[3,4,14],_notokstatusexcept:[],_np:2,_num_output:[],_one:[3,4,14],_p:[5,8,27],_plot_arg:[],_process_traceback_fram:[],_r:[],_ratio:11,_reduce_and:[3,4,14],_reduce_max:[3,4,14],_reduce_min:[3,4,14],_reduce_or:[3,4,14],_reduce_sum:[3,4,14],_reduce_window_max:[3,4,14],_reduce_window_min:[3,4,14],_reduce_window_prod:[3,4,14],_reduce_window_sum:[3,4,14],_sampl:9,_select_forward_and_backward_funct:[],_split:[6,9,21],_src:[3,4,14],_stateful_fn:[],_stateless_fn:[],_sy:[3,4,14],_t:13,_test:[6,21],_trace:[2,13],_tri:[3,4,14],_upcast_fp16_for_comput:[3,4,14],_valu:[2,13],_varianc:11,_weight:9,_xla_comput:[3,4,14],_zero:[3,4,14],a0:3,a0faa0:[9,10],a1:[0,26],a2:[0,26],a3:[0,26],a4:[0,26],a_0:[0,26],a_1a:[0,26],a_2a:[0,26],a_3:[0,26],a_3a:[0,26],a_4:[0,26],a_4a:[0,26],a_:[0,1,16,20,26,27],a_h:1,a_i:[0,1,2,12,26],a_j:[1,12],a_k:[0,1,12],a_ndim:2,aaron:25,ab:[0,2,5,13,14,26,27,28],ab_channel:19,abandon:1,abbrevi:22,abid:23,abil:[0,10,26],abl:[0,1,4,5,6,7,10,12,13,16,21,27],abort:[],about:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,19,20,21,24,28,29],abov:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,25,26,27,29],abovement:[6,29],abscissa:13,absolut:[0,2,5,6,13,26,27,28,29],absorb:[27,28],acceler:13,accept:[0,3,6,9,21,26,27],access:[0,3,11,23,27],accid:[4,6,29],accident:[3,4,14],accmod:[],accompani:[0,26,27],accomplish:[8,9,13],accord:[0,1,2,5,6,9,12,13,14,15,16,23,26,28,29],accordingli:11,account:[0,3,5,13,15,23,26,28,29],accumul:[12,13,23],accur:[0,3,4,6,10,13,29],accuraci:[0,1,3,4,5,6,7,9,10,11,12,26,27],accuracy_scor:[0,1,10,26],accuracy_score_numpi:1,achiev:[0,1,5,6,8,12,20,26,28,29],aco:23,acquaint:[19,26],acquir:[1,19,26],acr:[0,27],across:[1,3,6,9,19,26,29],act:[1,3,20],action:23,activ:[0,2,3,4,9,22,24,26,29],actual:[0,1,4,5,6,8,11,16,20,23,26,27,28,29],ad:[1,3,4,5,8,13,14,15,16,20,28,29],ada_clf:10,adaboostclassifi:10,adadelta:13,adagrad:[],adam:[1,3,4,24,26],adapt:[4,6,13,25,27,29],add:[0,1,2,3,4,5,6,8,10,11,12,15,16,17,21,23,26,27,28,29],add_lin:[],add_outgrad:2,add_subplot:[1,7,12,14],addendum:5,addit:[0,2,3,5,6,7,8,9,10,12,13,15,17,19,20,21,23,24,25,26,27,28,29],addition:[12,13],address:[1,9,11,13,26],adjac:[3,12],adjoint:[5,27,28],adjust:[0,5,12,13],admir:[0,26],advanc:[4,6,12,25,26,29],advantag:[1,3,5,6,10,13,20,28,29],adversari:26,afecionado:26,affect:3,affin:[0,3,8,11,27],afford:3,aficionado:26,aforement:14,african:[0,27],after:[0,1,2,4,5,6,9,11,12,13,15,19,20,21,23,26,27,28,29],afterward:[0,26],ag:[0,7,22,26,27],ag_0:2,again:[0,1,4,5,6,7,8,10,11,12,13,15,16,21,23,26,27,28,29],against:[1,4,7,10],agegroup:7,agegroupmean:7,aggreg:[9,10],agorithm:10,agre:[5,6,23,28,29],agreement:13,ahead:9,ai:[0,25],aid:11,aim:[0,1,4,6,7,11,14,15,16,19,20,21,27,29],ainv:5,airplan:3,aka:[5,28,29],al:[0,2,4,15,16,17,25,26,27,28,29],alarm:[5,7,28,29],algebra:[0,3,5,13,19,27,28,29],algo:[],algorithm:[0,1,2,4,5,6,7,8,13,14,19,20,21,23,25,26,28,29],align:[0,2,5,6,7,8,13,21,23,26,27,28,29],all:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,15,19,20,21,22,23,24,25,26,27,28,29],all_gath:[3,4,14],all_to_al:[3,4,14],allevi:[1,13],alloc:[3,20],allow:[0,1,2,3,5,6,8,10,13,15,16,19,20,21,26,27,28,29],almost:[0,1,6,8,11,13,23,26,29],alon:[2,9],along:[2,3,4,5,6,9,10,11,19,20,26,27,28,29],alpha:[0,1,2,3,4,6,7,8,9,10,13,14,23,26,27,29],alpha_0:3,alpha_1:3,alpha_2:3,alpha_:10,alpha_i:[3,13],alpha_k:13,alpha_m:10,alpha_n:3,alpha_opt:13,alreadi:[2,3,4,5,6,10,12,19,20,23,26,27,28,29],also:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,26,27,28,29],alter:1,altern:[0,1,4,5,6,7,8,9,11,13,20,21,26,27,28,29],although:[0,1,5,6,8,10,13,16,26,28,29],alwai:[0,3,5,6,12,13,16,23,26,27,28,29],am:[4,27],ame2016:[0,26],american:[0,27],among:[0,3,5,9,10,12,20,26,27,28],amongst:[5,28,29],amount:[0,1,3,4,6,8,10,14,19,29],an:[1,2,3,5,6,7,8,9,11,12,13,14,15,16,17,19,20,21,23,24,25,27,28,29],an_:23,anaconda3:[],anaconda:[0,1,15,19,21,26],analog:13,analys:[6,28,29],analysi:[1,3,4,7,14,15,16,17,20,25],analyt:[2,3,5,6,7,12,13,15,19,21,26,27,28,29],analyz:[0,1,3,4,5,6,14,16,17,21,23,26,27,28],andrew:1,angl:[0,3,9,27],anharmon:3,ani:[0,1,2,3,4,5,6,7,8,9,10,12,14,18,23,26,27,28,29],anim:[4,12],ann:12,annot:[0,1,3,7,8,26,27],announc:26,anoth:[0,1,3,4,5,6,7,8,10,11,12,13,20,21,23,26,27],anp:2,ans_vspac:2,ansatz:[0,26],answer:[0,1,3,5,6,20,21,24,26,28,29],antialias:[2,6,21],anticip:4,anymor:[1,8],anyon:[4,8],anyth:[1,23],anytim:[24,26],apach:[1,3,4,14],apart:[11,13],api:[1,3,4,14,19,26],appar:2,appear:[0,1,3,13,16,20,23,26],append:[1,3,4,8,9,13,26],appendcon:[],appendix:21,appendvar:[],appli:[0,1,2,3,4,6,7,8,9,10,11,12,13,15,16,21,23,25,26,27,29],applic:[0,1,3,4,5,6,7,9,12,13,20,21,23,25,26,27,29],apply_gradi:4,approach:[1,2,4,5,6,9,10,11,12,13,17,19,23,25,27],appropri:[2,6,9,12,13,19,23,29],approv:26,approx:[0,2,3,6,10,11,13,23,26,29],approxim:[0,1,2,3,4,5,6,7,10,11,13,18,21,23,26,27,28,29],apt:[0,15,19,21,26],aq:23,ar:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,22,23,24,25,26,27,28,29],aragorn:26,arang:[1,3,4,6,7,9,10,12,13,21,26],arbitrari:[1,4,6,8,12,13,23,29],arbitrarili:[0,1,11,26],arc:[6,21],architectur:[3,4,12],area:[0,3,6,9,21,25,26],arg:[0,2,13,26],argc:[],argmax:[1,11],argmin:[4,10,14],argnum:[2,13],argnum_0:2,argnum_1:2,argsort:11,argu:[1,13],argument:[0,2,3,5,6,11,12,13,21,26,27,28,29],argv:[],argval:2,aris:[0,6,12,13,23,26,29],arithmet:[0,13,20,26],arm:[3,4,6,14],arma:[],armadillo:20,around:[0,1,4,5,6,11,23,26,28,29],arr:2,arrai:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,19,21,23,27,28,29],arrang:[3,26],arraybox:13,arriv:[0,6,9,11,20,23,26,29],arrow:12,arrowprop:8,art3d:13,art:[0,1,15,19,26],articl:[0,3,4,6,10,18,26,27,28,29],artifici:[0,2,7,12,25,26],artificialneuron:12,arug:13,arxiv:[3,4],as_fram:27,asap:26,asarrai:[0,2,6,9,27],asc:[],ascii:[],ashrafi:[],asid:27,ask:[5,6,11,12,28,29],aspect:[0,6,19,21,26,27],assembl:[0,3,26],assert:4,assess:[0,6,21,26,27,29],assici:4,assign:[0,7,8,9,12,13,14,22,24,25,26,27],associ:[0,6,9,12,14,23,26,29],associative_scan:[3,4,14],assum:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],assumpt:[0,3,5,6,9,11,18,21,23,26,27],ast:[0,5,6,18,21,26,28,29],astronomi:[26,27,28,29],astyp:[4,9,10],asymmetri:[0,26],asymptot:[4,6,29],async_wait:[],athbf:[],atla:26,atoi:[],atom:[0,26],attempt:[0,4,6,7,8,10,26],attend:22,attent:[0,20,26],attr:28,attract:[0,10,26],attribut:[0,3,4,9,13,14,26,28,29],attributeerror:[3,4,13,14,28],audi:[0,26],audio:[3,4],august:[15,16,26],aurelien:[0,15,22,25,26],austfjel:[6,21],author:[0,1,3,4,10,14,23,26],authour:26,auto:[6,9,10,23],autocor:23,autocorrelation_tim:23,autocorrelform:23,autocovari:23,autoencod:[4,19,26],autoencond:19,autograd:[19,26],autograph:[3,4,14],autom:[0,19,25,26],automac:20,automag:26,automat:[0,1,2,3,4,11,16,19,20,26],automobil:3,autonom:4,avail:[0,1,4,6,10,11,15,19,20,21,22,24,25,26,29],averag:[0,1,3,6,9,10,13,14,23,24,26,27,29],avg:[],avoid:[0,3,4,5,6,9,11,13,14,20,26,27,29],avx2:[],avx:[],awai:[2,3,6,27],awar:[2,10],award:[24,26],ax:[0,1,2,3,4,6,7,8,9,10,11,12,13,14,20,21,26,27,29],axes3d:[2,6,13,21],axes_grid1:6,axessubplot:27,axhlin:8,axi:[0,1,2,3,4,6,7,8,9,10,11,12,13,14,20,21,23,26,27,29],axiom:[5,28,29],axlabel:[0,26],axvlin:[4,8],axvspan:4,b1:8,b2:8,b3:8,b:[0,1,2,3,4,5,6,8,9,10,12,13,14,15,16,23,24,26,27,28,29],b_0:0,b_1:[0,2,12,13],b_2:[0,13],b_5:13,b_:[0,1,20],b_group:9,b_i:[0,1,2,12,26],b_ia_:[0,26],b_ia_i:0,b_index:9,b_is_vec:2,b_j:[1,12],b_k:[0,1,12,13],b_m:12,b_meta:2,b_score:9,b_valu:9,bachelor:[22,24],back:[0,3,4,5,6,8,9,10,15,20,23,26,27,28],backbon:20,backend:[1,4],backend_specific_transl:[3,4,14],background:[25,26,27],backpropag:1,backtrack:9,backup:20,backward:[1,2,4,12,20],backward_pass:2,bad:[6,27],badli:23,bag:[9,19,26],bag_clf:10,baggin:26,baggingboot:10,baggingclassifi:10,baggingtre:10,balanc:[6,29],band:20,bandwidth:20,bar:[0,6,11,15,16,21,26],barber:25,bare:[4,10],base:[0,1,3,4,5,7,8,9,10,14,19,23,24,25,26,27,28],basi:[5,7,8,10,11,12,13,20,27,28],basic:[2,6,8,12,13,14,19,21,23,26],batch:[3,4,11,12,13,14],batch_shap:4,batch_siz:[1,3,4],batchnorm:4,bay:7,bayesian:[5,19,25,26,28,29],bbupeevmxtg:[],bc298b802fe2:[],beam:[26,29],becattini:24,becaus:[0,1,2,3,4,5,6,8,9,12,13,14,26,28,29],beccatini:26,becom:[0,1,2,5,6,7,9,12,13,23,26,27,28,29],been:[0,1,2,3,4,5,6,11,12,13,19,20,21,26,27,29],befor:[0,1,2,3,4,5,6,7,8,12,13,14,20,21,23,26,27,28,29],beforehand:[0,23,26],begin:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,21,23,24,26,27,28,29],behav:[1,6,13,29],behavior:[0,1,13,26],behaviour:12,behind:[0,1,6,8,13,26],behnoosh:[],being:[0,1,2,3,4,5,7,8,10,11,12,13,17,23,26,27,28,29],believ:[9,20],belong:[7,8,9,13,14],below:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],benchmark:10,bendik:[],benefici:[1,13],benefit:[0,1,4,11,13,15,19,26],bengio:[1,22,25,26,27],benign:[1,7,9],bennosh:[],besid:[4,5,28],bessel:[5,27,28,29],best:[0,1,2,3,4,5,6,7,8,9,10,12,13,21,24,26,27,29],beta:[0,1,3,5,6,7,10,11,13,16,17,18,21,26,27],beta_0:[0,1,3,5,6,7,13,26,27,28],beta_0x_:[0,26,27],beta_1:[0,1,3,5,6,7,10,13,26,27,28],beta_1x_0:[0,26],beta_1x_1:[0,7,26],beta_1x_2:[0,26],beta_1x_:[0,26,27],beta_1x_i:[7,13,27],beta_2:[0,3,13,26,27],beta_2x_0:[0,26],beta_2x_1:[0,26],beta_2x_2:[0,7,26],beta_2x_:[0,26,27],beta_2x_i:27,beta_3:3,beta_3x_i:27,beta_4x_i:27,beta_:[0,3,6,7,13,26,27],beta_i:[0,3,5,17,26,27,28],beta_j:[0,5,6,13,18,21,26,27,28,29],beta_k:13,beta_linreg:13,beta_m:10,beta_mg_m:10,beta_n:3,beta_p:7,beta_px_p:7,betavalu:5,better:[0,1,2,3,4,6,9,10,11,12,13,26,27,28,29],between:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,17,21,23,26,27,29],beyond:[0,1,5,6,8,13,21,26,27],bf:[13,14,20,23],bgd:13,bia:[0,1,2,3,5,8,9,10,12,13,26,27,28],bias:[1,2,3,5,6,9,12,28],big:[0,1,2,5,6,14,18,26,28,29],bigger:[1,6,27],bigr:12,bike:9,bilbo:26,bilek:[],billion:[3,12,19],bin:[0,7,23,27],binari:[0,3,5,7,9,10,12,26,28,29],binarycrossentropi:4,bind:[0,27],binomi:[19,23,26],binsboot:[6,29],bioinformat:[0,26],biolog:[1,12],bios1100:[19,26],bird:[0,3,26],birth:26,bishop:[22,25,26],bit:[1,4,20,23,26],bitwis:[3,4,14,23],bivari:2,bk:[0,13,27],bla:[20,26],black:[8,9,14],bledso:26,blob:[21,26],block:[6,10,19,20,23,26,29],blockingavg:[],blockingstd:[],blockingvar:[],blocksiz:[],blocksizemax:[],blocksizemin:[],blogpost:4,blue:[0,3],bluntli:26,bm:27,bmatrix:[0,1,3,5,7,8,11,13,20,26,27,28],bmi:1,bodi:[0,1,4,12,26],bold:1,boldfac:[0,5,16,27,28],boldsymbol:[0,1,2,3,5,6,7,8,10,11,13,14,15,16,17,18,21,26],boltzmann:[12,19,26],book1:25,book:[21,25,26,28],boost:[1,9,19,26],boostrap:10,bootavg:[],bootstd:[],bootstrap:[1,13,19,21,26],bootvar:[],bootvec:[],borrow:26,boston_dataset:[0,27],bot:8,both:[0,1,4,5,6,8,9,10,13,14,15,16,19,20,21,23,24,26,27,29],bottl:7,bound:[0,8,12,27],boundari:[2,4,8,11,12],boundkei:[],box:[4,9],boxed_arg:2,boyd:[8,13],bracket:[4,23],brain:[1,7,12],branch:9,breast:[5,7,11,28,29],breviti:13,brew:[0,15,19,21,26],brg:8,brief:[21,27],briefli:[0,26],bring:[0,5,6,10,26,27],broad:[0,26],brought:[13,19,26],brownle:4,browser:26,brunton:[],brute:[3,5,11,27,28],bs:[8,9,10],buffer_s:4,bui:4,build:[0,4,5,6,10,20,23,26,28,29],built:[0,1,3,4,6,27,29],bunch:11,busi:[0,27],bx:[],bzl:[],c1:[8,11],c2:[8,11],c95af3df0cdd:[],c:[0,1,2,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,23,24,25,27,28,29],c_0:23,c_1:12,c_2:12,c_3:12,c_4:12,c_:[0,8,9,10,13,23,27],c_i:[12,13],c_k:23,ca:[1,26],cabc613b8702:[],cach:10,cal:[0,8,10,12,13,15,16,26],calcul:[0,1,2,4,5,6,8,9,10,11,12,13,14,15,16,18,20,23,26,28,29],california:27,call:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,24,26,27,28,29],callabl:[],callback:[],calor:[0,27],cambridg:[13,25],can:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,25,27,28],cancel:[0,13,26,27],cancellation_manag:[],cancer:[5,10,28,29],cancerpd:7,candid:[8,9,10],cannot:[0,1,4,5,6,7,8,9,22,23,26,27],canopi:[0,15,19,21,26],canva:[21,26],cap:[5,28,29],capabl:[0,1,8,13,19,26],capac:[2,24],capita:[0,27],caption:21,captur:[4,11,12],captured_input:[],car:[3,4],card:[0,7,26],cardin:1,care:11,carefulli:13,carlo:[0,6,19,23,25,26,29],carri:[2,6,7,21,29],cart:10,carvalho:[],casella:25,cast:1,cat:[3,4],categor:[0,1,3,9,11,26],categori:[0,1,3,7,10,12,14,26,27],categorical_crossentropi:[1,3],caus:[0,5,6,23,26,27,28,29],causal:0,causat:[0,26],cax:1,cb:[6,26],cbar:1,cbook:[],cc:[0,1,5,13,26,27,28,29],ccc:[5,12,28],cd_fast:6,cdf:23,cdot:[0,2,6,12,13,14,20,23,26,29],celebr:13,cell:[0,2,3,4,6,9,10,13,14,15,21,26,28],center:[0,1,6,7,8,9,11,14,21,23,26,29],centr:[],central:[0,3,5,6,8,16,20,26,27,28],centroid:[14,23],centroid_differ:14,centuri:3,certain:[0,3,6,7,9,23,26,27,29],certainti:29,cg:13,cha:[0,27],chain:[0,1,13,19,23,26],chanc:[1,5,13,23,28,29],chang:[0,1,2,3,4,5,6,8,9,11,12,13,14,20,21,23,26,27,29],channel:3,chapter3:[0,21],chapter:[0,6,10,11,18,20,21,25,26,27,28,29],charact:[0,3,5,8,26,27],character:[8,9,10,12,23],characterist:[0,1,3,10,13,26],charg:[0,26],charl:[0,27],chase:4,chat:26,chd:7,chddata:7,cheap:[5,27],cheaper:[1,13],check:[0,1,3,4,5,6,11,13,20,26,27],checkmark:3,checkpoint:[3,4,14],checkpoint_dir:4,checkpoint_prefix:4,chemic:[],chen:10,chiaramont:2,choic:[0,1,2,3,4,6,9,12,13,14,20,26,27,29],choleski:[5,20,27],choos:[2,3,6,9,10,11,13,14,21,29],chosen:[0,1,2,6,8,9,10,13,16,23,26,29],chosen_datapoint:1,christian:25,christoph:[22,25,26],cifar10:3,cifar:3,cin:[],circ:[1,12],circl:[0,8,12,27],circuit:3,circumfer:9,circumv:[1,5,13,27],ckpt:4,clariti:23,class_nam:[3,9],class_val:9,class_valu:9,class_weight:[],classic:[7,9,13],classif:[0,3,5,6,7,8,11,12,19,21,25,26,27,28,29],classifi:[0,1,4,7,9,10,11,26],classificaton:1,classifii:10,clean:1,clear:[1,5,10,12,13],clearli:[0,3,5,6,7,8,18,23,27,28,29],clever:[1,10],clf3:0,clf:[0,6,8,9,10,26,27],clf_lasso:6,clf_ridg:[6,26],clip:[3,23],clone:24,close:[0,1,2,4,6,8,9,11,12,13,14,21,23,25,26,29],closer:[3,5,13,27],closest:[8,11,13,14],closur:[19,26],cloud:[19,26],cluster:[0,1,4,6,11,19,26,29],cluster_label:14,cm:[1,2,3,6,8,9,13,21],cmap:[0,1,2,3,4,6,8,9,10,21,26],cmap_arg:6,cmath:[],cmb:22,cmd:9,cmu:27,cn_:23,cnn:12,cnn_kera:3,cntk:[19,26],co:[0,2,3,6,9,13,26,29],code:[3,4,6,7,8,15,16,17,19,20,21,23,25],coef0:8,coef:[0,26],coef_:[0,5,6,8,9,13,26,27,28],coeff:5,coeffici:[0,3,5,6,7,8,9,13,15,16,20,26,27,28,29],coerc:[0,6,26,29],coin:[10,23],coin_toss:10,col:[0,11,26,27],colab:[19,26],cold:9,colinear:[0,27],collabor:21,collaps:8,collect:[0,2,6,10,11,19,23,25,26,29],collinear:[5,27,28],color:[0,3,4,6,8,9,10,21,23],color_channel:3,color_cod:6,colorbar:[1,6,21],colsample_bytre:10,colsaobject:10,colspec:[0,26],column:[0,1,2,5,6,7,8,9,11,12,17,20,26,27,28,29],columntransform:9,com:[3,4,6,14,19,21,25,26,28],combin:[1,2,5,6,7,10,23,28,29],come:[0,1,3,4,5,12,13,14,15,26,27,28,29],command:[0,1,26],comment:[0,4,5,6,15,16,21,26,27],commerci:[0,15,19,21,26],commod:[0,26],common:[0,1,3,5,6,7,9,11,13,14,21,23,26,27,29],commonli:[0,1,4,6,7,9,13,14,27,29],commun:[0,12,21],commut:3,commutatitav:3,compact:[0,1,3,5,6,7,9,11,12,13,14,26,27,28,29],compair:0,compar:[0,3,4,5,6,11,13,15,16,20,21,26,27,28,29],comparison:[2,4,13],compat:[3,4,7,14],compet:[0,26],competit:10,compil:[0,1,3,4,13,15,19,20,26],complet:[0,2,3,4,9,12,26],completenn:12,complex:[1,5,8,9,11,12,13,15,16,26,29],complic:[0,1,9,13,26,29],compoment:27,compon:[0,1,3,4,5,6,7,9,14,16,19,26,27,28,29],components_:11,compos:[9,12,13,14,19,26],compphys:[0,6,17,19,21,22,24,25,26,27],compress:[0,26,27],compris:6,compromis:[5,27],compulsori:[19,26],comput:[0,1,2,3,4,5,6,7,8,10,11,12,13,15,16,17,19,20,21,22,23,25,26,27,28,29],computation:[0,3,6,9,13,23,26],computationalscienceuio:26,computerlab:21,con:[],concat:26,concaten:[2,4,6,14],concav:[1,9,13,27],concentr:[0,10,27],concept:[0,2,19,26,27],conceptu:[12,13],concern:[0,1,4,7,26],concic:26,conclud:[0,5,13,28,29],conclus:1,cond:[2,3,4,14],conda:[0,1,15,19,21,26],condis:27,condit:[0,2,4,5,6,8,9,11,13,23,26,27],conduct:[19,26],condwav:2,coneqp:[],confid:[0,5,6,7,8,18,21,26,27,28],config:[3,4,14],config_pb2:[],configur:3,confirm:[5,12,28,29],confus:[5,6,7,10,20,27,28,29],confusion_matrix:9,congruenti:23,conjug:[4,8],conjugaci:13,conjunct:3,connect:[0,1,3,4,9,11,12,13,20,26,27],consequ:[5,6,8,10,12,13,27,28,29],conserv:[5,14,27],consid:[0,1,2,3,5,6,7,8,9,10,12,13,16,20,21,23,26,27,28,29],consider:[0,1,5,13,26,27,28,29],consist:[0,1,2,3,4,6,12,13,21,23,27,29],constant:[0,2,3,4,5,6,8,12,13,14,23,26,27,28,29],constitu:[0,26],constitut:[2,6,29],constrain:[1,3,5,7,11,28],constraint:[5,6,8,13,17,27,28,29],construct:[0,1,2,3,5,6,7,8,9,10,11,20,23,26,27,28,29],contact:[0,26],contain:[0,2,3,4,5,6,7,8,9,11,12,13,14,17,20,21,23,25,26,27,28,29],contemporari:26,content:[1,19,20,26],context:[6,10,13,21,29],contigu:20,continu:[0,1,2,3,4,5,6,7,8,9,10,12,13,17,18,20,21,23,26,27,28,29],contour:[9,10,13],contourf:[8,9,10],contrast:[1,4,9,10,12],contribut:[0,3,5,13,23,26,27,28],contributor:[0,21,26],control:[0,1,3,9,13,15,19,26],control_flow:[3,4,14],conv2d:[3,4],conv2dtranspos:4,conv:[3,4],conv_general_dilated_patch:[3,4,14],convei:26,conveni:[0,5,6,12,13,20,21,26,28,29],convent:[12,27],converg:[1,2,4,5,6,7,8,11,13,14,27,28],convergencewarn:[1,6,7,8,11],convert:[0,1,3,4,5,9,11,13,14,20,26,27],convert_phas:[3,4,14],converter_error_data_pb2:[3,4,14],converttomatrix:4,convex:[4,5,7,27],convinc:13,convolut:[1,4,19,26],cool:[4,9],coolwarm:[6,21],coordin:[5,12,14,27,28],coorel:[0,27],copi:[0,1,2,14,27],copyright:[3,4,14,29],core:[2,3,4,10,13,14,26],corel:26,coronari:7,corr:[0,5,7,11,27],correalt:[11,19],correct:[0,1,2,3,4,5,7,13,20,23,26,27,28,29],correctli:[1,2,6,7,10],correl:[0,1,3,5,6,7,10,12,13,19,23,26,28,29],correlation_matrix:[0,5,7,11,27],correspond:[0,3,5,6,8,9,11,12,15,16,17,19,20,21,23,26,27,28,29],cortex:12,cosin:[3,6,29],cost:[0,2,3,5,6,7,8,9,12,13,16,17,21,26],cost_deep_grad:2,cost_funct:2,cost_function_deep:2,cost_function_deep_grad:2,cost_function_grad:2,cost_grad:2,cost_sum:2,costol:13,could:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,17,20,21,23,26,27,28,29],coulomb:[0,26],count:[0,9,22,23,24,26],countor:13,coupl:[4,5,6,28,29],cours:[0,1,3,5,11,21,22,24,27,29],courvil:[22,25,26,27],cout:[],cov:[5,6,11,20,23,26,27,28,29],cov_xi:[5,11,27],cov_xx:[5,11,27],cov_yi:[5,11,27],covari:[0,7,19,20,26,28],covariance_matrix:[5,11,14,27],cover:[0,5,17,19,24,25,27,28],covert:[0,26],covxi:23,covxx:23,covxz:23,covyi:23,covyz:23,covzz:23,cpj4mh7m9no:[],cpu:[1,3,4,14],cpu_feature_guard:[],craft:3,crawford:26,creat:[1,2,3,4,5,6,9,10,11,12,13,19,21,26,28],create_biases_and_weight:1,create_convolutional_neural_network_kera:3,create_neural_network_kera:1,create_x:[5,11,27],credit:[0,7,24,26],crim:[0,27],crime:[0,27],criteria:[0,4,9,10,14,23,26],criterion:[9,10,13],critic:[6,21,26,27],critiqu:21,cross:[0,1,3,7,9,10,13,19,23,26,27],cross_entropi:4,cross_val_scor:[6,29],cross_valid:[7,10],crossvalid:[6,29],crucial:[1,23],cs231:3,cs:[22,24],csr_matrix:[20,26],cstdlib:[],csv:[0,4,6,7,9,26,29],ctnk:1,ctx:[],cubic:0,cumbersom:[5,28,29],cumsum:[10,11,26],cumul:[7,10,23],cumulative_heads_ratio:10,cup:[5,28,29],current:[1,2,3,4,6,13,14,21,25],curs:[0,27],curv:[6,7,10,12,21],curvatur:13,custom:[6,14,21],custom_cmap2:[9,10],custom_cmap:[9,10],cutpoint:9,cv:[6,7,10,29],cvxbook:13,cvxopt:[5,8,27],cyber:[],cycl:[1,3,4,12,14],cyclotron:[27,28],d1:[],d2:[],d2_g_t:2,d3:[],d670a873ab0c:[],d985fb40c43d:[],d:[1,2,3,4,5,6,7,8,9,10,11,13,14,20,23,24,26,27,28,29],d_f:13,d_g_t:2,d_net_out:2,da:3,dagger:[5,20,27],dai:[1,9,19],dalen:[],damp:3,daniel:[24,26],darget:9,darkr:23,dat:[0,26],dat_id:[0,6,7,9,26,27,29],data1:14,data2:14,data3:14,data4:14,data:[2,4,5,8,10,12,13,14,17,18,20,25,28,29],data_handl:[],data_id:[0,6,7,9,26,27,29],data_indic:1,data_modul:27,data_panda:26,data_path:[0,6,7,9,26,27,29],data_url:27,databas:1,databookuw:[],datafil:[0,6,7,9,21,26,27,29],datafram:[0,4,5,7,9,11,26,27],datapoint:[1,5,6,7,11,13,27,29],dataset:[0,4,6,7,8,9,10,11,13,14,15,16,21,26,27,29],datatyp:4,date:[15,16,17,18,21,26,27,28,29],daughter:10,david:25,davison:29,dbh:1,dbo:1,dcomposit:20,ddot:2,dead:1,deadlin:[16,17,18,22],deal:[0,1,3,5,6,8,11,13,14,17,18,20,21,23,26,27],dealt:0,debt:7,debug:[0,5,6,28,29],decad:[0,3,26],decai:[0,13,23,26],decemb:[22,24,26],decent:10,decid:[0,2,3,5,6,9,21,28,29],decim:[0,26],decis:[0,1,8,11,19,25,26],decision_funct:8,decision_tre:9,decisiontreeclassifi:[9,10],decisiontreeregressor:[0,9,10],declar:[0,4,20,26],decompos:[5,6,20,27],decomposit:[0,6,12,21,26,28],decompost:[5,27],deconvolut:3,decor:[0,26],decorrel:[10,13],decreas:[1,2,4,5,6,10,11,13,28,29],deduc:[0,26],deep:[3,7,12,13,19,22,25,26,27],deep_neural_network:2,deep_param:2,deep_tree_clf1:9,deep_tree_clf2:9,deep_tree_clf:[9,10],deepen:[5,19,26],deeper:[0,3,4,26],deeplearningbook:25,deer:3,def:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,21,23,26,27,28,29],def_covari:23,def_funct:[],default_tim:4,defect:[5,27],defici:[5,27],defin:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,27,28,29],definit:[1,2,5,6,7,8,10,11,12,13,17,20,23,27,28,29],defint:23,defun:[],defvjp:2,degre:[3,5,6,8,9,10,11,15,16,17,21,23,26,28,29],del:1,delet:6,delimit:4,deliv:[22,26],delta:[0,2,3,6,8,12,13,14,26],delta_0:3,delta_1:3,delta_2:3,delta_3:3,delta_4:3,delta_5:3,delta_:[1,20],delta_h:[0,1,26],delta_j:[3,12],delta_k:12,delta_l:[1,3],delta_momentum:13,delta_n:[0,3,26],delug:19,delv:[0,26],demand:13,demonstr:[0,3,5,6,7,11,12,19,26,27,29],den:4,denomin:[1,5,28,29],denot:[1,2,6,7,13,23],dens:[1,3,4],dense_1:[],densiti:[0,2,6,23,26,29],depart:[24,26,27,28,29],depend:[0,1,2,4,5,6,7,8,11,12,13,15,19,20,21,23,26,27,28,29],depict:23,deploy:[0,15,19,21,26],deprec:[2,6,13,21,26,27],deprecate_nonkeyword_argu:[0,26],depth:[0,3,9,10,20,29],deriv:[0,1,2,6,7,8,10,11,13,16,17,19,21,26],derivati:13,derivative_fn:13,descend:[5,9,11,27,28],descent:[0,1,3,7,8,12,26,27],descr:27,describ:[0,2,4,5,6,8,10,11,12,13,18,20,21,26,28,29],descript:[0,8,9,26],design:[0,1,3,4,5,6,7,10,11,12,13,15,16,17,18,21,26,28,29],designmatrix:[0,26],desir:[0,2,4,5,13,14,26,27],despit:[1,12],destroi:20,det:[5,20,27],detail:[0,6,11,13,14,20,21,27],detect:[3,8,12],determin:[0,2,3,4,5,6,8,9,10,11,12,13,16,20,23,26,27,28,29],determinist:[7,13,23],dev:1,develop:[0,3,5,8,10,11,12,19,20,21,26,27,28,29],deviat:[0,1,2,4,5,6,21,23,26,27,29],device_nam:[],devicearrai:[3,4,14],devis:12,df1:26,df:[4,8,11,13,26],di:[0,27],diag:[5,8,27,28],diagnost:[1,10],diagon:[0,5,7,13,18,20,21,23,26,27,28],diagonaliz:[5,27],diagram:10,diagsvd:6,dice:[6,23,29],dict:[6,8],dict_kei:27,dictionari:[0,27],did:[0,1,5,6,7,10,11,14,21,26,28,29],die:1,diff1:2,diff2:2,diff:2,diff_ag:2,diffeent:8,differ:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,15,17,19,20,21,23,25,26,27,28,29],differenti:[0,3,16,19,20,26,27],differential_oper:[2,13],difficult:[0,1,6,10,13,21,23,26,29],difficulti:[0,1,13,26],diffonedim:2,digit:[0,1,3,4,6,21,22,24,26],dilemma:13,dilut:1,dim:[4,11,14,20],dimens:[0,1,2,3,4,5,8,9,11,14,16,17,20,26,27,28],dimension:[0,4,5,6,9,11,13,14,19,20,26,28,29],dimensionless:[0,3,26],diment:20,dimnsion:4,diod:3,direct:[0,1,2,4,11,12,13,14,26,27],directli:[1,4,5,6,23,27],directori:9,disabl:[3,4,14],disadvantag:[0,26],disappear:[3,6,29],disc_loss:4,disc_tap:4,discard:[6,11,29],disciplin:[0,3,12,26],disclaim:23,discord:26,discourag:[13,27],discov:[0,26],discover:[5,28],discret:[1,3,5,7,13,28,29],discrimin:[4,7,10,11],discriminator_loss:4,discriminator_loss_list:4,discriminator_model:4,discriminator_optim:4,discuss:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,25,26,27,28,29],diseas:7,disguis:[6,27],disord:[1,7],displai:[0,1,3,4,5,6,7,8,9,10,11,12,14,21,23,26,27,29],displaystyl:[0,5,17,26,27,28],displot:27,disregard:[0,26],dissimilar:[11,14],dist:14,distanc:[0,8,9,11,14,23,27],distance_list:9,distinct:[3,7,8,9,10,14],distinctli:8,distinguish:[0,4,7,8,23,26],distplot:[0,27],distribut:[0,1,4,6,7,10,11,13,14,15,16,18,19,20,21,26,27],distrubut:[0,15,19,21,26],div:[],dive:[0,8,20,26],diverg:[1,13],divid:[0,1,3,5,6,7,8,9,11,12,23,26,27,28,29],divis:[6,8,9,13,20,23,29],dna:7,dnn1:4,dnn2_gru2:4,dnn:[0,1,2,4,12,26],dnn_kera:1,dnn_model:1,dnn_numpi:1,dnn_scikit:[0,1,26],doamin:[28,29],doc:[0,6,17,19,21,22,24,25,26,27],document:[4,7,11,13,27],doe:[0,1,2,3,4,5,6,8,10,11,12,13,16,20,21,23,26,29],doesn:[3,9,12],dog:[1,3,4],domain:[5,8,13,21,28,29],domin:[0,26],don:[0,1,3,5,6,8,11,13,15,19,21,26,27],done:[0,2,3,4,5,6,9,10,11,13,17,20,21,26,27,28,29],dot:[0,2,3,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],doubl:[3,4,20,26],doubli:1,down:[0,3,6,9,11,12,13,26],download:[0,1,3,5,6,20,21,25,26],downsampl:3,dozen:1,dq:[6,29],drag:13,dramat:11,drastic:4,draw:[4,6,10,13,29],drawback:[0,1,3,13,27],drawn:[1,4,6,7,11,23,26,29],drive:[3,4],driven:3,drop:[0,1,5,6,11,13,23,26,27,28,29],dropna:[0,6,26,29],dropout:4,ds:[],dt:[2,3,13,23],dtype:[0,1,2,3,4,14,20,26,27],dualiti:6,dub:[0,26],due:[1,2,5,6,8,10,12,13,24,26,27,29],dummi:[0,27],dure:[0,1,3,4,8,9,11,17,19,21,26,29],dwell:[0,27],dwh:1,dwo:1,dx:[2,3,8,23],dx_1:23,dx_1p:[6,29],dx_2p:[6,29],dx_mp:[6,29],dx_n:23,dxp:[6,29],dy:[1,8,23],dynam:4,dysth:[],dz:8,e7cu:[],e:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,18,23,24,26,27,28,29],e_:[0,2,26],each:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,19,20,22,23,24,26,27,28,29],eager:29,eapprox:[0,26],earli:[1,13],earlier:[0,5,7,8,9,11,12,13,26,27],earthexplor:[6,21],eas:[6,9,14,29],easi:[0,5,6,7,8,9,10,11,12,13,18,19,20,26,27,28,29],easier:[5,6,8,9,13,23,26,27,29],easiest:13,easili:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,26,27,28,29],eastern:[24,26],ebind:[0,26],eblock:9,econometr:26,economi:5,ecosystem:[19,26],ect:22,edg:3,edgecolor:[6,29],edu:[13,27],educ:[0,21,26,27],eface79dac2c:[],eff:23,effect:[1,4,10,13,23],effic:1,effici:[0,3,10,13,19,20,23,26],efron:[6,29],egrad:13,eig:[5,11,13,20,23,26,27],eigen:23,eigenpair:[5,11,27],eigenvalu:[0,5,8,11,13,20,26,27,28],eigenvector:[5,11,13,17,27,28],eight:[20,26],eigval:[20,23,26],eigvalu:[11,13],eigvec:[20,23,26],eigvector:[11,13],eispack:[20,26],either:[1,5,6,7,8,9,10,11,13,15,16,21,23,26,27,28,29],ekstrom:[],elabor:23,elarn:3,electr:[0,3,12,26],electron:26,electur:[],eleg:11,element:[1,2,3,4,5,6,7,8,11,12,13,17,18,19,20,21,22,25,27,28,29],elementari:[10,13,20],elementwis:[3,13],elementwise_grad:[2,13],elessar:26,elif:[2,14],elim:20,elimin:[3,8],els:[1,2,3,4,7,9,12,13,20],elu:1,elus:[0,26],email:[22,24,26],embed:[0,11,27],embodi:[6,21,29],emit:23,emner:25,emphas:[0,10,19,26],emphasi:[0,19,25,26],empir:[1,11,23],emploi:[0,1,5,6,11,13,21,23,26,27,28,29],employ:[0,26,27],empti:[6,10,29],emul:12,en:[19,25],enabl:11,enable_check:[3,4,14],enbodi:[6,29],encapsul:[3,4,14],encod:[0,3,5,9,11,14,17,26,27,28],encompass:[0,21,23,26],encount:[0,1,5,6,7,13,23,26,27,29],encourag:21,end:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,20,21,23,24,26,27,28,29],end_box:[2,13],end_nod:[2,13],end_valu:[2,13],endl:[],endpoint:[3,6],energi:[0,4,6,27,29],enet_coordinate_desc:6,enforc:12,eng:25,engin:[0,1,3,4,19,26],english:21,enorm:3,enough:[0,6,13,26,29],ensembl:[1,9],ensur:[0,1,2,3,5,6,11,13,23,27,28,29],ensure_initi:[],entail:26,enter:[5,6,17,27],enthought:[0,15,19,21,26],entir:[1,3,7,9,19,23,26],entiti:[9,12,20,26],entri:[0,5,8,11,12,20,26,27,28,29],entropi:[1,3,7,10,13,26],enumer:[0,1,2,3,4,6,8,26,27],env:[0,1,2,3,4,6,7,8,11,13,14,23,26,27,28],environ:[2,19,21],eo:[0,6,26,29],eol:[0,26],eosfit:[0,26],epoch:[0,1,3,4,12,13,26],epsilon:[0,5,6,7,13,21,26,27,28,29],epsilon_0:[0,26],epsilon_1:[0,26],epsilon_2:[0,26],epsilon_:[0,26],epsilon_i:[0,26,27],eq:[3,13,14,20,23],eqnarrai:[3,5,6,28,29],equal:[0,1,2,3,4,5,6,8,9,11,12,13,14,15,20,21,23,26,27,28,29],equat:[1,3,4,5,6,7,8,9,10,11,13,14,16,17,18,20,21,23,29],equilibrium:[2,12],equiv:[3,13,20,23],equival:[0,1,5,7,8,11,13,19,20,26,27,28,29],erf:23,eriador:26,eridg:26,err:[0,10],err_:[6,29],err_sqr:2,errat:13,errno:9,erron:2,error:[1,2,4,5,6,7,9,11,12,13,15,16,17,18,19,20,21,23,28],error_estimate_corr_tim:23,error_handl:[],error_hidden:1,error_output:1,escap:13,especi:[1,3,9,12,13,21],essenti:[0,5,6,9,10,12,14,21,23,27],establish:[0,6,10,11,15,21,26],estim:[0,1,5,6,7,10,11,13,19,23,26,27],estimated_mse_fold:[6,29],estimated_mse_kfold:[6,29],estimated_mse_sklearn:[6,29],et:[0,2,4,15,16,17,25,26,27,28,29],eta0:[8,13],eta:[0,1,3,8,12,13,26],eta_:13,eta_t:13,eta_v:[0,1,3,26],etc:[0,1,3,5,7,8,9,11,12,13,14,15,19,20,21,23,27],ethic:[19,26,27],etsim:29,euclidean:[0,14,27],evalu:[0,2,3,4,5,6,9,13,21,23,26,27,28,29],evaluationform:21,evaluationgrad:21,evalut:13,even:[0,1,3,4,5,6,8,9,10,11,12,13,14,19,20,23,26,27,28,29],evenli:4,event:[3,4,5,7,10,14,23,28,29],eventu:[0,5,6,11,12,13,21,24,27,28,29],everi:[0,1,2,3,4,5,6,9,10,11,12,13,14,19,23,24,26,27,28,29],everyth:[4,12],everywher:[4,13],evolv:[0,26],exact:[0,2,5,11,12,13,20,23,26,27],exactli:[0,3,4,6,12,19,21,27,29],exam:26,examin:[6,29],exampl:[5,11,12,13,15,16,19,20,21,23,25],exce:[1,12,13],excel:[0,1,4,5,10,26,27],except:[3,4,6,8,9,14,20],excess:[0,26],excit:[0,26],exclud:[1,6,12,21,27,29],exclus:[0,1,3,6,23,26,29],execut:[2,5,13,27],executing_eagerli:[],exemplifi:13,exercic:[24,26],exercis:[5,19,21,22,24,28],exercisesweek35:17,exhaust:[6,29],exhibit:[0,5,6,8,26,27,29],exist:[0,1,2,3,5,6,7,8,9,13,18,20,21,26,27,28,29],exit:[5,20,27],exp:[0,1,2,5,6,7,8,10,11,12,13,15,16,21,23,26,27,28,29],exp_term:1,expand:[5,7,11,13,27],expans:[0,3,5,8,10,12,13,26,27],expect:[0,1,5,6,7,11,12,13,15,16,19,21,26,27],expectation_value_of_h_wrt_p:23,expens:[6,10,13],experi:[0,1,6,8,13,19,21,26,27,29],experiment:[0,3,4,6,9,14,23,26,29],experimental_get_tracing_count:[],expert:[1,9],explain:[0,6,9,10,11,13,21,26],explained_variance_ratio_:11,explanatori:[0,26],explicit:[0,3,6,13,20,21,26,27],explicitli:[0,4,26],explod:1,exploit:[0,3,12,13,26],explor:[1,4,6,8,13,19,21],expon:1,exponenti:[0,1,5,6,10,13,21,23,26,28,29],export_graphviz:9,export_text:9,exporttext:9,expos:[19,26],express:[0,2,3,5,6,7,10,12,13,16,20,21,23,28,29],exptmean:23,exptvari:23,extend:[0,2,7,11,13,17,19,26],extens:[0,12,15,19,26],extent:[0,1,6,25,26,29],extern:[3,6,9],extra:[1,3,5,24,26,27],extract:[0,3,5,6,7,8,11,13,20,26,27],extrapol:[0,26],extrem:[0,1,4,5,6,7,8,9,13,16,20,27,28,29],extremum:13,extrins:11,ey:[0,5,6,13,14,20,26,28],f11:[0,26],f12:[0,26],f13:[0,26],f1:13,f1_grad:13,f1d:13,f2:13,f2_grad_x1:13,f2_grad_x1_analyt:13,f2_grad_x2:13,f2_grad_x2_analyt:13,f3:13,f3_grad:13,f3_grad_analyt:13,f3ckh:[],f401:[3,4,14],f4:13,f4_grad:13,f4_grad_analyt:13,f5:13,f5_grad:13,f6:13,f6_for:13,f6_for_grad:13,f6_grad_analyt:13,f6_while:13,f6_while_grad:13,f6d7a289d493:[],f7:13,f7_grad:13,f7_grad_analyt:13,f8:13,f8_grad:13,f9:[0,13,26],f9_altern:13,f9_alternative_grad:13,f9_grad:13,f:[0,1,2,3,4,5,6,7,8,10,12,13,14,16,18,20,23,24,26,27,28,29],f_0:[3,10],f_1:[10,13],f_2:[12,13],f_3:12,f_:10,f_d:23,f_grad:13,f_grad_analyt:13,f_i:[0,6,12,16,29],f_m:[3,10],f_n:3,f_raw:2,f_vec:2,f_wrap:2,face:13,facecolor:[6,8,23,29],facil:[0,15,19,26,29],facilit:12,fact:[0,1,3,5,9,11,12,13,26,27,28,29],factor:[0,1,3,5,6,9,10,11,13,20,23,26,27,28,29],factori:13,fade:6,fafab0:[9,10],fahimeh:[24,26],fail:[0,6,7,8,11,13,24,26,29],failur:7,fairli:[1,2,23],fake:4,fake_loss:4,fake_output:4,fall:[8,9,22],fals:[0,1,2,3,4,5,6,7,9,10,14,20,21,26,27,28,29],famili:[0,7,8,23],familiar:[0,3,5,6,8,15,19,20,21,23,26,28,29],famou:[6,12],far:[0,3,4,5,6,8,11,12,13,14,26,27],fashion:[0,9,10,26],fast:[1,3,6,10,12,13,19,23,26,29],faster:[1,11,13],fastest:[13,20],favor:7,favorit:23,fc:3,fdf8a5d7c717:[],fdfcc778e1f8:[],fe5b9d300cc0:[],featur:[0,1,3,5,6,7,8,10,11,12,13,18,19,21,23,26,28,29],feature_nam:[0,1,7,9,27],feautur:9,fed:1,feed:[0,2,3,11,19,26],feed_forward:1,feed_forward_out:1,feed_forward_train:1,feedback:[4,26],feeddorward:4,feedforward:[1,4,12],feel:[0,5,6,11,13,15,16,19,21,24,26,28],feet:[0,27],fei:26,felt:21,fetch:[6,21,27],fetch_california_h:27,fetch_openml:27,few:[1,3,4,5,9,23,26],fewer:[0,9,11,26],ffnn:[1,12],fft2:[3,4,14],fft:[3,4,14],fft_batching_rul:[3,4,14],fft_p:[3,4,14],fftfreq:[3,4,14],fftn:[3,4,14],fftshift:[3,4,14],fgakzw6k1qq:[],field:[0,3,6,12,19,26],fifth:[0,6,15,16,21,26],fig:[0,1,2,3,4,6,7,12,13,14,21,26],fig_id:[0,6,7,9,26,27,29],figaxi:23,figsiz:[0,1,2,3,4,6,7,8,9,10,26,27,29],figur:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,19,21,26,27,28,29],figure_id:[0,6,7,9,26,27,29],figurefil:[0,6,7,9,26,27,29],file:[0,2,3,4,5,6,7,9,13,14,21,26,27,28,29],file_prefix:4,filenam:[26,27],filenotfounderror:9,fileout:[],filepath_or_buff:[0,26],fill:[5,9,27],filter:[3,4],filtered_flat_arg:[],filtered_tb:[],financ:[0,26],find:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,19,21,23,26,27,28],find_top_boxed_arg:[],fine:[0,14,15,26],finish:2,finit:[3,5,6,12,13,17,23,27,28,29],first:[0,1,2,3,5,6,7,8,9,10,11,13,14,15,16,17,20,21,23,24,25,29],firsteigvector:11,fit:[1,3,4,5,6,7,8,9,11,12,13,15,16,17,21,23,27,29],fit_beta:[6,27],fit_intercept:[0,5,6,27,28,29],fit_mod:9,fit_transform:[0,6,8,9,11,29],fiti:[0,26],five:[0,9,17,21,26,27],fix:[0,3,4,6,10,11,12,13,21,26,29],fixedformatt:6,fixedloc:6,fkkt:[],flag:4,flake8:[3,4,14],flat:[12,13],flatbuff:[],flatten:[1,3,4,5,20],flexibl:[1,6,8,10,12,26,27,29],flip:[24,26],float32:[4,9],float64:[4,20,26,27],flop:[5,20,27],flow:[1,4,12],fluctuat:[5,28],fly:11,flyvbjerg:[],fm:[0,26],fma:[],fmax:3,fmesh:13,fn:7,focu:[0,3,4,5,6,15,19,25,26,27,28,29],focus:[1,6,7,20,27],fold:[6,9,21],folder:[0,1,4,6,13,21,26,29],follow:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],font:[0,7,23,26],fontdict:23,fontsiz:[1,6,8,9,10,23],fontweight:1,footprint:3,foral:[8,27],forc:[0,5,6,10,11,26,27,28],forcast:4,forecast:[4,12],forelesningsvideo:[],forest:[0,1,9,19,26],forget:11,form:[0,3,4,5,6,7,8,9,11,12,13,17,19,20,21,23,26,27,28,29],formal:[3,4,14,23],format:[0,1,2,3,4,6,7,8,9,10,11,19,23,25,27,28,29],format_data:4,formatstrformatt:[6,13,21],formul:[4,6,11,14],formula:[3,13,23],forth:[4,12],fortran2003:[19,26],fortran2008:21,fortran90:23,fortran:[0,15,19,20,26],fortun:[0,11,27],forward:[0,3,6,19,20,26,29],forward_backward:[],forward_compatibility_horizon:[3,4,14],found:[1,2,4,5,6,12,13,21,26,27,28,29],foundat:[19,26],four:[4,5,6,8,12,20,21,22,24,26,28],fourier:[0,26],fourierdef1:3,fourierdef2:3,fourierseriessign:3,fourth:[12,26,27],fp:7,fr:[],frac:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,17,20,21,23,26,27,28,29],fractal:9,fraction:9,frame:[7,26],framework:[1,8,10,23],frank:[5,11],frankefunct:[5,6,11,21,27],fredli:[24,26],free:[0,6,11,13,15,16,19,20,21,23,24,25,26],freecodecamp:19,freedom:[5,28],freeli:[0,21,26],frequenc:[3,6,7,23,29],frequent:[0,8,9,13,26],frequentist:19,fresh:10,frida:[],fridai:[16,24,26],friedman:[6,18,21,22,25,26,27],friendli:4,frodo:26,frog:3,from:[0,1,2,3,4,6,7,8,9,11,13,14,16,17,18,19,20,21,23,24,25,26],from_cod:9,from_logit:[3,4],from_tensor_slic:4,fromnumer:2,front:[0,4,5,15,16,26,27,28],fstream:[],fulfil:[2,5,12,27],full:[0,1,3,5,7,9,10,13,23,26,27],full_matric:[5,27],fulli:[3,6,12,22,23,29],fun:[2,13,19,26],fun_nam:2,func:[0,2,26,27],functionali:11,functool:[3,4,14],fundament:[0,6,19,26,29],funtion:2,further:[2,7,9,26],furthermor:[0,3,5,6,7,11,12,13,17,19,21,26,27,29],futur:[0,4,8,9,26,27],futurewarn:[0,26,27],fx:[],fy:[15,21,22,24,25,26],fys4155:21,fys5419:[25,26],fys5429:[25,26],g0:2,g42mrgv128v34gnnhxwk9nrc0000gp:[],g:[0,1,2,3,4,6,8,9,10,11,13,14,23,26,29],g_0:2,g_1:[2,10],g_2:[2,10],g_:[2,9,10],g_analyt:2,g_dnn_ag:2,g_euler:2,g_i:2,g_m:[3,10],g_n:3,g_re:2,g_t:2,g_t_d2t:2,g_t_d2x:2,g_t_dt:2,g_t_hessian:2,g_t_hessian_func:2,g_t_jacobian:2,g_t_jacobian_func:2,g_trial:2,g_trial_deep:2,g_vec:2,gain:[1,5,7,9,10,13,27,28,29],galleri:[0,26],game:4,gamge:26,gamma1:8,gamma2:8,gamma:[0,2,8,9,10,11,13,26],gamma_0:10,gamma_1:10,gamma_1x:10,gamma_:[0,26],gamma_i:[0,8,23,26],gamma_j:13,gamma_k:13,gamma_m:10,gamma_x:[0,26],gap:[6,8],gate:[4,12],gather:[0,1,12,27],gaug:12,gaussbacksub:20,gaussian:[4,5,6,8,14,23,28,29],gaussian_point:14,gaussian_rbf:8,gave:13,gbc:[22,26],gca:[2,6,8,13,21],gd:1,gd_clf:10,gdclassiffiercgain:10,gdclassiffierconfus:10,gdclassiffierroc:10,gdm:13,gdregress:10,ge:[1,5,7,23,27],gemv:[],gen:[],gen_loss:4,gen_tap:4,gender:[0,26],genener:4,gener:[0,1,2,3,5,6,8,10,11,12,13,14,15,16,17,20,21,23,25,27,28,29],generallay:12,generate_and_save_imag:4,generate_imag:4,generate_latent_point:4,generate_simple_clustering_dataset:14,generated_imag:4,generator_loss:4,generator_loss_list:4,generator_model:4,generator_optim:4,genexpr:[],genom:19,geodes:11,geometr:[0,13,26],geometri:[5,28,29],georg:25,geotif:[6,21],geq:[2,5,8,9,13,27,28],geron:[0,15,22,25,26],get:[0,1,2,3,4,5,6,7,9,10,11,13,15,19,20,21,23,24,26,27,28,29],get_dummi:9,get_loc:[],get_next_color:[],get_paramet:2,get_split:9,get_yaxi:8,get_yticklabel:6,getattr:2,getsolutionslic:[],getval:2,gg:[],gibb:[19,26],gif:4,gini:10,gini_index:9,ginvers:13,git:[0,15,19,26],giter:13,github:[0,3,4,6,14,15,17,19,21,22,24,25,26,27],gitlab:[0,15,19,21,26],give:[0,1,2,3,5,6,7,8,9,10,12,13,14,17,19,21,23,26,27,28,29],given:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],global:[6,7,13,21],glorot:1,gmail:[],gnew:13,go:[0,1,3,5,6,8,9,11,12,13,21,26,27,28,29],goal:[0,7,9,26],goe:[0,1,2,5,6,13,14,18,20,26,27,28,29],golden:13,gone:[5,27,28],gong:1,good:[1,3,4,5,6,9,10,11,13,17,19,23,25,27,28,29],goodfellow:[4,22,25,26,27,28],googl:[1,3,4,14,19,26],got:[1,6,21],gov:[6,21],gp:25,gpu:[1,13,19,26],grad:[2,13],grad_analyt:13,grade:22,gradient:[0,3,4,7,8,9,12,19,26,27],gradient_desc:[],gradientboostingclassifi:10,gradientboostingregressor:10,gradients_of_discrimin:4,gradients_of_gener:4,gradienttap:4,gradual:[1,14],grai:[4,6,21],graph:[1,9,11,12,13],graph_debug_info_pb2:[],graph_from_dot_data:9,graph_funct:[],graphic:[0,1,9,26],grasp:[0,26],gray_r:[1,3],grayscal:3,great:[5,13],greater:[1,7,23],greatli:13,greedi:9,green:[0,3,9,23],grei:4,grid:[1,3,6,7,8,12,23,27,29],grossli:13,ground:[0,26],group:[0,6,7,9,14,19,21,22,24,26,29],groupbi:[0,26],grow:[1,3,9,10],growth:[0,26],gru:4,guarante:[0,4,13,23,26,27],guess:[1,4,10,13,14],guestrin:10,guid:1,guilherm:[],h1:2,h21:[],h:[0,1,5,6,8,13,16,18,21,23,24,25,26,27],h_1:[2,13],h_2:[2,13],h_:[0,13,26],h_m:10,ha:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,20,21,23,26,27,28,29],haa:[24,26],habit:[0,27],had:[0,1,6,7,13,26,29],hadamard:[1,12,13],half:[1,8,9],halv:10,hand:[0,1,2,3,5,11,12,13,15,19,20,21,22,23,24,25,26,27,28,29],handi:[3,21],handl:[0,1,2,5,9,11,19],handle_unknown:9,handsid:12,handwrit:12,handwritten:[1,5,26],handwrittennot:26,happen:[1,2,3,4,5,6,10,13,23,27],hard:[1,7,8,10,13],hardcopi:[19,26],harder:[0,1,27],harmon:3,hasn:1,hassl:[0,15,19,26],hast:[19,26],hasti:[0,6,15,16,17,18,21,22,25,26,27,28,29],hat:[0,1,5,6,7,9,10,11,12,13,17,18,20,21,27,28,29],have:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,27,28,29],haven:1,hdtta7prrfi:[],he:7,head:[0,4,10,23,27],header:[0,26,27],heads_proba:10,health:[0,27],hear:[0,13,26],heart:[0,7,26],heatmap:[0,1,3,7,26,27],heavili:[0,26],heavisid:1,height:[1,3,6,27],held:13,help:[0,1,4,12,13,16,21,26],helper:[4,14],henc:[0,5,6,8,9,10,12,13,18,21,26,27,28,29],henrik:[24,26],her:7,here:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,26,27,28,29],hereaft:[0,8,12,26],hermitian:20,hessenberg:20,hessian:[0,2,5,13,16,28],heterogen:[9,10],hfft:[3,4,14],hi:7,hidden:[1,3,4,12],hidden_bia:1,hidden_bias_gradi:1,hidden_layer_s:[0,1,26],hidden_neuron:4,hidden_weight:1,hidden_weights_gradi:1,hierarch:[5,27,28],high:[0,1,2,3,4,5,6,9,10,11,13,14,19,20,21,26,27,29],higher:[0,1,3,5,6,8,13,21,26,27,28,29],highest:[1,2],highli:[0,3,4,10,18,19,20,25,26,27,28,29],highwai:[0,27],hing:8,hint:[13,16,27],hip:19,hire:[0,26],hist:[4,6,7,23,29],histogram:[0,6,7,23,27],histor:[7,11],histori:[3,4,12],histplot:27,hit:[],hitherto:[5,28],hjorth:[24,26,27,28,29],hm6ay5ds6o0:[],hobbi:23,hoc:[5,27],hoff:25,hold:[1,3,6,13,14,29],holder:[0,26],holomorphic_grad:[2,13],home:[0,27],homepag:[21,26],homework:[6,13],homogen:[1,3,9,10,13],honchar:2,hopefulli:[0,11,23,26],horizont:11,hors:[3,7],hot:[1,9],hour:[1,19,22,23,24,26,29],house_pric:27,how:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,26,27,28,29],howev:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,23,26,27,28,29],hs:[],hspace:[0,4,8,10,23,26],hstack:[1,27],htf:[22,26],html:[0,7,11,17,19,21,22,24,25,26,27],http:[0,3,4,6,7,11,13,14,17,19,20,21,22,24,25,26,27,28],huang:[0,26],huber:[0,26],huge:[1,3,4,19],human:[0,1,3,6,9,12,26,27],humid:9,hundr:1,hungri:1,hybrid:22,hydrogen:[0,26],hyperbol:[1,4,12],hyperparam:8,hyperparamet:[3,4,5,6,9,13,17,26,27,28],hyperplan:11,i0:[0,26],i1:[0,6,8,12,26,27],i2:[0,8,12,26],i3:[0,12,26],i5:[0,26],i:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,24,26,27,28,29],i_1:[5,6,28,29],i_2:[5,6,28,29],i_:13,ian:25,ic:1,id:[7,13],ida:[24,26],idea:[0,1,2,3,4,6,9,10,12,13,20,21,26,27,29],ideal:[0,2,6,8,13,23,26,27,29],idem:[6,29],ident:[5,6,12,13,17,20,21,27],identical:[28,29],identifi:[0,1,7,9,11,12,13,14,26,27],idum:[],ieor:23,ifft2:[3,4,14],ifft:[3,4,14],ifftn:[3,4,14],ifftshift:[3,4,14],ifi:25,ifs:[19,26],ignor:[0,1,3,9,27],ihfft:[3,4,14],ii:[20,23],iii:[20,26],ij:[0,1,3,6,8,12,14,16,18,20,21,23,26,27],ik:[0,20,26,27],illustr:[5,7,10,12,13,14,19,26,29],im:6,imag:[1,3,4,6,9,11,12,14,25,26],image_at_epoch_:4,image_batch:4,image_height:3,image_path:[0,6,7,9,26,27,29],image_width:3,imageio:[6,21],imagenet:26,images_from_seed_imag:4,imagin:1,immedi:[0,3,4,6,15,19,26],implement:[0,2,3,4,5,6,8,9,10,11,12,13,14,17,21,23,26,27,28],impli:[3,5,6,7,13,20,27,28,29],implicit:3,implicitli:[11,23],importantli:3,importerror:[3,4,14],impos:[0,6,11,12,26],imposs:[0,5,26,27],impress:[0,12,26],improv:[0,4,5,9,10,11,13,21,27],impur:9,imread:[6,21],imshow:[1,3,4,6,21],in1:[],in2:[],in3050:[25,26],in3310:26,in4080:[25,26],in4300:[25,26],in4310:25,in5400:3,in5550:25,in_out_neuron:4,inaccur:13,inacio:[],inact:12,inadequ:[0,26],inch:[6,27],includ:[0,1,2,3,4,5,6,7,11,12,15,16,17,19,21,23,24,25,26,27,28,29],include_bia:[6,9,29],inclus:17,incom:12,incorrect:1,incoveni:8,increas:[0,1,3,4,5,6,7,8,9,11,12,13,21,23,26,27,28,29],increasingli:23,ind:6,inde:[0,2,4,5,6,13,26,27],indefinit:4,indent:[],indentationerror:[],independ:[0,5,6,7,8,12,13,23,26,27],index:[0,1,3,4,10,14,19,20,23,25,26,27],index_col:[0,26],index_of:[],indic:[0,1,3,4,5,6,9,10,11,13,16,21,26,27],indispens:[6,29],individu:[1,6,7,10,12,23,26,27,29],indu:[0,27],indx1:2,indx2:2,indx3:2,indx:20,ineffici:[3,13],inequ:[8,13],inequaltii:[],inertia:13,inf1000:[19,26],inf1100:[19,26],inf1100l:[19,26],inf1110:[19,26],inf3000:26,inf4490:[],inf5860:[],infeas:9,infer:[0,1,4,6,25,26,29],infer_nrow:[0,26],inferenc:1,infil:[0,6,7,9,26,29],infin:[5,6,7,11,18,27,28,29],infinit:3,infinitesim:23,influenc:[6,10,29],influenti:1,info:26,inform:[0,1,3,4,6,9,11,12,13,14,20,21,25,26,29],infti:[3,6,13,23,29],ingeni:13,ingrad:2,ingredi:[0,9,26],ingular:[],inher:[6,29],inherit:[20,26],initi:[0,1,2,3,4,6,10,13,14,20,23,26,29],initial_epoch:[],initialis:[],initialize_root:[],inject:14,inlin:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],inner:[0,13,27],innov:[],inp:4,inplac:13,input:[0,1,3,4,5,6,7,8,9,10,12,13,14,15,21,23,26,27,28,29],input_dim:1,input_shap:[3,4],inputs:1,inputs_shuffl:[0,1,27],insert:[3,5,6,8,10,23,27,28,29],insid:[0,4,7,27],insight:[0,1,5,19,26,27,28,29],insist:[6,13,27],inspir:[0,1,12,26],instabl:2,instal:[0,1,5,6,9,15],instanc:[0,1,2,4,6,9,11,13,26,27,29],instanti:10,instead:[0,1,2,3,4,5,6,8,9,11,13,14,20,21,23,26,27,29],institut:1,instruct:[0,1,15,16,26],int32:10,int64:27,int64index:26,int_0:23,int_:[3,6,23,29],int_a:23,intak:[0,27],integ:[1,2,13,14,20,23,26],integer_vector:1,integr:[3,6,23,26,29],intellectu:26,intellig:[0,14,25,26],intend:[10,26],intens:1,intention:14,interact:[0,6,9,12,19,21,26],intercept:[0,6,8,11,13,16,21,26,29],intercept_:[0,6,8,9,13,26,27],interchang:[5,12,20,28,29],interconnect:1,interest:[0,1,2,3,4,5,6,7,8,9,12,19,21,23,26,27,28,29],interfac:[0,1,20,27],interior:[0,9,26],intermedi:[20,27],intern:[1,10,12],interpol:[1,3,4,6,12,21],interpr:[5,27,28],interpret:[0,1,3,4,6,9,10,12,13,14,17,20,21,23],interv:[0,3,5,6,7,13,18,21,23,26,27,28],intial:13,intract:[0,4,27],intrins:[3,11,20,23,26],intro:[19,25,26],introduc:[0,1,5,6,8,10,12,17,20,21,23,26,29],introduct:[1,2,4,13,17,25,27],introductori:[0,4,20,25,26,27],intuit:[0,5,6,8,12,13,21,26,28,29],inv:[0,5,13,26,27,28],invalid:[1,8],invalu:[0,13,15,19,26],invari:1,invd:[5,28],inver:8,invers:[0,3,6,13,15,16,17,21,26,27],invers_period:[],inverse_transform:8,invert:[0,5,7,10,13,26,28],invh:13,invok:[0,8,27],involv:[0,2,6,7,11,12,27,29],io:[0,17,19,21,22,24,25,26,27],iomanip:[],iostream:[],ip:[0,8,23,26],ipca:11,ipykernel_16145:[],ipykernel_16213:[],ipykernel_16261:[],ipykernel_16293:[],ipykernel_19350:[],ipykernel_20528:[],ipykernel_20614:[],ipykernel_20669:[],ipykernel_20702:[],ipykernel_41811:[],ipykernel_42331:[],ipykernel_42376:[],ipykernel_42449:[],ipykernel_42456:[],ipykernel_42530:[],ipykernel_42541:[],ipykernel_42553:[],ipykernel_42573:[],ipykernel_42580:[],ipykernel_42586:[],ipykernel_47411:[],ipykernel_47448:[],ipykernel_47647:[],ipykernel_47724:[],ipykernel_47735:[],ipykernel_65914:[],ipykernel_7604:1,ipykernel_7650:6,ipykernel_7714:13,ipykernel_7740:21,ipykernel_7752:26,ipykernel_7768:29,ipykernel_9414:[],ipykernel_94478:[],ipykernel_94529:[],ipykernel_94582:[],ipykernel_96694:[],ipynb:[19,26],ipython:[0,5,7,9,11,14,15,19,21,26,27],iq:[6,29],irfft2:[3,4,14],irfft:[3,4,14],irfftn:[3,4,14],iri:[8,9],irreduc:[6,29],irrelev:[5,27],irrespect:[0,26],is_integ:[],isbox:[2,13],iscomplexobj:2,isinst:[],isn:[5,28,29],isnul:[0,27],isomap:11,isotop:[26,29],issu:[1,3,4,9,14,20,27],it_arrai:13,item:[0,13,26],items:[20,26],iter:[1,2,4,6,7,8,11,13,14,23,29],itr:[],its:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,18,19,20,21,23,26,28,29],itself:[5,6,12,21,23,27,28,29],ix:[],j1:20,j:[0,1,2,3,4,5,6,8,9,11,12,13,14,16,17,18,20,21,23,25,26,27,28,29],j_:6,j_lasso_sk:6,j_ridge_sk:6,j_sk:6,jackknavg:[],jackknif:[6,19,26,29],jackknstd:[],jackknvar:[],jackknvec:[],jacobian:[2,13],jacobian_shap:2,jakobsen:[24,26],jargon:[],jason:4,jax:[3,4,14,19,26],jaxlib:[3,4,14],jensen:[24,26,27,28,29],jerom:[18,21,25],jhfgq77fpq:[],ji:[12,20],jit:13,jj:[0,5,6,26,28,29],jk:[0,1,6,12,20,26],jl:[0,26],jm:20,jnp:[3,4,13,14],joao:[],joaogca:[],job:[2,8,10],join:[0,4,6,7,9,26,27,29],joint:[4,5,28,29],journal:[],judg:13,judgement:[6,21],julia:[19,20,21],jump:[21,23],junk:4,jupit:26,jupyt:[0,15,19,21,25,26,29],just:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,21,23,26,27,28,29],justif:[0,26],justifi:[3,10],jy:[],jydg2xzka5:[],k0:7,k1:7,k:[0,1,3,5,6,7,8,9,10,11,12,13,14,19,20,21,23,24,26,27],kaggl:[6,21],kappa_d:23,karim:[],karl:[24,26],karlsen:[],karush:8,kate:26,katrin:[24,26],keep:[0,1,4,5,6,11,13,14,20,21,26,27,28,29],keepdim:[1,6,10,20,29],kei:[0,1,3,6,12,27],kept:[4,6,14,29],kera:[0,4,19,21,26],kernel:[0,1,3,19,26,27],kernel_regular:[1,3],kernel_s:4,kernelpca:11,kev:[0,26],kevin:[25,26],keyboardinterrupt:2,keyerror:[],keyword:[6,13,20,21,26],kfold:[6,29],kg:1,ki:20,kick:[1,13],kiener:2,kilomet:[6,27],kind:[0,2,3,4,8,12,13,14,26,27],kj:[6,12,20,27],kjm:[19,26],kkt:8,kktsolver:[],kl0m3:[],kl:23,km:[12,26],kmean:14,kmeanspoint:14,kn_k:14,know:[0,1,2,5,6,8,13,19,26,27,28],knowledg:[0,19,26],known:[1,3,4,5,6,7,8,9,12,20,21,23,25,27,28,29],kondev:[0,26],kp:23,kpca:11,kroneck:14,kuhn:8,kutz:[],kvalsund:[24,26],kwarg:[0,2,13,26],kwd:[0,26],kwown:[0,26],l0:7,l1:[0,1,3,7,26],l1_l2:[1,3],l1regl:5,l1regls_mosek2:[],l1regls_mosek:[],l2:[1,3],l:[0,1,2,3,5,6,7,8,10,11,12,13,20,21,23,26,27],l_1:7,l_2:[7,13],l_:20,l_j:12,la:13,la_i:12,la_k:12,lab:[19,21,26],label:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,19,20,21,23,26,27,28,29],label_prob:[],labelencod:[7,10],labels:[6,8,9],labels_shuffl:[0,1,27],laboratori:[22,27,28],lack:[0,26],lagari:2,lagrang:[8,11],lambda:[0,1,2,3,5,6,7,8,10,12,13,17,18,21,23,26,27,28,29],lambda_0:11,lambda_1:[5,8,11,27],lambda_2:[8,11],lambda_:11,lambda_i:[8,11],lambda_iy_i:8,lambda_jy_iy_j:8,lambda_k:8,lambda_n:[5,8,27],lamda:1,lamdbda:[],land:[0,8,27],landmark:8,landscap:13,langl:[0,6,11,23,26,27],languag:[0,1,4,8,15,19,20,21,25,26],lapack:[20,26],laplac:[5,28,29],laptop:19,larg:[0,1,2,4,5,6,8,9,10,11,13,15,19,20,21,23,25,26,27,29],larger:[0,3,5,6,8,10,11,13,17,23,26,27,28,29],largest:[4,8,11],lasso:[0,7,19,26],lasso_sk:6,last:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17,18,20,21,23,24,26,28],latent:4,latent_dim:4,latent_point:4,latent_space_value_rang:4,later:[0,1,4,6,7,8,12,13,14,15,19,21,26],latest:[4,19],latest_checkpoint:4,latex:26,latter:[0,3,6,7,8,11,13,16,17,20,21,23,26,27,29],lattic:12,law:[0,26],lax:[3,4,14],layer:[0,4,13,14,26],lbfg:[7,9,10,11],lcc:[5,6,28,29],lda:11,ldot:[0,6,11,21,26,29],le:[5,7,10,13,17,23,27,28],lead:[0,1,3,5,6,7,8,9,10,11,12,13,16,17,20,23,26,27,28,29],leaf:9,leaki:1,leakyrelu:4,lear:13,learn:[3,4,5,6,7,8,9,10,12,18,20,22,24,25],learnabl:3,learner:10,learnig:26,learning_r:[8,10],learning_rate_init:[0,1,26],learning_schedul:13,learnt:21,least:[0,2,7,8,10,11,15,16,17,19,20,23,29],leat:13,leav:[0,1,3,5,6,9,11,26,28,29],lectur:[0,1,5,10,11,12,13,15,17,19,20,21,22,24,25],lecturenot:[0,17,19,21,25,26],lecturenovember11:[],lecturenovember12:[],lecturenovember19:[],lecturenovember25:[],lecturenovember26:[],lecturenovember4:[],lecturenovember5:[],lectureoctober14:[],lectureoctober15:[],lectureoctober1:[],lectureoctober21:[],lectureoctober22:[],lectureoctober28:[],lectureoctober29:[],lectureoctober7:[],lectureoctober8:[],lectureseptember10:[],lectureseptember16firstpart:[],lectureseptember16secondpart:[],lectureseptember17:[],lectureseptember23:[],lectureseptember24:[],lectureseptember2:[],lectureseptember30:[],lectureseptember3:[],lectureseptember9:[],lecturethursdayaugust26:[],lecturethursdayaugust27:[],left:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,16,17,20,21,23,26,27,28,29],leftarrow:[8,12],legend:[0,2,3,4,5,6,7,8,9,10,13,26,27,28,29],len:[0,1,2,3,4,5,6,8,9,10,11,12,20,26,27,28,29],len_index:[0,26],length:[0,1,2,3,4,8,9,13,16,19,26,27],length_of_sequ:4,leq:[0,5,7,8,13,14,17,23,26,27,28],less:[0,1,3,4,5,6,8,9,13,19,23,26,27,28,29],lessen:1,let:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,23,26,27,28,29],letter:[0,16,20,23,26,27],level:[0,1,5,6,9,19,20,21,22,24,26,27,29],li:[8,11,26],lib:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],liblinear:[8,10],librari:[0,1,2,3,4,5,6,9,10,11,15,20,21,23,25,27,28],licens:[0,1,3,4,14,15,19,21,26,29],lie:[0,6,11,23,27,29],life:[0,1,8,12,26],lifetim:13,like:[0,1,2,3,4,5,6,7,9,10,11,12,13,15,16,19,20,21,23,26,27,28,29],likelihood:[0,1,5,9,26,27],lim_:23,lima:[24,26],limit:[0,5,6,7,8,11,12,20,21,26,27],lin_clf:8,lin_model:[0,27],lin_reg:9,linalg:[0,2,3,4,5,6,8,11,13,14,20,23,26,27,28],line1:8,line2:8,line2d:13,line3:8,line:[0,2,3,4,6,8,9,10,11,13,14,15,21,26,27,28,29],linear:[1,3,5,6,7,9,10,11,12,17,18,19,21,23,29],linear_model:[0,5,6,7,8,9,10,11,13,26,27,28,29],linear_regress:[6,29],linear_util:[3,4,14],linearli:[5,27],linearloc:[6,13,21],linearregress:[0,6,7,9,26,27,29],linearsvc:8,lineat:28,liner:[1,3],linerar:10,linewidth:[0,2,4,6,8,9,10,21,29],link:[0,4,9,12,19,21,24,26],linlag:[5,28],linpack:[20,26],linreg:[0,26],linspac:[0,2,3,4,6,8,9,10,13,15,16,20,23,26,27,29],linu:4,linuek:[],linux:[0,1,15,19,21,26],liquid:[0,26],list:[0,1,2,3,4,9,19,21,26,27],listcomp:2,listedcolormap:[9,10],lite:[3,4,14],lite_const:[3,4,14],literatur:[1,7,14,25,29],littl:[1,3,9,12],live:8,ll:[0,23,26,27],llc:[3,4,14],lle:[0,27],lloyd:[4,14],lmb:[0,2,5,6,28,29],lmbd:[0,1,3,26],lmbd_val:[0,1,3,26],lmbda:13,ln:[1,13],lo:[],load:[0,1,4,6,7,9,10,21,27],load_boston:[0,27],load_breast_canc:[1,7,9,10,11],load_data:[3,4],load_digit:[1,3],load_iri:[8,9],loc:[0,3,6,7,8,9,10,26,29],local:[0,1,2,3,7,12,13,26,27],locat:[2,3,8],lock:[],log10:[0,5,6,28,29],log1p:[],log:[0,1,2,3,4,5,6,7,9,10,11,13,14,20,21,26,27,28,29],log_:[0,26],log_clf:10,logarithm:[0,5,7,20,26,28,29],logbook:21,logic:[0,1,9,26],logist:[0,1,2,8,9,10,11,12,13,19,27],logistic_predict:[],logisticregress:[7,9,10,11],logit:7,logreg:[7,9,10,11],logspac:[0,1,3,5,6,26,28,29],longer:[2,3,8,10,14,20,23,26],loocv:[6,29],look:[0,1,2,3,4,5,6,7,8,9,10,11,13,18,20,21,23,26,27,28,29],lookup:[3,4,14],loop:[1,4,6,10,12,14,19,20,26,29],lose:1,loss:[0,1,3,4,5,6,7,8,10,11,13,20,21,26,28,29],loss_fil:4,lossfil:4,lost:4,lot:[0,1,4,6,27,29],low:[0,6,9,10,11,21,26,27,29],lower:[0,1,3,6,9,10,16,20,27],lowercas:[20,26],lowest:[9,13,23],lr:[1,3,4,10],lstat:[0,27],lstm:4,lstm_2layer:4,lstsq:[0,26,27],lt:[6,29],lu:[0,3,4,5,14,26,27],lubksb:20,luckili:2,ludcmp:20,lux:20,lvert:1,lw:[0,26],m:[0,1,2,3,5,6,8,9,10,11,12,13,20,22,23,24,25,26,27,28,29],m_1:14,m_:[9,12],m_h:[0,26],m_k:14,m_l:12,m_n:[0,26],m_p:[0,26],m_t:13,ma:11,mac:[3,4,14],machin:[1,3,4,5,6,7,9,10,11,12,14,15,20,22,25,27,29],machinelearn:[0,6,17,19,21,22,24,25,26,27],machinelearningmurphi:[],mackai:25,made:[0,1,3,4,5,6,7,9,11,12,18,21,26,27],mae:[0,26],magic:4,magnitud:[1,6,7,13,27],mai:[0,1,2,3,5,6,7,8,9,11,12,13,18,19,20,21,23,27,28,29],mail:[22,24],main:[0,1,3,4,5,6,7,9,20,21,25,26,27],mainli:[0,5,6,7,9,26,27,28,29],maintain:[6,27,29],major:[1,6,9,10,13,20,26,29],make:[1,2,3,4,5,6,7,8,11,12,13,19,20,21,23,25,28,29],make_axes_locat:6,make_moon:[8,9,10],make_pipelin:[0,6,10,27,29],make_vjp:[2,13],makedir:[0,6,7,9,26,27,29],makeplot:[0,26],malcondit:20,malign:[1,7,9],mammographi:[5,28,29],manag:[0,2,3,15,19,21,26],mandatori:[24,26],mani:[0,1,3,4,5,6,7,8,9,11,13,14,19,20,21,23,25,26,27,28,29],manifold:11,manner:3,manual:[6,27],map:[0,1,2,3,4,6,7,8,11,12,14,21,23,26],margin:[0,5,8,26],mari:[],marit:[0,26],mark:26,marker:[0,7,20,26,27],markov:[19,26],marsaglia:23,mask:[],mass:[0,1,5,13,27],massag:[0,26],masses2016:[0,26],masses2016ol:[0,26],masses2016tre:0,masseval2016:[0,26],master:[6,21,22,24,26],mat1100:[19,26],mat1110:[19,26],mat1120:[19,26],mat3155:[],mat4155:[],mat:[19,26],match:[0,1,4,5,13,14,26,27],materi:[4,5,7,13,17,20,24],math:[3,7,12,13,20,23,25,26],mathbb:[0,4,5,6,7,8,11,12,13,14,17,18,20,21,23,26,27,28,29],mathbf:[0,5,6,7,8,13,18,20,21,26,27,28,29],mathcal:[1,5,6,7,13,21,28,29],matheemat:3,mathemat:[0,6,11,12,13,17,19,20,23,25,26],mathemati:26,mathrm:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,17,18,21,23,26,27,28,29],matmul:[1,2,5,28],matmul_adjoint_1:2,matmul_vjp_0:[],matmul_vjp_1:2,matnat:25,matplotlib:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,26,27,28,29],matplotlibdeprecationwarn:[6,13,21],matric:[0,1,3,4,6,7,8,11,13,16,17,19,27],matrix:[0,2,3,4,6,7,8,10,13,15,16,17,18,21,23,29],matshow:1,matter:[2,3,13,27],max:[0,1,2,3,4,9,10,12,13,24,26],max_depth:[0,9,10],max_diff1:2,max_diff2:2,max_diff:2,max_it:[0,1,7,8,11,13,26],max_iter:14,max_leaf_nod:10,max_queue_s:[],max_sampl:10,maxdegre:[0,6,10,27,29],maxdepth:10,maxim:[1,4,5,7,8,11,28,29],maximum:[0,1,2,3,5,7,8,9,10,13,14,26,27],maxpolydegre:[5,6,28,29],maxpooling2d:3,mbox:[5,6,18,21,27,28,29],mc:[],mccorduck:26,mcculloch:12,mcint:[],mcintsqr2:[],md:[11,21],mdoel:4,mdyt6uwokt0:[],mean:[1,2,3,4,5,6,7,9,10,11,12,13,14,15,16,17,18,19,20,21,23,29],mean_absolute_error:[0,26],mean_divisor:14,mean_i:23,mean_matrix:14,mean_squared_error:[0,4,6,7,10,26,27,29],mean_squared_log_error:[0,26],mean_vector:14,mean_x:23,meaning:[0,4,7,26],meansquarederror:[0,26],meant:[2,3,7,10,13],meantempvec:[],meanvec:[],measur:[0,1,2,5,6,9,11,12,14,21,23,26,27,28,29],mechan:[0,4,23,26],median:[0,26,27],medicin:12,medium:[4,8,13],medv:[0,27],meet:[0,24],mehta:[0,26,27,28],memori:[3,4,11,12,13,20,26],mention:[0,12,13,21,23,26],mere:[0,15,26],mersienn:[],meshgrid:[2,5,6,8,9,10,11,21,27],messag:[5,13],messi:2,met:[0,3,8,26,27],metadata:2,meteorolog:9,meter:[6,27],method:[0,1,2,3,4,5,7,8,11,12,14,16,17,18,19,20,23,25,26,27,28],metion:[6,21],metric:[0,1,3,4,6,7,9,10,14,15,16,26,27,29],metropoli:[19,26],mev:[0,23,26],mgd:13,mglearn:[19,26],mgrid:13,mhjensen:[1,2,3,4,6,7,8,11,14,27],mi:10,mia:[24,26],michigan:[26,27,28,29],microsoft:25,mid:1,midel:4,midnight:[16,17,18],midpoint:9,might:[0,1,2,4,6,9,13,27],mild:9,miller:[],millimet:[6,27],million:[0,26,27],mimic:12,min:[0,2,5,8,9,26],min_:[0,2,5,14,17,26,27,28],min_samples_leaf:9,mind:[0,6,13,21,26,27,29],mindboard:4,mine:[19,26],mini:[1,11,12,13],minibatch:[1,11,13],minibathc:13,miniforge3:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],minim:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,21,27,28,29],minima:[0,1,7,13,26],minimum:[0,1,2,6,8,9,11,13,27,29],minmaxscal:[0,27],minor:[6,13,21,23],minst:1,minu:7,mirror:9,misc:[6,21],misclassif:[8,9,10],misclassifi:[8,10],miser:0,mismatch:1,miss:[0,7,10,27],mistak:4,mit:25,mix:[1,2,26],mixtur:13,mk48pfcxgyk:[],mk:[9,20],mkdir:[0,6,7,9,26,27,29],ml:[0,1,10,13,20,21,27],mlab:23,mle:[5,7],mline:[],mlir:[],mlir_graph_optimization_pass:[],mlp:1,mlpclassifi:1,mlpregressor:[0,26],mm:20,mn:[12,23],mnist:[1,11],mod:23,mode:[22,24,26],model:[2,3,5,7,8,9,10,11,13,14,15,16,18,19,21,23,25,27,28,29],model_select:[0,1,3,5,6,7,9,10,11,26,27,28,29],modelanalyz:[3,4,14],moder:10,modern:[0,6,7,19,26,29],modif:[2,12,13],modifi:[0,1,3,5,7,8,10,12,13,26,27,28],modul:[0,3,4,7,11,14,20,26,28],modular:23,modulenotfounderror:[],modulo:23,moe:[11,27],moment:[5,6,13,23,28,29],monitor:13,monoton:[5,12,23,28,29],mont:[0,6,19,23,25,26,29],montecarlocycl:[],moor:[5,6,28],more:[0,1,2,4,5,7,8,9,10,11,12,13,14,18,19,23],moreov:[0,3,26],morten:[24,26,27,28,29],mortenimac:[],mosek:[],most:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,26,27,28,29],mostli:[1,11],motion:[0,13,26],motiv:[1,4],move:[0,4,5,6,7,9,12,13,14,21,23,26,27,28,29],mp4:[],mpl:[0,7,26],mpl_toolkit:[2,6,13,21],mplot3d:[2,6,13,21],mplregressor:1,mse:[0,4,5,6,9,10,15,16,17,21,26,27,28,29],mse_simpletre:10,mselassopredict:[5,28],mselassotrain:[5,28],mseownridgepredict:6,msepredict:[5,28],mseridgepredict:[0,5,6,28],msetrain:[5,28],msg:[0,26,27],msle:[0,26],mt19937_64:[],mt:[7,12],mu0:23,mu1:23,mu2:23,mu:[0,6,11,13,23,26,29],mu_1:27,mu_:[6,23,27,29],mu_i:[6,27],mu_n:11,mu_x:23,much:[0,1,2,3,4,5,6,8,9,10,11,12,13,20,21,23,26,27,28,29],mul:[],multi:[0,1,3,7,19,26,27],multiclass:[1,7],multidimension:[11,12,26],multilay:1,multinomi:7,multipl:[2,4,5,6,7,12,13,23,27,29],multipli:[3,5,6,11,13,20,23,27,28],multiplum:8,multivari:[0,2,10,11,19,23,26],multivariate_norm:[11,14],murphi:[11,25,26,28],must:[0,1,2,5,6,8,10,12,13,14,23,26,27,29],mutat:7,mutual:[1,3,6,13,29],mx_:23,myenv:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],myriad:[0,15,19,26],mz1:23,mz2:23,n1:20,n2:20,n:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],n_0:[12,23],n_:[1,2,3,8,12,23],n_b:[],n_boostrap:[6,10,29],n_bootstrap:[6,29],n_categori:[1,3],n_cluster:14,n_compon:11,n_epoch:13,n_estim:10,n_examples_to_gener:4,n_featur:1,n_filter:3,n_hidden:2,n_hidden_neuron:[0,1,26],n_i:23,n_input:[0,1,3,27],n_instanc:9,n_iter_i:[7,11],n_job:10,n_k:14,n_l:[12,23],n_layer:1,n_m:9,n_neuron:1,n_neurons_connect:3,n_neurons_layer1:1,n_neurons_layer2:1,n_point:14,n_sampl:[6,8,9,10,14,29],n_split:[6,29],n_step:4,n_t:2,n_x:2,nabla:[1,13],nabla_:[2,13],nabla_w:13,nag:13,naimi:[0,26],naiv:7,naive_kmean:14,najafi:[24,26],nall:[0,26],name:[0,1,3,4,5,6,7,8,9,10,12,13,14,15,19,20,21,23,24,26,27,28,29],nameerror:[6,10,15,21],namespac:[3,4,14],narrow:13,nary_f:[2,13],nary_op_arg:[2,13],nary_op_kwarg:[2,13],nary_oper:[2,13],nation:[1,5,27,28,29],nativ:[19,26],natur:[0,1,4,8,9,12,13,21,23,25,26],navier:12,nb:23,nb_:20,nbconvert:26,nboot:[],nd:14,ndarrai:[2,6],ndef:13,ndim:2,ne:[9,10,20,23,27],ne_xcl2ctm0:[],nearest:[1,3,6,11],nearli:13,neat:26,neccesari:[6,29],necess:2,necessari:[0,1,3,4,8,14,26],necessarili:[0,4,11,23,26],necesserali:[5,28,29],neck:7,need:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,23,27,28,29],neg:[0,1,3,5,6,7,10,13,20,23,26,28,29],neg_mean_squared_error:[6,29],neglect:23,neglig:23,neighbor:[3,6,11],neither:[4,13],neq:[13,14,23],nervou:12,nest:[2,9,12],nesterov:13,net:[2,4,12],netlib:[20,26],network:[0,9,13,19,25,27],neural:[0,13,19,25,27],neural_network:[0,1,2,26],neuralnetwork:1,neuron:[1,2,3,4,12],neutral:[0,26],neutron:[0,26],never:[1,4,6,9,23,29],new_box:[2,13],new_chang:13,new_hobbit:26,new_root:[2,13],new_trac:[2,13],new_tracing_count:[],newaxi:[0,3,6,9,29],newli:[0,26],newton:[1,7,8,13,23],next:[0,1,2,3,4,5,6,8,9,13,14,26,27,28],next_guess:13,next_input:4,nf8_grad:13,nfrom:13,ng:1,ngini:9,ni:14,nian:[],nice:[0,1,5,11,26,27,28,29],nichola:[],nicholaskarlsen1102:[],nimport:13,niter:13,nitric:[0,27],nlambda:[0,5,6,28,29],nlevel:[],nlp:25,nm:23,nm_n:[0,26],nmse:[6,29],nn:[2,5,6,12,20,28,29],nn_model:1,nnmin:2,node:[1,2,3,9,10,12],node_constructor:2,nois:[0,4,5,6,8,9,10,13,15,16,21,26,27,28,29],noise_dimens:4,noisi:[1,6,21,29],non:[0,1,3,5,6,7,9,10,11,12,13,14,20,21,23,26,27,28,29],noncommerci:29,none:[0,1,2,3,4,5,9,10,13,14,23,26,27,28,29],nonlinear:[3,6,8,9,11,12,29],nonneg:[6,9,13,29],nonparametr:6,nonsens:23,nonsingular:20,nonumb:[3,7,8,13,20],noqa:[3,4,14],nor:[1,4,13],norm:[0,1,5,6,8,11,13,17,26,27,28,29],normal:[3,4,5,6,7,8,9,10,11,12,13,15,16,18,19,20,21,23,26,27,28],normali:[20,26],normalize_kwarg:[],norvig:26,norwai:[6,21,26],notat:[0,2,5,6,13,14,23,26,27,28,29],note:[0,1,2,3,4,5,6,7,8,11,12,13,14,15,16,19,20,21,23,25,26,29],notebook:[0,1,3,9,15,19,21,26,29],notesexercise5week452022:26,noth:[1,2,5,8,12,14,23,27,28],notic:[4,5,12,13,20,23,26,28],notimplementederror:2,notion:3,notrace_primit:2,novel:[3,6,10],novemb:[1,22,24,26],now:[0,2,4,5,6,7,8,10,11,12,14,15,16,17,19,20,21,23,26,27],nowadai:[0,1,3,9,19,26],nox:[0,27],np:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,20,21,23,26,27,28,29],npr:2,nprint:13,nsampl:[6,9,29],nspin:[],nt:2,nthi:[0,26],ntrained_model:6,nu:23,nuclear:[5,27],nuclei:[0,23,26],nucleon:[0,26],nucleu:[0,26],num:4,num_allow_arg:[0,26],num_coordin:2,num_hidden_neuron:2,num_it:2,num_neuron:2,num_neurons_hidden:2,num_output:[],num_point:2,num_tre:10,num_valu:2,number:[1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,20,21,22,24,26,28,29],numberid:7,numberparamet:3,numer:[0,5,6,9,10,11,12,13,17,19,20,25,26,27,28,29],numpi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,21,23,27,28,29],numpy_box:[],numpy_rank_promot:[3,4,14],numpy_vjp:2,numpy_wrapp:2,nunmpi:[5,27],nvalu:9,nx:[2,13],nx_test:6,nx_train:6,nx_train_mean:6,ny:23,ny_pr:6,ny_train:6,ny_train_mean:6,o:[0,6,7,8,9,11,15,20,24,25,26,27],obei:[6,11,13,27],object:[0,1,2,4,6,8,10,13,20,26,27],objsens:[],obliqu:[5,27],observ:[0,1,3,5,6,7,8,9,10,11,12,13,14,23,28,29],obtain:[0,1,5,6,7,8,9,10,12,13,14,17,20,21,23,26,27,28,29],obviou:[5,6,11,23,27],obviouli:26,obvious:[0,4,5,6,20,26,28,29],oc:27,occupi:[0,27],occur:[0,6,8,9,20,23,26],octob:[22,24,26],od:0,odd:[0,3,7,26,27],odenum:2,odesi:2,oen:0,off:[1,3,4,5,9,13,23,28,29],offer:[6,11,19,20,22,24,26,29],offic:[24,26],offici:[22,26],ofil:[],ofstream:[],often:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,16,19,20,23,26,27,29],ofter:[20,26],oh6i_oscwpc:[],ol:[0,13,15,16,17,18,26,27],old:[1,5,10,13,28,29],ols_sk:6,ols_svd:6,olsbeta:[0,5,28],olxksel3p4a:[],omega:[2,3,6],omega_0:3,omit:[0,5,26,27,28,29],on_train_batch_begin:[],onc:[1,6,9,11,13,29],one:[0,1,3,4,5,6,7,8,9,10,11,13,14,19,20,21,23,24,26,27,28,29],oneapi:[],onednn:[],onehot:1,onehot_vector:1,onehotencod:9,ones:[0,2,5,6,8,9,10,11,13,20,21,26,27,28,29],ones_lik:4,onl:3,onli:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,23,26,27,28,29],onlin:[11,22],onto:[5,11,27],op_nam:[],opbdi:[],open:[0,1,4,6,7,9,15,19,21,22,24,26,29],oper:[0,1,3,4,5,6,10,11,12,13,14,15,19,23,26,27,28,29],operation:23,ophint:[3,4,14],oplu:23,opmiz:13,opportun:[0,26],oppos:[6,13],opposit:[1,5,8,27],opresolvertyp:[3,4,14],opt:[1,5,26,28],optim:[0,2,3,4,5,6,7,9,10,11,14,15,16,17,18,21,28,29],optimis:[1,3],optimizer_v2:[],option:[0,1,3,5,6,7,8,11,20,21,27,28,29],optionalxlacontext:[],optmiz:[1,8,13,27],oral:26,orang:0,order:[0,1,2,3,5,6,7,8,9,10,11,12,15,16,20,21,23,26,27,28,29],ordinari:[0,2,3,7,11,13,15,16,17,19,29],oreilli:25,org:[0,3,4,7,11,19,20,25,26],organ:[6,7,10,20,29],orient:[1,5,23,27,28],origin:[0,3,5,6,8,11,12,13,20,26,27,29],orthogn:[5,27],orthogon:[0,5,6,8,11,13,17,20,26,27,28],orthonorm:[5,27,28],os:[0,1,4,5,6,7,8,9,24,26,27,28,29],oscar:1,oscil:[3,13],oslo:[0,15,19,21,22,24,26,27,28,29],osx:[0,15,19,21,26],other:[0,1,2,3,5,6,7,8,10,13,14,15,19,21,22,23,24,25,28,29],otherwis:[0,1,4,7,13,20,26,27],ouput:[5,7,12,28,29],our:[1,2,3,6,7,8,9,10,12,14,15,16,17,18,19,20,21,23,28,29],ourmodel:0,ourselv:[0,5,6,8,11,13,26,27,28,29],out1:[],out2:[],out:[0,1,2,4,5,6,7,8,9,10,11,12,13,16,19,20,21,23,26,27,28,29],out_fil:9,outcom:[0,7,9,10,12,23,27],outdoor:9,outer:[6,12,13],outfil:4,outfilenam:[],outgrad:2,outlier:[0,8,26,27],outlin:[6,10,11,29],outlook:9,outperform:10,output:[0,1,3,4,5,6,7,8,9,10,12,13,16,20,21,23,26,27,28,29],output_bia:1,output_bias_gradi:1,output_shap:4,output_weight:1,output_weights_gradi:1,outputlayer1:12,outputlayer2:12,outsid:4,over1:13,over:[0,1,3,4,5,6,9,10,12,13,20,21,26,27,28,29],overal:[1,10],overcast:9,overcom:[12,13],overdetermin:[0,26],overfit:[0,1,3,6,9,10,13,29],overflow:[1,5,28,29],overhead:12,overlap:[3,7,8,9],overlin:[0,5,6,9,10,11,14,20,26,27],overst:[0,26],overtrain:4,overview:3,own:[4,5,6,8,12,13,19,20,21,28],owner:[0,27],ownmsepredict:0,ownmsetrain:0,ownridgebeta:[0,6],ownypredictridg:0,ownytilderidg:0,oxid:[0,27],oyvinssc:[],p0:2,p1:2,p:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,17,20,23,26,27,28,29],p_:[2,4,8,9],p_hidden:2,p_i:[5,23,28,29],p_j:23,p_n:23,p_output:2,p_x:23,pack:[0,26],packag:[0,1,2,3,4,5,6,7,8,11,13,14,15,19,21,23,27,28],pad:[3,4],page:[0,19,26],pai:[0,1,9,13],pair:[0,2,3,9,19,23,26,27],pamilla:26,panda:[0,4,5,6,7,9,11,15,19,21,28,29],panel:26,paper:1,paradigm:[0,26],parallel:[3,4,10,13,14,19,20,26],param:2,paramat:2,paramet:[0,1,2,3,4,5,6,7,8,9,10,12,13,16,17,18,21,23,28,29],parameter:[0,6,10,21,26,27],parametr:[0,6,15,16,26,27,29],paramt:[3,5,28,29],parent:2,parent_argnum:2,park:[],parser:[0,26],part:[0,1,3,5,6,10,17,18,20,22,23,24,26,28,29],partial:[0,1,3,4,5,6,7,8,10,11,12,13,14,16,23,26,27,28],partialmethod:[3,4,14],particip:[19,22,24,26],particl:[0,4,13,23,26],particular:[0,1,2,3,5,6,9,10,11,12,13,16,21,23,25,26,27,28,29],particularli:[5,6,8,11,13,23,27,29],partit:[1,4,9],partli:[6,26],pass:[2,3,12,14],password:21,past:[10,23],patch:[6,23,29],path:[0,4,6,7,9,15,19,26,27,29],patient:7,patter:4,pattern:[0,3,4,12,22,25,26],pauli:[0,26],pc:[11,19],pca:[0,7,19,26,27],pcolor:6,pcolormesh:6,pd:[0,4,5,6,7,9,11,26,27,28,29],pde:2,pdf:[0,3,4,5,6,9,21,25,26,28,29],pedagog:[0,26,27],penal:[6,27],penalti:[6,13,21,27],penros:[5,6,28],pentagon:13,peopl:[0,1,9,13,19,27],per:[0,1,6,22,24,26,27,29],percentag:[0,10,11,24,27],perceptron:[0,1,7,26],peregrin:26,perfect:[0,1,13,26],perfectli:[4,6,29],perform:[0,2,3,4,5,6,8,10,11,12,13,14,15,16,17,19,20,21,23,26,27,28,29],performac:4,perhap:[0,5,13,26,27,28],perimet:[1,9],period:[1,4,23],permut:11,persist:13,person:[5,6,7,22,24,26,27,28,29],perspect:25,pertin:12,petal:[8,9],peter:25,petersen:[],phantom:23,phase:[6,12],phawkin:[3,4,14],phenomena:23,phi:8,phi_k:8,philip:[],philosophi:13,phone:[24,26],photo:4,phrase:[0,26],physic:[0,1,4,7,12,13,23,24,25,26,27,28,29],pi:[2,3,5,6,7,9,12,13,23,28,29],pick:[1,9,10,11,13,14],pickl:1,pictur:[0,26],pie:[19,26],piec:[11,14],pillow:[0,15,19,21,26],pinv:[5,6,13,27,28],pip3:[0,1,15,21,26],pip:[0,1,15,19,21,26],pipelin:[0,6,8,10,27,29],pippin:26,pit:4,pitfal:[6,27],pitt:12,pixel:[1,3,4],pixel_height:[1,3],pixel_width:[1,3],place:[0,4,6,8,13,20,21,26,29],plai:[0,3,4,5,6,8,11,19,26,27,28,29],plain:[8,10,12,13,14],plan:[6,9,24,25,26],plane:[8,9],plateau:[5,28],platform:[19,26],plausibl:12,pleas:[3,4,6,7,11,13,14,21,24,26,27],plenti:1,plethora:[3,12],plot:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,23,26,27,28],plot_confusion_matrix:[7,10],plot_count:6,plot_cumulative_gain:[7,10],plot_data:1,plot_dataset:8,plot_decision_boundari:[9,10],plot_import:10,plot_max:4,plot_min:4,plot_model:4,plot_numb:4,plot_predict:8,plot_regression_predict:9,plot_result:4,plot_roc:[7,10],plot_surfac:[2,6,13,21],plot_train:9,plot_tre:[9,10],plt:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],plu:[0,3,5,7,26,27,28],pm:[8,29],pmatrix:2,pml:25,pn:3,png:[0,4,6,7,9,26,27,29],pocketfft:[3,4,14],point:[0,1,2,3,5,6,7,8,9,10,11,13,14,15,16,17,20,21,23,24,26,27,29],point_1:4,point_2:4,poisson:[19,23,26],poli:[6,8,29],pollut:[3,4,14],poly100_kernel_svm_clf:8,poly3:0,poly3_plot:0,poly3dcollect:13,poly_featur:[8,9],poly_features10:9,poly_fit10:9,poly_fit:9,poly_kernel_svm_clf:8,polydegre:[0,5,6,10,27,28,29],polygon:13,polym:12,polymi:21,polynomi:[0,5,6,7,8,9,10,11,15,16,17,21,26,27,29],polynomial_featur:[6,29],polynomial_svm_clf:8,polynomialfeatur:[0,6,8,9,27,29],polytrop:[0,6,26,29],pool:3,pool_siz:3,poor:[1,13],poorli:[0,27],pop:2,popul:[0,5,26,27,28,29],popular:[0,1,3,6,7,8,9,11,12,15,19,20,21,23,26,27],popularli:[0,26],portabl:10,portion:[11,13],pose:[0,4,5,6,11,23,26,29],posit:[0,1,2,3,5,7,8,10,11,13,14,17,20,23,26,27,28,29],possibl:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,15,19,20,21,23,24,26,27,28,29],posterior:[5,28,29],postpon:[0,26,27],postscript:21,postul:[5,28,29],potenti:[0,3,5,6,12,13,26,27,28,29],potr:[],potrf:[],pott:12,power:[0,1,5,6,8,9,12,13,26,27,29],pp:[5,6,18,28,29],practic:[0,5,6,7,8,21,23,27,28,29],practition:[0,1,3,26],pre:26,preced:[1,11,12,23],preceed:4,preceq:8,precis:[0,2,5,11,13,20,21,23,26,27,28,29],pred:[6,29],predicit:0,predict:[0,1,5,6,7,8,9,10,15,16,19,21,25,26,27,28,29],predict_prob:1,predict_proba:[7,10],predictor:[0,5,6,7,9,10,11,26,27],prefer:[0,1,6,8,9,11,13,15,19,21,26],prepar:[0,6,20,21,26,27],preprocess:[0,4,6,7,8,9,10,11,29],prerequisit:0,prescript:21,presenc:13,present:[0,5,6,7,9,12,13,20,21,23,26,27,28],preserv:[3,11,20],press:[13,25],pretrain:[1,4],pretti:[0,4,8,9,15,19,21,26],prev_centroid:14,prevent:[13,23],previou:[0,1,2,3,4,5,6,8,10,11,12,13,20,21,23,27],previous:[2,3,9,10,23],price:[0,4,9,13,27],primal:8,primari:[0,7,26],prime:23,primit:2,primitive_batch:[3,4,14],princip:[0,5,7,19,26,27],principl:[0,6,7,8,14,26,29],print:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,20,23,26,27,28,29],print_funct:[8,9],printout:[0,26],prior:[0,5,6,26,27,28,29],privat:[0,26],prob:[1,23],probabilist:[0,25,26,27],probabl:[0,1,3,4,6,7,10,13,19,26,27],problem:[0,3,4,5,6,7,8,9,10,11,12,14,16,17,19,20,21,23,29],probml:25,proce:[0,5,6,7,8,9,10,11,13,20,26,27,28,29],procedur:[2,4,5,6,8,10,11,13,21,27,29],proceed:20,process:[0,2,4,6,9,10,12,13,15,19,20,21,23,25,26,29],prod:25,prod_:[1,5,7,28,29],produc:[0,3,4,5,6,9,10,11,12,13,19,20,21,23,26,27,28,29],product:[0,1,3,5,6,7,8,12,13,16,19,20,26,27,28,29],profess:[0,26],program:[0,1,4,5,6,8,12,14,15,16,19,20,22,23,24,26,27],programm:20,progress:[1,4,14],prohibit:[6,29],project1:[6,21],project:[0,1,2,3,5,11,13,17,18,19,22,24,27,28,29],project_root_dir:[0,6,7,9,26,27,29],promin:12,promis:8,promot:[24,26],prone:9,pronounc:[13,19,26],proof:[0,11,12,13,26,29],propag:[2,3,13],proper:[0,2,6,7,21,26,29],properli:[1,6,8,10,13,21],properti:[0,1,3,12,13,16,17,20,26,29],proport:[0,1,5,9,11,13,16,23,26,27],propos:[1,4,6,10,21,26],propto:[5,13,28,29],protect:[],protobuf:[],proton:[0,26],prove:[3,13],provid:[0,1,3,4,5,6,8,9,10,12,13,15,19,20,21,23,26,27,28,29],proxi:[1,13],prune:9,pseudo:[20,23],pseudocod:21,pseudoinv:[5,28],pseudoinvers:[5,6,28],pseudorandom:[6,23,29],psycholog:[0,26],pt:13,ptratio:27,punish:[0,1,26],pure:[3,9,23],purest:9,puriti:9,purpos:[0,3,10,12,14,26,27],put:[1,26],putarow:[],putboundslic:[],putclist:[],putobjsens:[],putqobj:[],py:[0,1,2,3,4,5,6,7,8,11,13,14,21,26,27,28,29],pycod:26,pydata:19,pydot:9,pyhton2:26,pylab:[0,7,26],pylint:[3,4,14],pypi:19,pyplot:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],pythagora:[5,28,29],python2:[0,15,21,26],python3:[0,1,2,3,4,6,7,8,11,13,14,15,19,21,26,27,28],python:[1,2,3,4,5,6,8,11,12,13,14,16,21,23,27],pytorch:[0,19,21,26],pywrap_tf:[],q:[5,6,8,11,23,27,29],qn_bavhmd8u:[],qp:8,qquad:[2,11,13,20],qr:[5,6,20,27],quad:[1,13,20],quadrat:[0,8,9,13,15,16,26],qualit:[4,9,21,23],qualiti:[0,9,15,16,19,26,27],quantifi:1,quantil:10,quantit:[0,6,9,21,26,29],quantiti:[0,2,5,6,7,9,10,11,12,14,16,20,23,26,27,28,29],quantum:[4,12,25,26],quartil:[0,27],quench:5,queri:9,question:[0,5,6,9,11,12,13,21,24,26,27,28,29],qugan:4,quick:[4,23],quick_execut:[],quickli:[1,3,9,11,13],quit:[1,5,6,9,10,12,27,29],quot:[4,26],r2:[0,5,6,26,27,28],r2_score:[0,26,27],r2score:[0,26],r:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,27,28,29],r_1:9,r_2:9,r_j:9,r_m:9,rad:[0,27],radial:[0,8,12,27],radioact:23,radiu:[0,1,9,27],rag:2,rain:9,rais:[0,2,13,26,28],ramp:1,ran0:23,ran1:23,ran2:23,ran3:23,rand:[0,4,5,6,9,10,13,15,16,20,26,27,28,29],rand_max:[],randint:[6,9,13,29],randn:[0,1,2,5,6,9,11,13,15,16,26,27,28,29],random:[0,1,2,3,4,5,6,8,9,13,14,15,16,19,20,21,26,27,28,29],random_devic:[],random_forest_model:10,random_index:13,random_indic:[1,3],random_st:[0,7,8,9,10,11,27],randomforestclassifi:10,randomli:[1,6,9,13,14,29],randomnumbergener:[],rang:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,20,23,26,27,28,29],rangl:[0,6,11,23,26,27],rangle_x:23,rank:[5,27],rankdir:4,raphson:[1,8,13],rapidli:[0,26],rare:[1,13,26,29],rate:[0,1,2,3,4,8,9,10,12,13,27],rather:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,23,26,27,28,29],ratio:[4,7,9,10,11],rational:[0,26],ravel:[5,6,7,8,9,10,11,13,20,27,29],raw:3,raw_df:27,rbahrf:[],rbf:[8,11,12],rbf_kernel_svm_clf:8,rbf_pca:11,rc:[0,23,27],rcond:[0,26,27],rcparam:[0,1,3,7,8,9,10,23,26],rd:[],rdbj50lv3go:[],re:[2,4,13],reach:[1,4,5,6,7,9,10,11,12,13,14,28,29],read:[0,2,3,4,5,6,7,8,11,12,16,20,21,22,23,25,28],read_csv:[0,6,7,9,26,27,29],read_fwf:[0,26],reader:[0,6,20,23,26,27],readi:[0,1,5,6,8,10,11,12,20,21,26,28,29],readili:1,readthedoc:19,real:[0,1,2,4,7,10,11,12,13,20,27,29],real_loss:4,real_output:4,realist:8,realiti:23,realiz:[1,12],realli:[0,1,26],rearrang:13,reason:[0,1,3,4,10,13,25,26],reassign:1,rebuild:[],recal:[5,6,9,10,11,12,20,23,26,27,28,29],recalcul:[],recast:3,receiv:[1,3,10,12,23],recent:[0,2,3,4,6,9,10,13,14,15,21,25,26,28,29],recept:[3,12],receptive_field:3,recip:[0,6,7,20,21,26,27],reciproc:[5,28],recogn:[0,4,5,10,26,28,29],recognit:[0,1,3,12,22,25,26],recommend:[0,2,3,4,5,6,8,13,15,18,19,20,21,25,28,29],reconsid:9,reconstruct:11,record:[10,21,22,24,26],recreat:[],rectangl:[9,13],rectangular:[5,27],rectifi:[1,3,12],recur:[0,19,26],recurr:[0,1,19,26],recurs:[9,19,20,26],recycl:[],red:[0,3,4,6,8,9,29],redefin:[0,10,26,27],redefinit:28,reduc:[1,3,5,6,9,10,11,13,26,28,29],reduct:[0,10,11,19,23,26,27],refer:[0,1,2,3,5,6,7,11,12,13,14,20,21,25,26,27,29],referenc:2,refin:12,refit:[6,29],reflect:[0,1,4,5,21,23,26],refresh:[19,26],refreshprogrammingskil:26,reg:[10,11],regard:[1,9,13],regardless:12,region:[3,4,6,9,12,21],regist:[6,21,23],reglasso:[5,28],regr_1:[0,9],regr_2:[0,9],regr_3:[0,9],regress:[1,8,11,12,15,16,19,20],regressor:[0,7,10,26],regridg:[0,5,6,28],regular:[0,3,4,5,6,7,9,13,17,24,26,27,28,29],regularis:6,reilli:[0,15,25,26],reinforc:[0,8,19,26],reiter:1,reject:7,rel:[0,4,6,7,9,12,13,23,26,27,29],relat:[0,1,3,4,5,11,13,14,20,23,26,28,29],relationship:[0,4,9,26],relativeerror:[0,26,27],releas:[1,6,13,19,21,26,29],relev:[0,1,5,7,11,15,19,21,23,26],reli:[0,3,4,6,8,14,26],reliabilti:21,reliabl:[7,23],relu:[3,4],remain:[1,2,4,6,12,20,23,27,29],remaind:23,reman:2,remark:1,rememb:[0,8,13,20,21,26],remind:[0,5,11,13,20,23,27,28,29],remov:[0,4,5,6,26,27],render:[0,26,27],reorder:[5,7,27,28],reorgan:[0,26],repeat:[0,1,3,4,5,6,9,10,11,13,14,20,21,23,26,28,29],repeated:26,repeatedli:[0,6,10,13,29],repet:3,repetit:[6,26,27,29],rephras:13,replac:[0,1,3,4,5,6,10,12,14,15,16,19,21,26,27,28,29],replica:[6,29],repo:21,report:26,reportexampl:21,repositori:[0,4,21,26,27],repres:[0,1,2,3,4,5,6,7,8,9,10,12,13,23,26,27,28,29],represent:[0,1,3,6,23,26,29],representd:3,reproduc:[0,5,6,9,12,15,16,19,21,23,26,27],repuls:[0,26],request:[0,13,26],requir:[0,1,3,4,5,6,8,9,11,12,13,17,20,26,27,28,29],res1:2,res2:2,res3:2,res_analyt:2,res_analytical1:2,res_analytical2:2,res_analytical3:2,resaml:[6,21],resampl:[0,7,10,19,26,27],rescal:[0,11,12,27],rescu:[5,28,29],reseach:[6,21],research:[0,4,13,19,25,26],resembl:[6,23,29],reserv:[1,5,6,23,28,29],reset:[],reshap:[0,1,2,3,4,6,8,9,10,15,16,20,26,27,29],residenti:[0,27],residu:[0,5,13,26],resiz:[5,27],resourc:26,respect:[0,1,2,3,5,6,7,8,10,11,12,13,14,16,17,21,23,26,27,28,29],respond:12,respons:[0,7,9,12,26,27],rest:[0,5,27],restat:[0,12,26],restor:4,restored_discrimin:4,restored_gener:4,restrict:[0,3,9,12,26],result:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,26,29],result_typ:2,retail:[0,27],retain:[5,6,27,29],rethink:29,return_data:14,return_kwarg:[],return_sequ:4,return_x_i:9,reus:[1,3,6,18,21],reveal:[0,12,26],revers:[1,20],review:[19,20],revisit:14,revolut:26,reward:[0,4,26],rewrit:[0,3,5,6,7,8,10,11,12,13,20,21,23,28],rewritten:[2,6,8,10,23,29],rewrot:13,rf:10,rfft2:[3,4,14],rfft:[3,4,14],rfftfreq:[3,4,14],rfftn:[3,4,14],rgb:3,rgoj5yh7evk:19,rh:[6,29],rho:[0,10,13,26],rho_1:10,rho_2:10,rho_m:10,rich:[0,26],ride:9,rideclass:9,ridedata:9,ridg:[7,11,13,19,26],ridge_sk:6,ridgebeta:[5,28],right:[0,1,2,3,5,6,7,8,9,10,12,13,14,16,17,20,21,23,26,27,28,29],right_sid:2,rightarrow:[0,1,5,6,8,11,12,13,23,26,27,29],rigor:[0,26],ring:6,rise:[0,26],risk:[0,13,15,16,26],rival:4,river:[0,27],rm:[0,23,27],rmse:[0,27],rmsporp:13,rmsprop:[1,3,4,13],rnd_clf:10,rng:23,rnn1:4,rnn2:4,rnn:[4,12],rnn_2layer:4,rnn_input:4,rnn_output:4,rnn_train:4,rntrick1:23,rntrick2:23,rntrick3:23,rntrick4:23,ro:[0,13,26],robert:[18,21,25],robust:[0,26],robustscal:[0,27],roc:[7,10],role:[0,2,5,6,8,19,26,27,28,29],roll:6,room:[0,24,26,27],root:[0,5,9,13,23,26,27,28],rot:26,rotat:[1,8,9,10],rotation_matrix:9,roughli:[1,3],round:[0,7,9,13,27],routin:[13,20,26],row:[0,1,2,5,6,7,9,11,20,26,27,28,29],rr:[5,27],rrr:[5,27],rthe:[],rug:13,rule:[0,1,5,6,13,21,26,27,28,29],run:[0,1,2,4,5,6,8,9,11,13,15,19,21,26,27,28,29],runtim:[1,6,14],runtimewarn:[1,6,29],russel:26,rust:[0,15,19,20,26],rustad:[],rvert:1,rvert_2:1,s:[0,1,2,3,4,5,6,7,9,11,12,13,16,17,18,19,20,21,23,26,27,28],s_1:6,s_:[3,6],s_i:[6,7],s_j:6,s_k:6,saddl:13,safe:[],safe_zip:[3,4,14],sai:[0,1,2,3,4,5,6,7,8,9,10,11,12,20,21,23,26,27,28,29],said:[6,9,13],sake:[0,5,7,11,26,27,28],sale:[0,26],sam:26,same:[0,1,2,3,4,5,6,8,9,11,12,14,17,20,21,23,26,27,28],samm:10,sampl:[0,1,2,3,4,5,6,7,8,9,10,13,14,15,16,19,20,21,23,26,27,28,29],sample_vari:14,sample_weight:[],sampleexptvari:23,samwis:26,sastri:11,satisfactori:[0,26],satisfi:[1,2,3,6,8,13,17,20,23,29],satur:[1,6,29],save:[0,4,6,7,9,13,26,27,29],save_fig:[0,6,7,9,10,26,27,29],savefig:[0,4,6,7,9,23,26,27,29],savetxt:4,saw:[5,27],scalabl:10,scalar:[2,5,6,10,13,27,28,29],scale:[0,1,3,5,6,7,8,9,10,11,12,13,15,16,17,19,20,21,24,26,28,29],scale_mean:4,scale_std:4,scalei:[],scaler:[0,7,8,9,10,11,27],scalex:[],scan:[5,7,28,29],scari:[5,28,29],scatter:[0,1,6,7,8,9,14,26,27,29],scenario:[6,13],schedul:[13,24],scheme:[1,13],schigsog2l:[],schrage:23,scienc:[0,1,10,12,13,19,22,23,24,25,27],scientif:[0,15,19,21,26],scientist:[0,26],scikit:[3,5,6,7,8,9,10,13,19,20,21,22,25],scikit_learn:[0,16],scikitlearn:26,scikitplot:[7,10],scipi:[0,3,5,6,13,15,19,20,21,26,27,28,29],scl:6,score:[0,1,3,6,7,9,10,11,15,16,21,24,26,27,29],scores_kfold:[6,29],scratch:[1,13],sdg:13,sdt_bfla8ua:[],seaborn:[0,1,3,6,7,26,27],seamless:[0,15,19,21,26],search:[0,1,3,5,9,13,26,28],sec:6,second:[0,2,3,4,5,6,7,8,9,11,12,14,15,16,19,20,21,23,24,26,27,29],secondeigvector:11,secondli:12,section:[4,11,17,20,23,27,28],sector:[0,26],see:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,18,19,20,21,23,26,27,28,29],seed:[0,1,2,3,4,5,6,8,9,11,13,14,15,16,21,23,26,27,28,29],seed_imag:4,seek:[1,2,8],seem:[1,3,4],seemingli:[0,26],seen:[0,1,3,5,10,12,17,23,26],segment:13,seismic:6,seldomli:[0,26],select:[1,5,6,8,9,10,11,17,21,22,23,24,25,26,27,28,29],self:[1,2,5,27,28],sell:4,semest:[7,22],semi:[8,13],semilogx:6,send:[5,12,13,24,26],senior:[22,24],sens:[0,4,6,8,21,26,29],sensibl:3,sensit:[0,5,6,9,13,26,27,28,29],sent:[2,29],sentenc:[4,12],sep:[27,29],separ:[0,1,2,3,4,6,8,9,12,14,15,19,21,23,26,29],septemb:[16,17,18,21,26,27],sequenc:[2,3,4,7,9,10,12,13,19,20,23,26],sequenti:[1,3,4,10,12,23],seri:[0,1,2,3,4,5,6,10,11,12,13,20,26,27,28,29],serif:[0,7,23,26],serv:[0,1,2,3,5,7,13,21,25,26,27,28,29],servic:21,session:[1,21,22,24,26,29],set:[1,4,5,6,7,8,10,11,13,14,16,17,19,20,21,23,24,28,29],set_major_formatt:[6,21],set_major_loc:[6,21],set_stream:[],set_tick:[1,8],set_ticklabel:1,set_titl:[0,1,2,3,7,12,14,26],set_xlabel:[0,1,2,3,7,12,26],set_xlim:[7,12],set_xticklabel:1,set_ylabel:[0,1,2,3,7,26],set_ylim:[7,12],set_ytick:7,set_yticklabel:[1,6],set_zlim:[6,21],seth:4,setiosflag:[],setminu:6,setosa:[8,9],setosa_or_versicolor:8,setp:[6,29],setprecis:[],setse:[],setup:[1,4,6,8,19,26,27],setw:[],sever:[0,3,5,6,7,8,9,11,12,13,19,20,23,26,27,28,29],sgd:[1,3],sgd_clf:8,sgdclassifi:8,sgdreg:13,sgdregressor:13,sgn:[5,27,28],sh:[],shallow:13,shape:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,20,26,27,28,29],shape_bas:2,share:[1,3,26],she:7,shift:[1,6,12,23],ship:3,shire:26,shortcom:13,shorten:4,shorter:23,shorthand:[26,29],shortli:[20,26],should:[0,2,3,5,6,8,9,11,12,13,15,16,20,21,23,26,27,28,29],should_sync:[],show:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,18,20,21,23,26,27,28,29],show_shap:4,shown:[0,4,5,7,8,11,12,13,20,27],showpoint:[],shrink:[3,5,6,8,11,21,27,28],shrinkag:[5,6,27,28],shrunk:11,shuffl:[0,1,4,6,13,27,29],side:[0,2,5,8,12,13,20,26,28,29],sigh:[19,26],sigma0:23,sigma1:23,sigma2:23,sigma:[0,1,5,6,7,10,11,12,13,17,18,20,21,23,26,27,28,29],sigma_0:[5,27,28],sigma_1:[5,27,28],sigma_2:[5,27,28],sigma_:[5,20,26,27,28,29],sigma_fn:[7,12],sigma_i:[0,5,26,27,28],sigma_j:[5,17,27,28],sigma_m:[6,23,29],sigma_n:[11,23],sigma_t:13,sigma_x:23,sigmoid:[1,2,4,7,8,10,12],sigmundson:[6,27],sign:[1,2,7,8,10,23,24],signal:[1,3,10,12],signatur:[],signifi:4,signific:1,significantli:[1,13,23],sigurd:[],sim:[4,5,6,13,18,21,23,28,29],similar:[0,1,2,3,4,5,6,7,8,9,10,11,14,17,19,20,21,26,27,28,29],similarli:[0,1,3,5,8,10,13,23,26,27,28],simpl:[1,2,3,5,6,7,8,10,11,12,14,15,16,17,19,20,21,23,29],simple_rnn:[],simplepredict:10,simpler:[0,1,5,6,7,13,16,19,21,26,28],simplernn:4,simplest:[0,1,3,4,9,10,12,14,26],simpletre:10,simpli:[0,1,2,4,5,6,8,9,10,11,12,15,19,20,21,23,26,27,28,29],simplic:[2,5,6,7,8,9,10,11,12,14,27,28],simplicti:[5,27],simplifi:[0,6,9,15,19,21,26,27,29],simplist:[3,6,23,29],simul:[6,29],simultan:[6,29],sin:[0,1,2,3,4,9,12,13,20,26],sinc:[0,1,2,3,5,6,7,8,9,10,11,13,20,23,25,26,27,28,29],sine:[3,12],singl:[0,1,2,3,5,6,7,8,9,12,13,20,23,26,27,28,29],singular:[0,6,13,20,21,26,28,29],sinusoid:3,site:[0,1,2,3,4,6,7,8,11,13,14,21,22,26,27,28],situat:[0,4,5,7,13,23,26,27],six:[3,4,14,23],size:[0,1,2,3,4,5,6,8,9,10,11,13,20,21,23,26,28,29],sketch:10,ski:9,skill:[0,26],skip:11,skiprow:27,skl:[0,6,26,27],sklearn:[0,1,3,5,6,7,8,9,10,11,13,14,26,27,28,29],skplt:[7,10],sl:6,slack:8,slice:[2,20,26],slide:[0,3,15,16,21,23,26,27],slight:[6,13,29],slightli:[1,2,3,5,6,7,10,23,27,28,29],slope:[8,11,12],slow:[0,2,8,13,27],slower:[5,20,26,27,28],slowest:20,slowli:12,slp:1,small:[0,1,2,3,5,6,8,9,10,11,12,13,19,20,21,23,26,27,28,29],smaller:[0,1,2,5,6,8,9,11,13,23,26,27,29],smallest:[0,4,14,15,16,26],smallest_row_index:14,smooth:[0,3,6,9,13,21,26],sn:[0,1,3,6,7,26,27],sne:11,sneak:26,so:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,18,19,20,21,23,24,26,27,28,29],soar:6,social:[0,26],societ:26,soft:[1,7,10,12],soft_pmap:[3,4,14],soften:8,softmax:[3,7],softwar:[0,8,15,19,20],sol:8,sole:[0,6,26],solid:[0,7,26],solitem:[],soltyp:[],solut:[0,1,2,3,5,6,8,10,11,13,17,18,20,21,23,26,27,28,29],solutionsummari:[],soluton:2,solv:[0,1,3,5,6,8,10,11,12,13,20,21,26,27],solve_expdec:2,solve_ode_deep_neural_network:2,solve_ode_neural_network:2,solve_pde_deep_neural_network:2,solveod:2,solveode_popul:2,solver:[2,7,8,9,10,11,20],some:[0,1,2,3,4,5,6,7,8,9,10,11,12,14,23,28,29],some_model:[6,27],somehow:4,someth:[0,1,3,4,7,9,11,21,23,26,27],sometim:[0,1,11,12,13,14,26,27],soon:[20,24],sophist:[0,26],sopt:13,sort:[5,6,9,11,23,27,29],sound:[3,5,28,29],sourc:[0,1,3,6,15,19,20,21,23,26,27,29],space:[0,1,4,5,8,9,11,12,13,14,23,27,28,29],span:[0,3,5,9,11,20,26,27],spare:1,spars:[3,6,20,26],sparse_mtx:[20,26],sparsecategoricalcrossentropi:3,sparsiti:10,spatial:[1,2,3,12],spdiag:[],speak:23,special:[6,7,10,12,13,20,23,26,27],specif:[0,1,2,3,4,5,6,7,8,9,11,12,16,19,20,21,23,25,26,27,28,29],specifi:[0,3,5,6,7,9,11,13,14,21,23,26,28,29],specifici:[0,10,26],spectacular:3,spectral:1,speech:[0,1,3,4,12,26],speed:[1,2,4,13],spend:23,spent:21,sphere:[0,27],spin:6,spite:[0,26],spline:8,split:[1,3,4,5,6,8,9,10,11,14,17,21,23,28,29],splite:0,splitter:[1,10],spmatrix:[],spontan:23,spot:3,spread:[0,11,23,26,27],springer:[18,21,25,26,28],spuriou:13,sqquar:28,sqrsignal:3,sqrt:[0,3,4,5,6,8,10,11,13,17,23,27,28,29],squar:[1,2,3,4,7,8,9,11,13,14,15,16,17,19,20,23,29],squarederror:10,squaredeuclidean:14,squash:12,squeez:2,srand:[],srtm:[6,21],srtm_data_norway_1:[6,21],sse4:[],stabil:[5,21],stabl:[0,4,5,6,7,9,11,19,26,27,28],stack:[2,3,4],stacklevel:[0,26],stage:[5,13,21],stai:[0,2,4,5,11,17,26,27],stand:[0,5,9,12,26,27,28],standadscal:27,standard:[0,1,4,5,6,7,8,10,12,15,16,17,20,21,23,26,28],standard_basi:2,standardscal:[0,6,7,8,9,10,11,27],stanford:13,start:[0,1,2,3,4,5,6,8,9,10,11,12,13,14,20,21,23,24,26,27,29],start_box:[2,13],start_nod:[2,13],start_tim:14,startpoint:[],stat:[6,27,29],state:[1,2,4,5,6,7,8,10,11,12,13,19,23,27,28,29],statement:[0,7,20],statist:[0,1,3,4,7,9,10,11,12,13,14,18,20,21,22,25,27],statu:[0,7,11,26,27],stavang:[6,21],std:[0,4,6,26,27,29],stdev:[],stdout:[],steep:13,step:[0,1,2,4,6,7,9,10,11,12,13,14,20,21,26],step_fn:[7,12],step_length:13,steps_list:9,steps_per_epoch:[],stereo:3,stian:[],still:[0,2,3,5,6,11,13,17,23,29],stimuli:12,stk2100:[25,26],stk3155:[15,21,22,24],stk4021:[25,26],stk4051:[25,26],stk4155:[22,24],stk5000:25,stk:[25,26],stochast:[0,1,5,6,8,11,12,15,16,21,26,28,29],stock:4,stoke:12,stone:[0,7,21,26],stop:[1,4,7,9,11,13,14],storag:[5,27],store:[0,1,2,3,6,11,13,21,23,26],storehaug:[24,26],str:[1,3,4],straight:[0,6,8,13,26,27,29],straightforward:[0,2,3,5,6,8,9,10,13,20,26,27,28,29],strategi:[0,1,9,26],stratifi:[6,29],streamtyp:[],strength:[0,5,14,27],stretch:11,strict:[8,13],strictli:[8,13],stride:[4,20],strike:6,string:1,stroke:7,strong:[3,6,9,10,12,20,23,29],strongli:[0,8,19,20,27],stronli:[0,27],structur:[0,1,2,3,6,9,10,12,19,26,29],stuck:[1,13],student:[0,21,22,24,25,26],studi:[0,3,4,5,6,7,8,11,12,13,17,19,21,25,26,27,28],studier:25,style:[0,7,9,20,26],sub:[9,12],subarg:[2,13],subdivid:[0,20,26],subfield:[0,26],subject:[6,8,23],submit:26,submodul:[3,4,14],subplot:[0,1,3,4,6,7,8,9,10,13,14,21,26,27,29],subplots_adjust:[8,23],subprogram:[20,26],subract:[0,27],subroutin:[0,26],subscript:1,subsequ:[1,4,5,6,12,20,23,27,29],subset:[1,6,9,12,13,19,26,29],subspac:[0,8,11,27],substanti:[9,10],substep:11,substitut:[3,6,12,20,29],subsubset:9,subtask:6,subtl:1,subtract:[0,4,5,6,11,13,17,20,21,23,29],subtre:9,subval:[2,13],succeed:[0,4,26],success:[3,7,9,13,23],successfulli:[4,9],sucess:[],sudo:[0,15,19,21,26],suffer:[0,1,2,5,10,26,27,28],suffici:[1,6,8,11,13,29],suggest:[1,13,25],suit:[8,12],suitabl:[0,23,27],sum:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28],sum_:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],sum_i:[0,2,5,6,8,13,17,21,27,28,29],sum_j:6,sum_ja_:0,sum_k:[6,8,12,20],sum_logist:13,sum_m:3,sum_n:3,sum_nx_:3,summar:[5,6,9,28,29],summari:[1,3,4,10,22,28],summat:[0,3,16,27],sundai:[17,18],sunni:9,superconduct:[27,28],superfici:3,superscript:[1,12],supervis:[0,5,6,7,9,12,19,26,27,29],supplement:7,support:[0,1,9,10,11,13,19,26,27],suppos:[0,5,6,7,8,10,11,12,13,20,26,27,28,29],suppress:[5,13,28],sure:[0,1,4,6,21],surf:[6,21],surfac:[0,6,21,26],surpass:6,surpris:[0,26],surround:[3,19],survei:[0,5,6,26,27,28,29],suyrzm0:[],svc:[8,9,10],svd:[0,6,11,17,26,29],svdinv:[5,28],svm:[8,9,10,11],svm_clf:[8,10],swap:2,swapax:2,swath:[5,27],sy:[3,4,13,14],symbol:[1,5,11,13,19,23,26,27,28],symmeteri:1,symmetr:[0,5,8,11,12,13,20,26,27],symmetri:[6,9],sympi:[0,15,19,21,26],synonim:23,syntax:[1,13],syntaxerror:[1,8],syrk:[],system:[0,1,3,4,6,7,9,10,12,13,15,19,20,21,26],systemat:[4,6,29],t0:[3,6,13],t1:[2,13],t2:2,t3:2,t:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,26,29],t_0:[2,9,13],t_1:13,t_:2,t_b:10,t_i:[1,2,5,12,27],t_j:12,t_k:9,tabl:[9,21,23,24,26],tabul:[0,26],tabular:26,tackl:4,tag:[2,3,4,5,6,7,12,13,14,20,23,27],taht:[0,26],tail:23,tailor:[2,8,11,26],taiwan:[0,26],take:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,19,20,21,23,26,27,28,29],taken:[0,1,3,6,10,13,20,26,29],tan:3,tangent:[1,4,12,13],tanh:[1,4,7,8,12],tape:[],target:[0,1,3,4,5,6,7,8,9,10,11,12,26,27,28,29],target_meta:[],target_nam:9,task:[0,1,3,6,9,11,12,14,21,26,29],tau:[3,5,23,28,29],taught:26,tax:[0,27],taylor:[2,13],taylornr:13,tc:8,td:[1,6,13,21,26,29],teach:26,team:1,teaser:0,technic:[0,5,6,13,15,26,27,28],techniqu:[0,1,8,10,13,19,23,25,26,27,29],technolog:[0,1,26],tek5040:[],tell:[0,4,6,10,11,13,23,29],temp1:1,temp2:1,temp:1,temperatur:[0,9,26],temporarili:1,ten:[3,17,26],tend:[3,5,6,8,9,10,12,13,14,27,28,29],tendenc:[0,26],tension:[6,29],tensor:3,tensorflow:[0,2,4,8,14,15,19,20,21,22,25,26,27],term1:[5,6,11,21,27],term2:[5,6,11,21,27],term3:[5,6,11,21,27],term4:[5,6,11,21,27],term:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,21,23,26,27,28],termin:[0,4,5,9,10,13,15,26,27],terrain1:[6,21],terrain:[6,21],test:[3,4,5,6,7,8,9,10,13,14,17,20,21,23,29],test_acc:3,test_accuraci:[1,3],test_error:6,test_imag:[3,4],test_ind:[6,29],test_input:4,test_label:[3,4],test_loss:3,test_pr:1,test_predict:1,test_rnn:4,test_scor:[7,10],test_siz:[0,1,3,5,6,10,26,27,28,29],test_split:9,tester:28,testerror:[0,6,27,29],testi:4,testpredict:4,testx:4,text:[0,1,2,4,5,8,9,11,13,15,20,23,25,26,27,29],textbook:[17,21,27,29],textual:9,textur:[1,9],tf:[1,3,4,13,14],tfe_py_execut:[],tflite:[3,4,14],th:[0,1,2,5,6,7,9,12,13,14,15,16,20,21,23,26,27,28,29],than:[0,1,2,3,4,5,6,7,9,10,11,12,13,17,19,23,26,27,28,29],thank:[4,6,27],theano:[1,19,26],thei:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,21,23,27,28,29],them:[0,1,3,4,6,8,9,10,11,12,13,20,21,26,27],theme:[0,26],themselv:[0,23,26],thenc:[6,29],theorem:[2,6,7,27],theoret:[0,4,10,26],theori:[0,1,3,8,9,12,13,18,19,21,25,26,28],thereaft:[0,5,6,11,12,15,16,17,20,21,26,29],therebi:[0,5,7,11,26,27,28],therefor:[0,1,2,3,4,6,7,8,11,13,23,26,27,29],therein:11,thereof:[0,6,13,26,29],theta:[1,4,13,23],theta_:[1,13],theta_i:1,theta_k:[],theta_linreg:13,theta_t:13,thi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,22,23,25,27,28,29],thing:[0,1,2,4,5,7,9,23,26,28,29],think:[0,1,3,4,6,9,12,13,14,23,26,29],third:[0,3,6,13,24,26],thirti:7,thorughout:26,those:[0,3,5,6,8,9,10,11,20,21,22,26,27,29],though:[1,2,3,4,13,20,23],thought:[6,14,21,23,29],thousand:[0,1,21,26,27],thread:[],three:[0,1,3,5,6,8,9,12,15,20,21,22,23,24,26,27,28,29],threshold:[1,3,9,10,11,12,13],through:[0,1,2,3,4,5,6,8,11,12,13,14,19,20,23,26,27,28,29],throughout:[0,4,5,14,19,20,23,26],thu:[0,1,2,5,6,7,8,10,11,12,13,21,24,26,27,28,29],thumb:[0,6,21,26,27],thursdai:[24,26],tibshirani:[6,18,21,22,25,26,27,29],tick_param:6,ticker:[6,13,21,23],tif:[6,21],tight_layout:[1,7],tightli:11,tild:[0,5,6,7,11,15,16,17,18,21,23,26,27,28,29],till:[0,4,7,8,9,10,12,20,26,27],time:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,19,20,21,23,26,27,28,29],timefunct:[],timeit:4,timer:4,tini:1,tip:3,titl:[0,1,2,3,4,6,7,8,9,10,13,21,23,26,27,29],tmp:13,tmp_log:[],tn:[2,3,7],to_categor:[1,3,4],to_categorical_numpi:1,to_numer:[0,6,26,29],to_str:[],todai:3,todo:[3,4,14],togeth:[0,3,6,8,11,13,19,26,27],toi:14,told:13,toler:[2,6,14],tolist:4,tomographi:12,too:[0,2,4,5,6,9,11,13,15,23,25,26,27,29],took:[8,26],tool:[0,1,3,6,13,19,27,29],toolbox:8,top:[0,3,4,5,6,9,10,14,18,19,26,28,29],top_node_typ:[],top_trac:[],topic:[0,5,6,7,8,19,21,27,28],topograph:21,topolog:[1,3,12],toposort:2,torkjellsdatt:[24,26],toss:[10,23],total:[0,1,2,3,4,6,7,8,10,11,12,13,14,15,16,20,23,24,26,27,29],total_loss:4,totalclustervari:14,totalscatt:14,totalvari:[],toward:[1,2,7,12,13,21],town:[0,27],tp:[4,7],tpng:9,tpu:[13,19,26],tqdm:6,tr:27,trace:[2,13],trace_stack:[2,13],traceback:[0,2,3,4,6,9,10,13,14,15,21,26,28],traceback_util:[],tracer:[2,13],track:[3,13,14,20],tract:[0,27],tractabl:[0,26,27],trade:[5,9,28,29],tradeoff:[0,5,21,26,27,28],tradit:[0,1,4,6,26,29],train:[2,3,5,6,8,9,10,11,12,13,17,21,28,29],train_accuraci:[0,1,3,26],train_dataset:4,train_end:[0,1,27],train_error:6,train_funct:[],train_imag:[3,4],train_ind:[6,29],train_label:[3,4],train_pr:1,train_siz:[0,1,3,27],train_step:4,train_test_split:[0,1,3,5,6,7,9,10,11,26,27,28,29],train_test_split_numpi:[0,1,27],trainabl:[],trainable_vari:4,trained_model:[6,27],trainerror:[0,27],traini:4,training_checkpoint:4,training_dataset:4,training_gradi:13,training_gradient_fun:[],training_loss:[],trainingerror:[6,29],trainpredict:4,trainscor:4,trainx:4,trait:[0,26],trajectori:4,tran:[],transfer:9,transform:[0,5,6,7,8,9,10,11,12,13,19,20,26,27,29],transit:[6,12],translat:[1,4,6,10,27],translate_vjp:[],transpos:[1,5,11,20,27,28],travers:[0,5],treat:[0,1,3,6,12,13,23,26,27,29],tree:[0,1,6,19,21,26],tree_clf:[9,10],tree_clf_:9,tree_clf_sr:9,tree_reg1:9,tree_reg2:9,tree_reg:9,trend:23,treue:7,trevor:[18,21,25],tri:[2,3,4,9,13],triain:0,trial:[0,2,4,6,13,23,26,29],triangl:13,triangular:20,trick:[3,4,8,11,13,23],trickier:23,tridiagon:20,trillion:19,trivial:[0,1,5,11,23,26,28],troubl:[0,8,12,27],truck:3,true_beta:[6,27],true_divid:1,true_fun:[6,29],truli:26,tucker:8,tuesdai:[24,26,29],tumor:[7,9],tumour:7,tunabl:1,tune:[4,9,13,20,26],tup:[],tupl:[2,13],turn:[0,1,5,6,7,8,9,10,11,12,13,20,21,23,26,27,29],tutori:[1,4],tv:2,tveito:2,tweak:[1,4,10,23],twice:13,twist:11,twister:[],two:[0,1,2,4,5,6,7,9,10,11,12,13,14,15,16,20,22,23,25,26,28,29],tx:13,tx_1:13,txt:4,ty:13,type:[0,1,3,6,8,10,13,16,20,21,23,27,28,29],typeerror:[2,13],typic:[0,1,2,3,4,5,7,9,10,12,13,23,26,27,28,29],typo:21,u:[0,2,5,6,8,10,11,12,17,20,26,27,28],u_:20,u_i:12,u_m:10,ua:[0,26],ubuntu:[0,15,19,21,26],uci:[0,27],ufunc:[],uio:[21,24,25],un:14,unari:[20,26],unary_f:[2,13],unary_oper:[2,13],unary_to_nari:[2,13],unbalanc:[6,9,29],unbias:[0,5,6,26,28,29],unbroadcast:[],unbroadcast_f:[],uncent:6,uncertainti:[0,5,26,28,29],uncertitud:23,unchang:[1,3],uncorrel:[10,23],undefin:[5,27],under:[0,1,3,4,5,6,10,13,14,15,16,19,21,26,27,28,29],underdetermin:[0,26],underfit:[1,6,29],underflowproblem:[5,28,29],undergo:[5,28,29],undergradu:[22,24],underli:[0,1,9,13,23,26],underset:[4,14],understand:[0,1,3,5,6,10,13,14,19,26,27,28],understood:[8,13],undesir:8,undetermin:[5,8,28,29],undo:4,unexpect:[6,29],unexpected:23,unfair:[6,27],unfortun:[1,8,9,10],unicode_liter:[8,9],uniform:[0,1,5,6,11,13,15,16,21,23,26,27],uniform_real_distribut:[],uniformli:[13,23],unifrompdf:23,unimport:13,union:[5,6,28,29],uniqu:[0,2,6,13,14,20,26,29],unique_cluster_label:14,unit:[0,1,3,4,5,10,12,23,26,27,28],unitari:[5,6,20,27],unitarili:[20,26],uniti:23,univari:23,univers:[0,1,2,13,15,19,21,22,24,26,27,28,29],unix:1,unknow:[0,20,26],unknown:[0,1,3,4,5,6,8,10,13,16,20,26,27,28,29],unknowwn:12,unlabel:1,unless:[0,3,6,11,13,21,26,27,29],unlik:[1,3,8,13],unnecessarili:9,unord:3,unravel:1,unrol:[3,11],unseen:[0,7,9],unstabl:1,unsupervis:[0,1,4,12,19,26],unsymmetr:[20,26],until:[1,2,4,9,12,13,14],untouch:0,unus:[3,4,14],unusu:12,up:[1,3,4,5,6,8,10,11,13,14,16,18,19,20,21,23,24,28],updat:[1,2,3,4,10,12,13,14],uploa:26,upload:[19,21,25],upon:[0,1,6,7,11,20,21],upper:[0,8,9,16,20,27],uppercas:[20,26],upsampl:4,upscal:4,url:28,us:[4,5,6,8,9,10,11,12,14,15,16,17,18,20,21,22,23,25,28,29],usa:26,usag:[0,8,19,26,27],usd10000:[0,27],usd:[0,27],use_bia:4,use_multiprocess:[],usecol:[0,26],useless:1,user:[0,1,2,3,4,6,7,8,11,14,15,19,20,21,26,27],usernam:21,userwarn:[3,4,6,14],usetex:23,usg:[6,21],usr:23,usual:[0,3,4,7,12,13,14,26],ut:[5,28],util:[0,1,3,4,6,7,10,14,26,27,29],ux:20,v0:23,v1:[3,4,14,23],v2:[3,4,14,23],v8ejbfezkui:[],v:[2,4,5,6,11,13,17,19,27,28],v_0:11,va:1,val:13,val_accuraci:3,val_loss:4,vale:2,valid:[0,1,4,7,9,10,13,19,23,26,27],validation_batch_s:[],validation_data:3,validation_freq:[],validation_split:4,validation_step:[],valu:[0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,19,20,21,26],valuat:9,valueerror:[0,2,26],valy:4,van:[0,18,21,26,27,28,29],vandenbergh:[8,13],vandermond:[0,26],vanilla:[0,6,11,14,27],vanish:[1,4,13,23],var_x:23,varabl:8,varepsilon:[5,6,18,21,28,29],varepsilon_:[5,6,28,29],varepsilon_i:[5,6,28,29],vari:[0,1,3,5,6,10,15,16,26,29],variabl:[0,1,2,5,6,7,8,10,11,12,13,14,20,26,27,28,29],varianc:[0,1,5,7,9,10,11,13,14,18,19,20,23,26,27],variance_i:[5,11,27],variance_x:[5,11,27],variant:[0,1,6,8,12,13,26,27],variat:[3,4,11,26],varieti:[0,3,12,15,19,21,26],variou:[1,3,5,6,7,8,9,11,12,13,16,18,19,20,21,23,27,28],vartempvec:[],varvec:[],varydimens:4,vastli:3,vaue:1,vault:0,vdot:[2,13],ve:21,vec:[6,29],vector:[0,1,2,3,4,5,6,7,9,10,11,13,14,16,17,19,28,29],vector_mean:14,ventur:[0,8,15,19,26],verbos:[1,3,4],veri:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,21,23,25,26,27,28,29],verifi:[3,11,20,26],versatil:[8,26],versicolor:[8,9],version:[0,3,4,10,13,14,15,19,20,21,23,26,27],versu:1,vert:[0,1,5,6,7,8,9,11,13,17,26,27,28,29],vert_1:[5,6,27,28,29],vert_2:[5,6,11,17,27,28,29],via:[0,5,6,7,8,9,10,11,12,15,19,20,21,22,23,24,26,27,28,29],vidal:11,video:[0,1,12,19,22,24,26,27,29],view:[1,3,5,6,12,13,23,25,26,28,29],violat:8,virginica:9,viridi:[0,1,2,3,26],virtual:1,viscos:13,viscou:13,visibledeprecationwarn:2,vision:[0,3,26],visual:[0,3,11,12,15,19,26,27],visualis:1,viz:[6,8,23],vjiseqm2lci:[],vjp:[2,13],vjp_0:2,vjp_0_fun:2,vjp_1:2,vjp_1_fun:2,vjp_argnum:2,vjpfun:2,vjpmaker:2,vjpnode:[2,13],vjps_dict:[],vmap:13,vmax:[1,6],vmc:[],vmin:[1,6],voic:3,volum:[0,3,26],vote:10,voting_clf:10,votingclassifi:10,votingsimpl:10,vrtx:[],vs:[0,4,6,27,29],vspace:[2,13],vstack:[5,11,20,23,26,27],vt:[5,27,28],w1:8,w2:[8,11],w3:8,w:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,20,23,26,27,29],w_1:[8,20],w_1x_1:8,w_1x_:8,w_2:[8,20],w_2x_2:8,w_2x_:8,w_3:20,w_4:20,w_:[1,12],w_hidden:2,w_i:[1,2,10],w_ix_i:12,w_j:20,w_m:20,w_output:2,w_px_:8,w_px_p:8,wa:[0,1,3,4,5,6,7,10,11,12,13,14,17,20,21,26,27,28,29],wai:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,17,20,23,26,27,28],walk:9,walker:23,wang:[0,26],want:[0,1,2,3,4,5,6,8,9,10,11,12,13,14,15,16,17,19,21,23,26,27,28,29],warn:[0,1,3,4,8,14,26,27],warrant:[6,29],wast:3,watch:19,wave:3,wavelet:8,we:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,29],weak:[9,10,14],weather:[1,12],web:[19,22,24,26,27],webpag:26,websit:[6,20,21,22,26],wedg:[8,23],wednesdai:[24,26,29],wee:11,week:[0,5,6,7,21,22,24],weekli:[19,21,24,25,26],weight:[0,1,2,3,6,7,9,10,12,13,21,23,27],weigth:2,welcom:[8,19],well:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,16,19,20,21,23,25,26,27,28,29],went:8,were:[0,1,3,4,5,6,7,8,10,11,12,14,23,26,27,29],wessel:[0,18,21,26,27,28,29],westbi:[],westby:[],what:[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,23],whatev:3,when:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,18,20,21,23,26,27,28,29],whenev:[13,23],where:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,26,27,28,29],wherea:[6,23,29],wherein:[1,12],whether:[0,3,5,7,9,21,23,26,28],which:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,28,29],whichev:[1,3],while_p:[3,4,14],white:9,whiteboard:27,who:[0,22,26],whole:[1,3,4,5,9,11,13,28,29],whose:[0,6,10,23,27,29],whow:[11,27],why:[0,1,3,6,13,21],wide:[0,1,3,6,7,12,15,19,20,21,26,29],widehat:[6,29],width:[0,3,8,9,26],wieringen:[0,18,21,26,27,28,29],win:10,wind:9,wing:[24,26],winther:2,wiothout:[6,27],wiscons:7,wisconsin:10,wisdom:6,wise:[0,1,5,12,13,27],wish:[0,2,5,7,8,11,13,14,20,21,26,27],with_std:[0,27],wither:6,within:[0,2,3,4,7,9,12,13,14,23,25,26],withinclust:14,without:[0,1,5,6,8,9,11,12,13,26,27,28,29],won:[0,26],wonder:8,woodi:26,word:[0,1,3,4,5,6,7,14,23,26,27,28,29],work:[0,1,4,6,7,8,9,13,15,19,21,22,23,24,26,27,29],worker:[],workshop:26,world:[0,8,26,27],worldwid:[0,26],wors:[0,1,3,4,6,26,29],worst:9,worth:9,worthi:21,would:[0,1,3,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],wrap:[2,3,4,6,14,20,26],wrap_toco:[3,4,14],wrap_util:[2,13],wrapper:[0,26],write:[0,1,2,3,5,6,7,8,12,13,15,16,17,20,21,26,27,29],written:[0,2,3,5,11,12,13,16,19,20,21,23,26,27,28,29],wrong:[1,8],wrongli:10,wrote:[5,11,27],wrt:[2,10,13],wth:[10,13],www:[19,20,21,25,26],wx_1:8,x0:8,x1:[4,8,9,10,13],x1_exampl:8,x1d:8,x2:[8,9,10,13],x2d:[8,11],x2d_train:11,x2dsl:11,x3:8,x:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,29],x_0:[0,5,11,20,26,27,28,29],x_1:[0,2,5,6,7,8,9,10,11,13,20,23,26,27,28,29],x_2:[0,2,5,6,7,8,9,10,11,13,20,23,26,27,29],x_3:[8,20,23],x_4:20,x_:[0,2,3,5,6,8,10,11,13,14,18,20,21,23,26,27,28,29],x_center:11,x_data:1,x_data_ful:1,x_hidden:2,x_i:[0,1,2,5,6,7,8,9,10,11,12,13,14,17,20,23,26,27,28,29],x_input:2,x_ix_:[0,26],x_iy_i:8,x_j:[0,2,8,9,12,16,23,27],x_jy_j:8,x_k:[12,14,20,23,27],x_l:23,x_m:[6,12,20,23,29],x_n:[0,2,3,6,8,11,12,13,20,23,26,29],x_new:[9,10],x_offset:[6,27],x_output:2,x_p:[3,7,9],x_poli:9,x_poly10:9,x_pred:4,x_prev:2,x_reduc:11,x_scale:8,x_small:13,x_test:[0,1,3,5,6,7,9,10,11,26,27,28,29],x_test_own:6,x_test_scal:[0,6,7,9,10,11,27],x_tot:4,x_train:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],x_train_mean:[6,27],x_train_own:6,x_train_scal:[0,6,7,9,10,11,27],x_val:1,xarrai:[19,26],xavier:1,xbnew:13,xcode:[0,15,19,21,26],xdclassiffierconfus:10,xdclassiffierroc:10,xeinsum:[3,4,14],xg_clf:10,xgb:10,xgbclassifi:10,xgboost:9,xgboot:10,xgbregressor:10,xgparam:10,xgtree:10,xi:[8,13],xi_1:8,xi_:8,xi_i:8,xinv:28,xk:8,xla:[3,4,13,14,19,26],xla_client:[3,4,14],xla_comput:[3,4,14],xlabel:[0,1,2,3,4,5,6,7,8,9,10,13,21,23,26,27,28,29],xlim:[6,10,29],xm:9,xmesh:13,xnew:[0,13,26],xp:23,xpanda:[0,27],xpd:[5,11,27],xplot:0,xs:9,xscale:[0,27],xsr:9,xt_x:13,xtest:[6,29],xtick:[3,6,8,9,29],xtrain:[6,29],xu:[0,26],xx:[0,20,26],xy:[0,6,8,20,21,26],xytext:8,xz:[20,26],y1:4,y2:4,y3:4,y:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],y_0:[0,5,11,20,26,27,28,29],y_1:[0,5,8,9,11,13,20,26,27,28,29],y_1y_1:8,y_1y_1k:8,y_1y_2:8,y_1y_2k:8,y_1y_n:8,y_1y_nk:8,y_2:[0,5,8,9,11,20,26,27],y_2y_1:8,y_2y_1k:8,y_2y_2:8,y_2y_2k:8,y_3:[0,9,20],y_4:20,y_:[0,1,5,6,10,11,20,26,27,28,29],y_data:[0,1,5,6,26,27,28],y_data_ful:1,y_decis:8,y_fit:[0,27],y_i:[0,1,5,6,7,8,9,10,11,12,13,15,16,17,18,20,21,26,27,28,29],y_if_:10,y_ix_:[0,26],y_ix_i:[7,8,13,27],y_iy_jk:8,y_j:[6,8,12,21,29],y_k:12,y_m:20,y_model:[0,4,5,6,26,27,28],y_n:[8,13],y_ny_1:8,y_ny_1k:8,y_ny_2:8,y_ny_2k:8,y_ny_n:8,y_ny_nk:8,y_offset:[6,27],y_plot:9,y_pred1:9,y_pred2:9,y_pred:[0,1,4,6,7,8,9,10,27,29],y_pred_rf:10,y_pred_tre:10,y_proba:[7,10],y_scaler:6,y_test:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],y_test_onehot:1,y_test_predict:[0,27],y_tot:4,y_train:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],y_train_mean:[6,27],y_train_onehot:1,y_train_predict:[0,27],y_train_scal:6,y_val:1,ye:[3,6,7,29],year:[0,19,26],yet:[0,1,6,8,11,13,26],yhd5w:[],yi:13,yield:[0,2,5,6,8,10,12,13,14,20,23,26,28,29],yk:8,ylabel:[0,1,2,3,4,5,6,7,8,9,10,13,21,23,26,27,28,29],ylim:[3,6,29],ym:9,ymesh:13,yn:0,yo:[8,9,10],yoshua:[1,25],you:[0,1,2,3,4,5,6,8,9,10,11,13,15,16,17,19,20,21,23,24,25,27,28,29],young:[0,26],your:[1,2,4,5,6,8,11,13,17,19,20,21,29],yourself:[11,13,26],youtu:[],youtub:19,ypred:[6,29],ypredict2:13,ypredict:[0,13,26,27],ypredictlasso:[5,28],ypredictol:[0,5,28],ypredictown:[6,27],ypredictownridg:6,ypredictridg:[0,5,6,28],ypredictskl:[6,27],yridg:26,ys:9,ytest:[6,29],ytick:[3,6,8,9,29],ytild:[0,6,26,27,29],ytildelasso:[5,28],ytildenp:[0,26,27],ytildeol:[0,5,28],ytildeownridg:6,ytilderidg:[5,6,28],ytrain:[6,29],yvqgvcsovpw:[],yx:[20,26],yy:[20,26],yz:[20,26],yzbxji6lgl0:[],z:[0,1,2,3,4,5,6,7,8,9,11,12,13,20,21,23,26,27,29],z_0:[20,26],z_1:[20,26],z_2:[20,26],z_:[1,2,12,20,26],z_c:1,z_h:1,z_hidden:2,z_i:[1,12],z_j:[1,12],z_k:[12,27],z_m:1,z_mod:9,z_o:1,z_output:2,zaman:23,zaxi:[6,21],zero:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,20,21,23,26,27,28,29],zeros_lik:4,zeroth:27,zfill:4,zip:[2,4,6,13],zl:[],zm_h:[0,26],zn:[0,27],zone:[0,27],zoom:26,zx:[20,26],zy:[20,26],zz:[20,26]},titles:["3. Linear Regression","14. Building a Feed Forward Neural Network","15. Solving Differential Equations with Deep Learning","16. Convolutional Neural Networks","17. Recurrent neural networks: Overarching view","4. Ridge and Lasso Regression","5. Resampling Methods","6. Logistic Regression","8. Support Vector Machines, overarching aims","9. Decision trees, overarching aims","10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods","11. Basic ideas of the Principal Component Analysis (PCA)","13. Neural networks","7. Optimization, the central part of any Machine Learning algortithm","12. Clustering and Unsupervised Learning","Exercises week 34","Exercises week 35","Exercises week 36","Exercises week 37","Applied Data Analysis and Machine Learning","2. Linear Algebra, Handling of Arrays and more Python Features","Project 1 on Machine Learning, deadline October 9 (midnight), 2023","Teaching schedule with links to material","1. Elements of Probability Theory and Statistical Data Analysis","Teachers and Grading","Textbooks","Week 34: Introduction to the course, Logistics and Practicalities","Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression","Week 36: Statistical interpretation of Linear Regression and Resampling techniques","Week 37: Statitsitcal interpretations and Resampling Methods"],titleterms:{"1":[0,15,16,17,18,21,26,27],"10":[],"11":[],"12":[],"13":[],"14":29,"15":[],"16":[],"17":[],"18":[],"19":[],"2":[0,15,16,17,18,26,27],"20":[],"2021":[],"2022":[],"2023":[21,24],"21":[],"22":[],"23":[],"24":[],"25":[],"26":[],"27":[],"28":[],"29":[],"3":[0,15,16,26,27],"30":[],"31":27,"34":[15,26],"35":[16,27],"36":[17,28],"37":[18,29],"38":[],"39":[],"4":[0,27],"40":[],"41":[],"4155":[],"42":[],"43":[],"44":[],"45":[],"46":[],"47":[],"5":0,"6":[],"7":28,"8":[],"9":21,"case":[8,10,23,27,28],"do":[1,26,28,29],"final":[12,27,28],"function":[0,1,6,7,8,10,11,12,13,21,23,26,27,28,29],"import":[5,20,26,27],"new":[4,28,29],A:[0,1,4,8,9,26,28,29],And:[26,27,28],In:24,Ising:6,The:[0,1,2,3,5,6,7,8,9,11,12,17,19,26,27,28,29],To:[26,27],With:[4,28],about:[26,27],abov:28,activ:[1,12,28],actual:[],ad:[0,6,17,21,26,27],adaboost:10,adagrad:13,adam:13,adapt:10,adjust:1,adversari:4,again:[3,9],ai:26,aim:[8,9,17,18,26],aka:[26,27],algebra:[20,26],algorithm:[9,10,11,12,27],algortithm:13,all:8,an:[0,4,10,26],analys:[5,27],analysi:[0,5,6,11,19,21,23,26,27,28,29],analyt:[0,16,17],ani:13,anoth:[9,28,29],appli:19,approach:[0,8,14,26,29],approxim:12,architectur:1,arrai:[20,26],assist:24,assumpt:[28,29],august:27,autocorrel:23,autograd:[2,13],automat:13,b:[17,21],back:[1,11,12],background:[19,21,29],bag:10,base:[13,29],basic:[0,5,7,9,10,11,20,27,28,29],batch:1,bay:[5,28,29],befor:11,beta:[28,29],better:8,bia:[6,21,29],binari:1,bind:26,binomi:[],bird:10,block:[],boldsymbol:[27,28,29],boost:10,bootstrap:[6,10,29],boston:[0,27],breast:1,brief:[26,29],bring:12,build:[1,3,9],c:[21,26],calcul:27,can:[26,29],cancer:[1,7,9,11],cart:9,center:27,central:[13,19,23,29],chain:12,chang:10,channel:26,chi:[0,26],choos:1,cifar01:3,classic:11,classif:[1,9,10],classifi:8,clip:1,cluster:14,cnn:3,code:[0,1,2,5,9,11,12,13,14,26,27,28,29],collect:[1,3],commun:26,compar:[2,10],comparison:28,complet:27,complex:[0,6,21,27],complic:6,compon:11,comput:9,computation:29,computerlab:26,con:9,concept:23,condit:[28,29],confid:29,conjug:13,continu:[],contn:26,convex:[8,13],convolut:[3,12],correctli:[28,29],correl:[11,27],cost:[1,10,27,28,29],cours:[19,25,26],covari:[5,11,23,27],cover:26,cross:[6,21,29],cumul:[],cython:26,d:21,data:[0,1,3,6,7,9,11,15,16,19,21,23,26,27],dataset:[1,3],deadlin:[21,26],decai:2,decis:[9,10],decomposit:[5,11,17,20,27],deep:[1,2],defin:[1,26],definit:[],degre:[0,27],deliveri:21,delta:29,demonstr:[],dens:[0,26],deriv:[5,12,27,28,29],descent:[2,10,13],descript:21,design:27,detail:[3,26],develop:1,deviat:[],diagon:11,dice:[],differ:8,differenti:[2,13],diffus:2,dimension:[2,3,8,21,27],disadvantag:9,discret:23,disguis:[],distribut:[5,23,28,29],distrubut:[28,29],doe:[27,28],domain:23,down:1,dropout:1,e:21,economi:27,electron:21,element:[0,23,26],elimin:20,energi:26,ensembl:10,entropi:9,environ:[0,15,26],equat:[0,2,12,26,27,28],error:[0,10,26,27,29],essenti:26,estim:[28,29],etc:26,euler:2,evalu:1,event:[],exampl:[0,1,2,3,4,6,7,8,9,10,26,27,28,29],exercis:[0,6,15,16,17,18,26,27,29],expect:[18,23,28,29],expens:29,experi:23,explor:[0,15,16,26],exponenti:2,express:[17,18,26,27],extrapol:4,extrem:[10,26],ey:10,f:21,fall:24,famili:[1,26,27],famou:20,fantast:27,featur:[9,20,27],feed:[1,12],find:29,fine:1,first:[4,12,26,27,28],fit:[0,10,26,28],fix:27,fold:29,forc:3,forest:10,format:[21,26],forward:[1,2,12],fourier:3,frank:[6,21,27],freedom:[0,27],frequent:27,frequentist:[0,26],from:[5,10,12,27,28,29],full:2,further:[3,5,27],fy:[],g:21,gan:4,gaussian:20,gd:13,gener:[4,9,26],geometr:11,gini:9,good:[0,26],grade:[24,26],gradient:[1,2,10,13],growth:2,ha:19,handl:[20,26,27],happen:[28,29],have:26,heard:26,hessian:27,hidden:2,histogram:29,hous:[0,27],how:[],hyperparamet:1,hyperplan:8,i:1,id3:9,idea:11,ident:[28,29],identifi:29,ii:26,iid:[28,29],illustr:28,implement:1,implic:[5,27],improv:1,includ:13,increment:11,independ:[28,29],index:9,inform:24,input:2,instal:[19,21,26],instructor:24,intercept:27,interpret:[5,11,26,27,28,29],interv:29,introduc:[11,13,27],introduct:[0,6,19,20,21,26],invers:[5,20,28],invert:27,invok:28,iter:10,its:27,jackknif:[],jacobian:27,jax:13,julia:26,jungl:10,k:29,kera:[1,3],kernel:[8,11],lagrangian:8,lasso:[5,6,21,27,28,29],last:[27,29],later:[5,27],layer:[1,2,3,12],learn:[0,1,2,11,13,14,15,16,19,21,26,27,28,29],least:[5,6,18,21,26,27,28],lectur:[26,27,28,29],level:10,librari:[19,26],likelihood:[7,28,29],limit:[1,13,23,29],linear:[0,8,13,20,26,27,28],link:[5,11,22,25,27,28,29],literatur:21,logist:[7,26],loss:27,lu:20,machin:[0,8,13,19,21,26],made:[28,29],mai:26,main:23,make:[0,9,10,15,16,26,27],mani:[10,12],manipul:27,margin:[28,29],mass:26,materi:[22,26,27,28,29],math:[5,27],mathemat:[3,5,8,27],matric:[5,20,26,28],matrix:[1,5,11,12,20,26,27,28],matter:[0,26],max:27,maximum:[28,29],mean:[0,26,27,28],meet:[5,10,23,26,27],mercer:8,mersenn:[],method:[6,9,10,13,21,29],midnight:21,min:27,minim:26,ml:26,mle:[28,29],mlp:12,mnist:[3,4],model:[0,1,4,6,12,26],moment:[],momentum:13,moon:[8,9],more:[3,6,20,21,26,27,28,29],multilay:12,multipl:[1,3],multipli:8,name:[],need:[21,26],network:[1,2,3,4,7,12,26],neural:[1,2,3,4,7,12,26],non:8,normal:[0,1,29],norwai:[],notat:12,note:[27,28],novemb:[],now:[1,9,13,28,29],nuclear:[0,26],nueral:[],numba:26,number:[0,2,23,27],numer:[2,21,23],numpi:[20,26],object:3,observ:[],obtain:11,octob:21,od:2,off:[6,21],ol:[5,6,21,28,29],one:[2,12],oper:20,optim:[1,8,13,19,26,27],order:13,ordinari:[5,6,18,21,26,27,28],organ:[0,26],oslo:25,other:[4,9,11,12,20,26,27],our:[0,4,5,11,13,26,27],outcom:[19,26],output:2,overarch:[0,4,8,9,17,18,26,27],overview:[10,26],own:[0,10,11,15,16,26,27],packag:[20,26],panda:[26,27],paper:21,paramet:[26,27],part:[13,19,21,27],partial:2,pass:1,pca:11,pdf:23,pencil:21,perceptron:12,perform:[1,9],period:3,perspect:1,plan:[27,28,29],plot:29,point:4,poisson:2,polynomi:[3,28],popul:2,practic:[13,24,26],pre:[1,3],predict:4,preprocess:27,prerequisit:[3,19,26],princip:11,principl:3,pro:9,probabl:[5,23,28,29],problem:[1,2,13,26,27,28],procedur:[9,26],process:[1,3],program:[2,13,21],project:[6,21,26],prop:13,propag:[1,12],properti:[5,23,27],pseudo:[],python:[0,9,15,19,20,26],quick:8,r:26,ran0:[],random:[10,11,23],read:[9,26,27,29],real:[6,21,26],recip:[],recommend:[26,27],rectangular:28,recurr:[4,12],reduc:[0,27],reduct:3,reformul:2,regress:[0,5,6,7,9,10,13,17,18,21,26,27,28,29],regular:1,relat:27,relev:[25,27,29],relu:1,remark:3,remind:[6,8,26],repeat:27,replac:13,report:21,repositori:29,requir:[2,19],resampl:[6,21,28,29],rescal:6,residu:27,resourc:2,result:[27,28],revisit:13,rewrit:[26,27,29],ridg:[0,5,6,17,18,21,27,28,29],rm:13,rng:[],rule:12,s:[8,10,29],same:[13,29],sampl:11,scale:27,schedul:[22,26],schemat:9,scheme:2,scienc:26,scikit:[0,1,11,15,16,26,27,28,29],second:13,select:[],semest:24,septemb:[28,29],session:28,set:[0,2,3,9,12,15,26,27],sgd:13,should:1,similar:13,simpl:[0,4,9,13,26,27,28],singl:10,singular:[5,11,17,27],situat:[],size:27,soft:8,softmax:1,softwar:[21,26],solv:[2,28],solver:13,some:[13,20,26,27],specifi:2,split:[0,15,16,26,27],squar:[0,5,6,10,18,21,26,27,28],standard:[13,27,29],state:[0,26],statement:26,statist:[5,6,19,23,26,28,29],statitsitc:29,steepest:[10,13],step:29,still:27,stk3155:[],stochast:[13,23],strongli:26,subtract:27,suggest:26,sum:29,summari:[24,26],superposit:3,supervis:1,support:8,svd:[5,27,28],systemat:3,t:[27,28],teach:[22,24],teacher:[24,26],techniqu:[6,11,21,28],technolog:19,tensorflow:[1,3],tent:26,term:29,test:[0,1,15,16,26,27,28],textbook:[25,26],thei:26,theorem:[5,8,11,12,23,28,29],theori:23,thi:[17,18,26],think:27,three:[],thursdai:[27,28,29],tip:13,togeth:12,tool:26,top:1,topic:26,toss:[],toward:11,trade:[6,21],tradeoff:[6,29],train:[0,1,4,15,16,26,27],transform:3,tree:[9,10],tuesdai:28,tune:1,two:[3,8,19,21,27],type:[2,4,12,26],uio:26,uncorrel:[],understand:29,uniform:[],univers:[12,25],unsupervis:14,up:[0,2,9,12,15,26,27,29],us:[0,1,2,3,7,13,19,26,27],usag:[28,29],valid:[6,21,29],valu:[5,11,17,18,23,27,28,29],variabl:23,varianc:[6,21,28,29],variou:[0,15,26,29],vector:[8,12,20,26,27],view:[0,4,10,27],visual:[1,9],vs:3,wai:[9,29],wave:2,we:26,wednesdai:28,week:[15,16,17,18,26,27,28,29],weekli:[22,29],what:[0,26,27,28,29],which:1,why:[26,27,29],wisconsin:7,wrap:[27,29],write:[4,11,28],x:[27,28],xgboost:10,yet:28,you:26,your:[0,10,15,16,26,27]}}) \ No newline at end of file +Search.setIndex({docnames:["chapter1","chapter10","chapter11","chapter12","chapter13","chapter2","chapter3","chapter4","chapter5","chapter6","chapter7","chapter8","chapter9","chapteroptimization","clustering","exercisesweek34","exercisesweek35","exercisesweek36","exercisesweek37","intro","linalg","project1","schedule","statistics","teachers","textbooks","week34","week35","week36","week37"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["chapter1.ipynb","chapter10.ipynb","chapter11.ipynb","chapter12.ipynb","chapter13.ipynb","chapter2.ipynb","chapter3.ipynb","chapter4.ipynb","chapter5.ipynb","chapter6.ipynb","chapter7.ipynb","chapter8.ipynb","chapter9.ipynb","chapteroptimization.ipynb","clustering.ipynb","exercisesweek34.ipynb","exercisesweek35.ipynb","exercisesweek36.ipynb","exercisesweek37.ipynb","intro.md","linalg.ipynb","project1.ipynb","schedule.md","statistics.ipynb","teachers.md","textbooks.md","week34.ipynb","week35.ipynb","week36.ipynb","week37.ipynb"],objects:{},objnames:{},objtypes:{},terms:{"0":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,24,26,27,28,29],"00":[0,1,5,6,9,11,26,27],"000":[1,3],"0000":9,"00000":[9,11],"000000":[5,11,26,27],"00000000e":[5,27],"000054":26,"0000e":[],"0001":[1,17],"00010403373827253124":[],"00011551":[],"000148":[],"00017471":[],"00018425":[],"00019998":5,"00024087":5,"00025531":[],"00028369228101198006":[],"00029012":5,"0003043256065368937":[],"0003153514830957865":6,"00031535148309580783":6,"0003153514830958081":[],"0003153514830958126":[],"0003153514830958235":[],"00033779":[],"00034944":5,"00040953":[],"000417932":2,"00042089":5,"000464088":2,"00046714":[],"00050694":5,"00054144":[],"00054272":[],"00057106":[],"00058016":6,"000599":[],"00060705":6,"00060708":[],"00061058":5,"00062582":[],"00062595":6,"00063364":29,"00063862":29,"00066668":6,"00068088":29,"00068734":6,"00068941":[],"00068946":6,"00069103":[],"00071305":[],"00072037":[],"00073541":5,"00074389":[],"0007480407244119591":[],"00075315":[],"000755":[],"00075534":[],"00075597":29,"00076495":6,"00076905":6,"00076919":[],"00076925":[],"0007698473260556325":6,"0007698473260556334":[],"0007698473260556339":[],"0007698473260556344":6,"0007790285013223805":[],"00079123":[],"00079129":6,"00079130":[],"00079910":29,"00079968":5,"00083346":29,"00084705":6,"00084711":[],"000854409696581":[],"00085883":[],"00085884":[],"00085889":6,"00086553":[],"00087697":6,"00088573":5,"00089187":29,"00089459":[],"00092645":[],"00092646":[],"00092647":6,"00092904":29,"00096314":5,"001":[1,2,8,13,17],"00100519":6,"0010059":[],"00102864":[],"0010479245355337968":[],"0010479245887943952":[],"0010479245926411787":[6,29],"00105081":6,"001060":11,"00106025":11,"00106677":5,"00107405":6,"00111756":6,"00112431":[],"0011526":6,"00115999":5,"0011828302640124":[],"00118368":[],"00118504":[],"00118508":6,"00118527":[],"00119699":29,"001263":[],"00128479":5,"00128634":[],"00129543e":[],"00131939":[],"001323":6,"00137818":6,"00137823":[],"00137835":[],"00139198":[],"00139705":5,"00140849":29,"00143188":[],"00145838":[],"00147197":[],"00149311":6,"0014937":[],"00149956":6,"00152117":6,"00152188":[],"00154733":5,"00154860":29,"00156376":6,"00156379":[],"00156382":[],"00158723":[],"00162091":[],"00168141":[],"00168251":5,"00174276":6,"00174508":[],"00175331":6,"00183687e":[],"00186347":5,"00187154":[],"001880":5,"00191249":[],"001988":[],"00198806":[],"00200":8,"00201915":[],"00202464":[],"00202624":5,"00202756":6,"00207627":[],"00217499":6,"0021877":[],"00219194":29,"00220229":[],"00224413":5,"00226569":[],"00228741":[],"00228742":6,"00228986":[],"00229179":20,"00232255":[],"00233155":[],"00233441":[],"0023548":6,"002381316302584885":[],"002381316302584886":6,"0023813163025848865":6,"00242954":29,"00242999":6,"00243186":6,"0024401":5,"00249435":6,"0025074":[],"0025125":[],"00252344":[],"002526":[],"0026081":[],"00266858":2,"00267599":[],"00270244":5,"00274989":6,"00275135":[],"00281835":[],"00283636":[],"00287154":[],"00289406e":[],"00289724":6,"00293838":5,"002948":[],"0030828":[],"003083":[],"003100":26,"00310113":2,"00310568":11,"003106":11,"00312361":6,"00315593":6,"003215318065760509":[],"0032153180657605116":[6,29],"0032153180657605125":[],"00321694":[],"00323332":6,"0032542":5,"00325450":29,"00327082":[],"00329264":[],"003301":6,"00330907":[],"0033955154592040923":[6,29],"003395515459204093":[],"0033955154592040944":[],"00353655":[],"00353823":5,"0035417":[],"00355118":[],"00355175":[],"003620":26,"0036237":6,"0036367":6,"00363986":[],"00369758":6,"003704":[],"003717":[],"00373019":[],"003738":[],"003759":[],"003774":[],"003788":[],"003788445263115483":5,"0038335":6,"00388494":[],"00390021":[],"00390805":[],"003909404072811217":[],"003909404072811221":[6,29],"003909404072811231":[],"00391839":5,"00393128":[],"00393744":[],"00397836":[],"00399247":[],"0039987":6,"004":[5,28,29],"00403785":[],"00406231":[],"00407056":[],"004091940707753925":[6,29],"004091940707753948":[],"0040919407077539514":[],"00410387":6,"00410478":6,"004113634617443116":[],"0041136346174431284":[],"004113634617443131":[6,27],"004113634617443135":[],"004113634617443139":[6,27],"00411363461744314":[6,27],"004113634617443141":[],"004113634617443147":[6,27],"00415473":[],"0041559863458613296":[6,29],"004155986345861364":[],"004155986345861374":[],"00415763":[],"00416461":[],"00420072":29,"00424909":2,"00424967":6,"00426027":5,"00427039":[],"00431511":[],"00432302":[],"00433417":11,"004339135748752641":9,"00437363":[],"00440346":6,"00443743":6,"00444561":[],"00445655":11,"00449439":[],"00452742":[],"004579219539673834":[6,29],"004579219539673836":[],"00458014":[],"00458671":[],"00458878":6,"004610275230656182":[6,29],"004610275230656187":[],"004610275230656294":[],"00462287":6,"00465099":29,"00465403":[],"00471782":5,"00471966":[],"00472199":6,"00472303":[],"00472512":6,"00477722":[],"00478786":[],"004788":[],"0049544":6,"004999999999999984":[],"0049999999999999845":0,"00499999999999999":[],"004999999999999994":[],"004999999999999996":[],"004999999999999997":[],"005":[],"0050000000000000044":[],"005000000000000011":[],"00503218":[],"00506501":[],"00506765":[],"00509527":[],"00512927":5,"00514913":[],"00517074":[],"00517114":6,"00523269":[],"00526348":6,"0053018":6,"00534085":23,"00537912e":[],"00538287":[],"00546438":[],"00548686":[],"0055043":[],"00554475":[],"00554552":6,"00556826":6,"00563446":[],"00564956":[],"0056799":5,"00569317":[],"00577016":[],"00577441":[],"00578738":[],"00578776":[],"00579953":6,"0058263":23,"00588657":6,"00605451":[],"00607783":6,"00611932":[],"006158537606875468":[],"006162":6,"00617499":5,"00620642e":27,"00622755":[],"00625843":[],"00626339":[],"006275173922218319":[],"00628788":[],"00630331":6,"00634741e":[],"00642221":6,"00645183":[],"00654702":[],"00658717":[],"00659545":[],"00660427":6,"00660801":[],"00661104":[],"0066916":[],"00672607":6,"00673393":[],"00673407":6,"00676387":6,"0068011":6,"0068094":[],"00681644":[],"00683748":5,"00683964":6,"00684221":[],"00689391":[],"00695357":[],"00695738":[],"00700232":[],"007012403613997257":[],"007024126888936694":[],"007024126888938144":[6,29],"0070241268889382116":[],"0070381":[],"00704468":[],"00706459":[],"00708794":[],"007098683295819775":[],"00712899":[],"00715835":[],"00719176":6,"00721135e":27,"0072559952738542694":[],"00727646693":[0,26],"007315":[26,27],"00732273":[],"007334080205654388":[],"00733497":[],"00740173378493795":[],"0074331":5,"00749579":[],"00751823":[],"00754532":[],"00758025":[],"00759119":6,"00770648":[],"00771046":[],"0077750600806588575":[],"00779538":[],"00779804":[],"00780259":[],"00781035":[],"00781918":29,"007839875253055148":[],"00784393":6,"00791019":[],"00791291":[],"00801078":[],"00803064":6,"00805074":29,"00806407":[],"00808144":[],"0080866254785146":[],"008089688732934681":[],"00813803":6,"00817056":[],"00817631":6,"00819724":[],"00820764":[],"00823002":5,"00827728":6,"008278885543361304":[],"00831018":6,"008334493024006613":[],"00834567":6,"00839839":[],"008435888964550838":[],"0084603":[],"00848904":6,"00850988":[],"00851638e":[],"00852965":[],"008550594684081035":[],"00857688":[],"008653750612762521":27,"0086649156":[0,26],"008675369724975977":5,"008675369724976501":[],"00867698":[],"00868337":[],"008754614401844201":[],"00877989":[],"008818897251043893":[],"008823125370709272":[],"00882596":[],"00886682":[],"00894639":5,"008947823579448178":[],"00895315":[],"009039197557276654":27,"00905423":6,"00910241":[],"00911938":[],"009163":[],"00916336":[],"009163470508352211":[],"009163470508352218":5,"009164545680330616":[6,29],"00917248":6,"00918914":[],"00928016":[],"009290411029763584":[],"00934327e":27,"00934499":6,"00934728":[],"00943261":[],"009450756365829578":[],"00950269957492606":[],"00953728":[],"00954257":[],"0096208":6,"00963223":[],"00965032":[],"0097018":[],"009710035292693498":[],"0097577":[],"0098629":[],"00990475":5,"00992331":6,"00996754":6,"009981":[],"00998135":[],"00it":[],"01":[0,1,2,5,6,9,11,13,17,25,26,27],"010018312644139205":[],"010018312644139219":[6,29],"010018312644139347":[],"01003763":[],"01006893":[],"0100706":6,"01011376":[],"0101339":[],"010154":[],"01023891":[],"010239":[],"01028231":[],"01028255":[],"01028995427580139":[],"01029589":[],"010331721306655144":[],"01033172130665515":[],"010331721306655165":[6,29],"01037947":[],"01039549":[],"01049472":[],"010516485576646504":[6,29],"010516485576646513":[],"010516485576652856":[],"010530":[],"01053024":[],"01056549":[],"010596520060222926":[],"01066519":6,"010687707010849668":20,"01072866":[],"01073921":[],"01073938":[],"010751622516868333":[],"01076611":5,"010902":26,"01097227":[],"0110":23,"0110407067093945":[],"01104071":[],"011076219011339788":[],"01107621901137465":[],"01107621901137467":[6,29],"01121347":[],"011225":2,"01122997":[],"0113104":6,"01132947":[],"01133171":[],"01135107":[],"0113975":[],"01152107":[],"0116726":[],"01167264":[],"01176895":[],"01178253":[],"01179792":6,"01191824":5,"01194406":[],"01199624e":[],"01199845":[],"012":[],"01200342":[],"01205286":[],"0120561":23,"0120561022775605":23,"01206708":[],"012073649439965807":[],"012073649469946107":[6,29],"012073649472576395":[],"01207435":[],"01212754811385597":[],"01219292":6,"01222822":27,"01223198":6,"01224109":[],"01227458":[],"01231917":6,"0123679":[],"01242273":[],"01242803":[],"01244295e":[],"01247056":[],"01248731":[],"012501013881387798":[],"01256189":[],"01257962":[],"012580":[],"012633802944855959":[],"01265755":27,"012658":27,"01285424":[],"01290947":6,"01294617":[],"01295356":5,"013121573975499602":[],"013121573975499604":[],"01312157406137079":[],"013121574061370796":[],"013121574062587286":[6,29],"01317074":[],"01318643":6,"01322099":[],"01322543":[],"01323884":[],"01326608":[],"01347916":6,"01348565":6,"013537225644261203":[],"01354598394614281":[],"0136671":[],"01367553":6,"01374437":[],"01381723":[],"01397146":6,"01400329":[],"01405935":6,"01406736":[],"01416528":6,"014209325470380275":[],"01422494":[],"01422853":[],"0143088":[],"01433809":5,"014436800088896274":[],"014436800088896381":[6,29],"014436800088969727":[],"01448144":[],"01449782":6,"01458337":6,"014586":27,"01458611":27,"014599964106338128":[],"0146081":6,"01463049":6,"01471878":[],"01472766":[],"01479194":[],"01487742":[],"01489692":[],"01491299":[],"01492855":[],"01494533":[],"01497781":[],"01505087e":27,"015072388895177088":[],"0150723888951771":6,"015072388895177109":[],"015072388895177157":6,"015072388895177239":[],"01509543":[],"01510929":[],"01515966":[],"01525206":[],"01530715721129232":[],"01531845":6,"01538461538462":[],"01539":[],"01542944":[],"01547992":[],"01549377":6,"01550656":[],"01551946":[],"01558197":5,"01558461":[],"0156548":[],"01572164":[],"01574091":[],"01574375":[],"01586048":[],"01588614":[],"01588619":[],"01591355407242949":[],"01594452":29,"01610677e":27,"01614046":[],"01619802":[],"016285782696017055":[],"016285782696017142":[6,29],"016285782696075054":[],"01628823":[],"01631138":[],"01633913":6,"01637147":[],"01640891":6,"0164286":[],"01644452":[],"01646892":[],"01654471":[],"01655318":6,"01655455":[],"0165866":[],"016587414993037307":[],"016587414993045335":[6,29],"016587414993045405":[],"01667873":[],"01678286":[],"01691985":6,"01693":[],"01693278":[],"0169643":5,"016970019460234076":27,"01708781":6,"01708852":6,"01710429":20,"01713366":6,"0171347":[],"01724499":5,"01725808":[],"01735584819559184":[],"017355848195591845":[],"01735584819559331":[6,29],"017355848195593312":[6,29],"017355848195593354":[],"01742183":[],"01751529":[],"01752962e":[],"01756972":[],"01761413":[],"017665":5,"01770163":[],"0177181":[],"01780471":[],"017805":[],"01783414e":27,"01788423":[],"01799917e":[],"01801774":[],"01816821":[],"01817152":[],"01817919":[],"018232":[26,27],"01826743":[],"01831036e":[],"01831130e":[],"01831200e":[],"01831207e":6,"01831251e":6,"0183718":[],"01840062":[],"01844381":[],"01844727":23,"01852793":[],"01854824":[],"01855624":[],"01866537":6,"01869785e":[],"01871582":[],"01873344":11,"01873409":[],"01873869":5,"01885351":[],"01889099091713886":[],"0189082":[],"01898855":6,"01904583":[],"01905883":6,"01907538":[],"01908936":6,"01923991":[],"01927709":[],"01929643":[],"01933058":[],"0193453":[],"01943082":[],"01944669":[],"01950574":[],"01957153":[],"019587":[],"01960258":[],"01963611":6,"01969004537138309":[],"01969145":6,"01975416527168255":[6,29],"019754165271682844":[],"01975416527179247":[],"01975848":6,"01990155":[],"02":[0,4,6,7,12,26,27],"02002781":[],"02008059":[],"0202458":[],"020246":[],"02024962":6,"0202731":[],"02035804":[],"02054837e":6,"02055673":20,"02058701":[],"020634":[],"02063403":[],"02068067":6,"02073509":5,"02082761":[],"02086933":[],"02090061":[],"0209518407597062":[],"020964":[],"02098261":6,"0211496":[],"02123176":6,"021250026482402":[],"02126934":[],"02129803":[],"02143332":[],"0214539":[],"02151219":[],"02158709":[],"02159270458799264":[],"021592704587992645":[],"021592704588021164":[],"021592704588021167":[],"021592704588021174":[6,29],"021592704588021178":[6,29],"02160095e":[],"02166269":[],"02185737":20,"02193035":[],"02195273":[],"02198702e":6,"02198703e":6,"022":[],"02200044":[],"022022882954618364":[],"02206138":[],"022075":[],"02207532":[],"02208512":6,"02212776":[],"02212893":[],"02226654":[],"02228115":6,"02229529":6,"02232779":[],"02233576":[],"02237669":[],"02251524":[],"02252765":5,"02255062":[],"022556":[],"02255619":[],"02259051":[],"02263868":[],"02283536":[],"02291121":[],"02298589":[],"02299949826036602":[],"022999498260366198":[6,29],"0229994982603662":[],"02335522":[],"02341603":[],"02344111":[],"02344605":[],"0234505":[],"02348765":6,"02354476":[],"023545":[],"02365049":6,"02376036":[],"02377668":[],"02383034":[],"02401344":[],"024023115996453476":[],"02408959":[],"0242106":[],"02421521":[],"02422503":[],"02424794":29,"02430520e":[],"02442398":[],"0245528":6,"02480935":[],"02492265":5,"02493054":[],"02498832":6,"02502101":[],"02503753":6,"02511518":6,"02521356":[],"02521597":[],"02522069":6,"025325212488352":20,"02533184":[],"02539993":[],"02541796":[],"02546675":29,"02551463":[],"02566258":[],"02568505":[],"025709":[11,27],"02586427":6,"02596838":[],"0260906":6,"02609885":[],"026121042099059435":23,"02612167":[],"02625193":8,"02625928":[],"026408391362671896":[],"02642055e":[],"02644748":[],"02649453":[],"026605727637176654":[],"026605727637184554":[6,29],"026605727637184558":[6,29],"02660572763718461":[],"026605727637184613":[],"02670606":[],"02697521163514974":[],"026986":[],"02698605":[],"02699523":[],"02701924":[],"02705154":[],"02707227":5,"02708158":[],"02712723":[],"02723445":6,"02724253":[],"0272773":23,"02747522":[],"027609773491022314":[],"02760977349102238":[6,29],"027609773491022387":[],"027609773491022394":[6,29],"027609773491022407":[],"02762405215108776":[],"02766489e":[],"02766981":[],"02768957":[],"02775293":[],"02783644":[],"02786488":[],"027865":[],"02803041":[],"02808201":[],"02818883":[],"0283589":[],"02840224":[],"02840884":[],"02857":4,"02862254":[],"02868501":[],"02897168":[],"02900131":[],"02903947":[],"02917662":[],"029177":[],"02917931":[],"02925853":[],"02926396":[],"02928941":[],"02944194":[],"029483":5,"02951554e":[],"029574060388349064":[],"029613363972682966":[],"02970592":[],"02971158":[],"029733":[26,27],"02976145":6,"02979908":[],"02994311":5,"02f":[6,21],"02it":[],"03":[1,6,27],"03012368":[],"030169":[],"03016916":[],"03026672":[],"03028379":[],"03032441e":6,"030407120354722424":[],"03054765":[],"03056589":[],"030566":[],"030693":[],"03069324":[],"03071040e":[],"03076923076924":[],"03077640549":4,"03078597":[],"03079528":[],"03080664":[],"03099776":5,"031":[5,28,29],"03105744e":27,"03123832":[],"03168642":[],"0317159":[],"03173163":[],"031846":[],"03184647":[],"03187054":[],"03187339273559067":[],"03192893":[],"03196357":6,"032":[],"03206303":[],"03224523":[],"03232013":[],"03232719":[],"03235599":[],"03251863":5,"03255258":[],"03256632e":1,"03260201":[],"03265139":[],"03267527":6,"03278964":[],"032790":[],"03279636":6,"03286166":5,"032862":5,"033":[],"0330308045180234":[],"0330308045183163":[],"0330308045183219":6,"0330308045188872":6,"0330308045190182":[],"03308408":5,"03324117":[],"03326365":[],"033264":[],"03330335":[],"03352185":[],"03352725":[],"033657685071527485":[],"033657685071527624":[],"03365768507152769":[6,29],"03371334":[],"03382304823545749":[],"03393764":11,"033938":11,"03399521":[],"034047":26,"03413302":[],"03416188":20,"0341899":[],"03423556":[],"03437582":[],"03447512":6,"034557":27,"0348171":[],"03483677":[],"03499693245669077":[],"03500198":[],"03507211":[],"03509912":[],"03510756":[],"03523311":[],"03533092":[],"03536062":[],"03536673":[],"03546779":[],"03562355":6,"03568439":6,"03570747":[],"03583342":[],"0358909447132981":[],"035909":26,"0359565":5,"03608726":20,"03610331":[],"03613949":[],"03617972":[],"03630548":6,"03634298":[],"0365416391166":23,"03654164":23,"03707133":11,"03718926":[],"03724642":[],"03739357":[],"03739417":[],"037503978471253":[],"037559":[],"03755944":[],"03761519":[],"03781367141738885":[],"03781367141738886":[],"037813671417388985":[],"03781367141738899":[],"03781367141738902":[6,29],"03791967":[],"03794112":[],"03807267":[],"038073":[],"03814292":6,"03815288":6,"03822007730337899":11,"038300":[11,27],"03850557":[],"038506":[],"03859372":[],"03893106":[],"039039":5,"0391902":[],"03932541e":[],"03952093":[],"03981057":[],"03995478":[],"0399676689527966":6,"0399676689527968":[],"0399676689527975":6,"04":[1,6,11,27],"04008203e":[],"040102":5,"04010697":6,"0402861":20,"04028979":[],"04038959":27,"04039087":[],"04046799":[],"04063602":6,"04075841e":[],"040863":[],"04086326":[],"04087592":[],"041":9,"04100738":[],"04101587":[],"041129694365319464":20,"041148729430502":[],"041148729430523":6,"0411487294305246":[],"0411487294305746":[],"041148729430595":6,"04117184":[],"04120281":[],"04179719":[],"041797190646905":[],"04184636":[],"04184782":[],"04193203":27,"04208001":[],"04208274":[],"04220758":6,"04234339":[],"042708":[],"04273116":[],"04279651270127165":[],"04284519":[],"04295757":29,"043":9,"04315108":5,"04316854":26,"04331431":[],"0433816":[],"04346721":5,"04355837":6,"04362":9,"04368707":[],"0437499":2,"04389027":6,"043927769551648":[],"04413933503955871":[],"04419696":[],"04423486":6,"04426744e":27,"04426816":[],"044334":[26,27],"0443659":[],"0444119":[],"044613":6,"04481443e":[],"04483457":[],"04495095":[],"04496448":[],"04516608":[],"04531375":[],"045314":[],"04537385":6,"04543942":6,"04554454":[],"045603":[],"0456042":[],"04566103":[],"04566964":6,"04573057":[],"04573105":[],"0457893":26,"0458":9,"0459682":[],"04615384615386":[],"04621521":[],"04637311":[],"04648335":5,"046491":[],"04649105":[],"0466":[],"04670006":[],"04673082":[],"046731":[],"04674926":26,"046785461905835435":[],"04678736":[],"04683565":5,"04691549":[],"04735806":[],"04771105":[],"047737":[],"04775904":[],"04784395":6,"04785727":[],"04787099":[],"047953":[],"04800573":26,"04818727730430286":[6,29],"04818727730430296":[],"048187277304303056":[],"04825655":5,"048257":5,"04826413e":27,"048547423739546604":[],"04860436":[],"048653428302840175":[],"048762":[],"048818":[],"048920":26,"04892055":6,"04909093":6,"04912436":6,"04926746":29,"04930820e":[],"049318":[],"049462":26,"0495569966278238":[],"0495569966278269":6,"0495569966278295":6,"0495569966278315":[],"04965397e":[],"049747":[],"04983109":20,"049858":[],"0499586":[],"049960":[],"04996008":[],"04it":[],"05":[1,4,6,9,13,21,27],"05004617":[],"05009826":6,"05023564":[],"05029894":[],"05069808":[],"05087958":[],"050910":[],"05100875":6,"051043":[],"05110883":[],"0511284278960114":11,"0512238":[],"051391":[],"051418":5,"051451":[],"05150928":[],"051649":[11,27],"051662":[],"0517473":5,"05177021":[],"05177092":[],"05177527":[],"05177695":[],"052206":[],"052227":[],"05227921801205679":[6,29],"05227921801205691":[],"05227921801205692":[],"052279218012057004":[],"052305":27,"052330":[],"05236797e":[],"052372":[],"0525063":[],"0525221":[],"05255759":[],"052581":[],"05263":9,"05268304":[],"0527437":[],"05279147850537411":[],"05288681":[],"05290417684691035":[],"05302":9,"053417":27,"05358851":[],"0536097":[],"05363779":[],"05364854":8,"05367466":[],"053678":[],"05383795":6,"053849":5,"053994":[],"05402365":[],"054182":[],"05421908":[],"054244842835462305":[],"054305":[],"054335":[],"054340":[],"05434571":[],"0543653729690416":23,"054411":[],"05447415":6,"054491":[],"05454172":[],"054582":[],"054585":[],"054617":27,"054650":[],"054655":27,"054774":[],"054785":[],"05480098":[],"05483267":[],"05491301":[],"05491655":[],"054963":[],"05505310046362":[],"05505310046363":2,"055137":[],"055153":[],"055302":[],"055320":[],"05533":9,"055337":[],"055344":[],"055369":[],"05554042":[],"055676":[26,27],"055684":[],"055734":27,"055859":[],"05587489":[],"055875":[],"055910":27,"055958":[],"055987":27,"056127":[],"056131":[],"05614483":5,"056165":[],"056169":[],"05622775":[],"05623":9,"056233":[],"056235":[],"05635863":[],"056468":[],"05648":9,"05651951":6,"056528":[],"05667":9,"056683":27,"05698425":[],"057088709963182":[],"057163681553428394":[],"05716368155342902":[6,29],"05716368155351588":[],"0572":[],"057300":27,"057393":27,"057418":27,"057457":27,"057463":5,"057469":[],"05748797":[],"05751737":[],"05756733":[],"057587":[],"057588":[],"05763394":[],"057648":[],"057686":[],"057722":27,"057835":27,"05785343":6,"057877":[],"05789007":6,"057899":[],"05796251":6,"058033":[],"058035":[],"058038":[],"058040":[],"058058":[],"05807125":6,"058088":[],"05833369":[],"058343":[],"05834332":[],"05835037":23,"058416":[],"05850532":[],"05851756":20,"058552":27,"05855607":[],"058585":[],"058596":[],"058645":27,"058663":[],"058715":[],"058732":[],"058740":[],"058816":[],"05881845":20,"05883":9,"05884":9,"058854":27,"05886306e":27,"058890":[],"05892262":[],"058948":[],"058952":27,"058955":[],"058957":[],"058974":[],"058996":27,"059013":[],"059031":27,"0590702":[],"05911793":[],"059118":[],"059180":[],"059245":5,"059272":[],"059278":[],"059294":[],"059378":[],"059437":[],"059453":[],"059601":[],"05971574":[],"059783":[],"059795":[],"059807":27,"059824":[],"059933":[],"059951":27,"059952":[],"05999":9,"059993":27,"06":[6,27],"060016":[],"060048":[],"0600658":[],"060080":[],"060083":27,"06008452":[],"060146":[],"060183":[],"06020587":6,"06021285":[],"060213":[],"060228":[],"060254":27,"060288":[],"060291":[],"060309":[],"06031681":[],"060349":26,"060369":[],"06038053":[],"060381":[],"060394":[],"060401":[],"060425":5,"06043581":6,"060441":[],"060502":[],"060507":[],"060534":[],"060550":[],"060603":[],"060649":[],"060655":[],"060711":[],"060716":27,"060756":27,"060781":[],"060822":11,"060841":[],"060863":[],"060872":27,"060875":[],"060924":[],"060963":[],"061025":[],"061028":[],"061037":[],"061048":[],"061054":[],"0610602":[],"061113":5,"061133":[],"061144":[],"061149":[],"061163":27,"061188":[],"061209":[],"06121384":[],"061272":[],"061297":[],"061308":[],"061321":[],"061359":27,"06136773":[],"061390":[],"061443":27,"061452":27,"061468":[],"06150169":[],"061502":[],"061505":[],"061509":[],"061529":11,"061538":[],"061604":[],"06160438":[],"061614":27,"061646":[],"061679":26,"061685":[],"06168748":[],"061701":[],"0617129312534124":23,"061716":[],"061725":[],"061745":[],"061765":[],"061821":[],"0618352118150005":9,"061898":[],"061917":[],"061923":[],"061966":[],"06200174":5,"062016":27,"062061":[],"062062":[],"062064":[],"06208238634231944":[],"062082386342319454":[6,29],"06208238634231953":[],"062089":[],"062097":[],"062136":[],"062142":[],"062195":[],"062197":5,"0622197":[],"062220":[],"062292565":4,"062294":[],"062295":11,"062298":[],"06229862e":[],"062305":[],"062333":[],"062337":27,"062370":[],"062391":[],"062409":[],"062411":[],"062424":[],"062435":[],"06243748":23,"062451":[],"062498":[],"062517":[],"062534":[],"062551":[],"062563":[],"062599":27,"0626202708457115":[],"062624":27,"062640":[],"062665":[],"062676":[],"062706":[],"062719":[],"062721":[],"06273487":[],"062734874360835":[],"062742":[],"062773":[],"062777":[],"06282378894371":5,"062827":[],"062834":[],"062874":27,"06287717":[],"062881":[],"062884":[],"062888":[],"062920":[],"06292011":[],"06293056":20,"062983":[],"063014":[],"063019":27,"063024":[],"063052":[],"063061":27,"063066":5,"063070":[],"063078":[],"063080":27,"063084":[],"063115":11,"063125":[],"063155":[],"063173":[],"063192":[],"06319374":[],"063241":[],"063250":[],"063260":27,"063287":[],"063343":[],"063353":[],"063364":[],"063371":[],"063375":[],"063401":[],"063443":26,"063493":[],"063513":[],"063524":[],"06353697":[],"063600":[],"063612":[],"063619":[],"063631":[],"063653":27,"063657":[],"063661":[],"063724":26,"063750":[],"063752":[],"063766":[],"063781":[],"063851":[],"063857":[],"063872":[],"063874":[],"063876":5,"06388888888888888":[],"063894":27,"063977":11,"063979":[],"064041":[],"064048":5,"064052":[],"064057":[],"064062":[],"064074":27,"064081":[],"064108":[],"064110":[],"064176":[],"064184":[],"064232":[],"064274":[],"064284":[],"06432223":[],"064329":[],"064384":[],"064388":[],"064390":[],"064431":[],"06444":9,"064442":[],"064459":[],"064461":[],"064469":[],"064483":[],"064501":27,"064526":[],"06453579006728315":[],"06453579006728322":[6,29],"064538":[],"064573":[],"064577":[],"064604":[],"064606":27,"064609":27,"064618":[],"064623":[],"064634":[],"064637":[],"064648":[],"064658":[],"064681":[],"064684":[],"064699":27,"064701":[],"064753":[],"064759":[],"064787":[],"064793":27,"064823":[],"064838":[],"064859":[],"06486061":[],"064880":[],"064896":[],"06491736":6,"064918":[],"064931":[],"064937":[],"064964":[],"06497046":[],"064974":[],"065020":[],"065026":26,"065030":[],"065032":[],"065077":27,"065095":[],"065101":5,"065106":5,"065110":[],"065118":[],"065165":[],"065168":[],"065175":[],"065183":[],"065228":[],"065237":[],"065249":[],"065252":[],"065268":5,"065282":[],"065314":[],"065348":[],"065349":[],"065350":[],"065378":27,"065389":[],"06538944":[],"065390":27,"065410":27,"06547790180152352":[6,29],"06547790180152353":[],"06547790180152355":[6,29],"06547790180152357":[],"06547790180152363":[],"065514":[],"065537":[],"065557":[],"065575":[],"065641":[],"065657":[],"065680":[],"065730":[],"065761":[],"065796":[],"065826":[],"065846":[],"065869":[],"065892":[],"065942":[],"065952":[],"065973":[],"065974":5,"065983":[],"066006":[],"06602663":[],"066051":[],"066056":[],"066064":[],"066066":27,"066093":[],"066110":[],"066189":[],"066194":[],"066209":[],"066220":[],"066223":[],"066228":[],"066241":[],"066270":[],"066284":[],"066309":[],"066317":[],"066340":[],"066348":[],"066356":[],"06637":9,"066388":[],"066449":[],"066474":27,"066500":27,"066519":[],"066528":[],"066553":[],"066556":[],"066558":[],"066569":[],"066580":[],"066590":[],"066609":[],"066612":27,"06662062":[],"066647":27,"066651":[],"066654":[],"066656":[],"066670":[],"066672":[],"066675":[],"0666807":2,"066700":5,"066714":[],"066728":[],"066729":[],"066741":[],"066742":5,"066753":[],"066787":27,"066789":[],"066808":[],"066828":[],"066837":[11,27],"066850":[],"066854":27,"066870":[],"066873":[],"066919":[],"066934":[],"06695337":[],"066954":[],"066969":[],"066999":27,"067011":[],"067079":[],"067089":[],"067110":[],"067141":[],"067162":[],"067183":5,"067191":[],"067213":[],"067224":[],"067233":[],"067236":[],"067240":[],"06724062":5,"067282":[],"067288":[],"067392":[],"067400":[],"067402":[],"067432":[],"067437":[],"067443":[],"067449":[],"067484":[],"067489":[],"067499":[],"067506":[],"067515":5,"067525":[],"067527":[],"067544":[],"067547":[],"067560":[],"067585":[],"067593":[],"067597":[],"067599":[],"067612":[],"067615":[],"067620":[],"067622":[],"067628":[],"067637":27,"067675":[],"067693":[],"067707":27,"067709":[],"067710":[],"067712":[],"067714":[],"067719":[],"067720":[],"067723":[],"067767":[],"067769":[],"067775":[],"067787":[],"067854":[],"067856":[],"067864":[],"06786925114666595":[],"067873":[],"067905":[],"067907":[],"067908":[],"067934":[],"067956":[],"067967":5,"067984":11,"067990":[],"068007":[],"068019":[],"068059":[],"068075":[],"068087":[],"068093":[],"068094":[],"068098":[],"068103":[],"068104":[],"068110":[],"068120":[],"068136":[],"068163":[],"068168":[],"068170":5,"068197":[],"068201":[],"068206":[],"068210":[],"068215":[],"068239":[],"06826023e":[],"068261":5,"068264":27,"068297":[],"068307":27,"068318":[],"068339":[],"068345":[],"068358":[],"068363":[],"068365":[],"068375":[],"068376":[],"068391":[],"068417":[],"068418":[],"068441":27,"06844519414009438":[],"06844519414009442":[],"06844519414009444":[6,29],"06844519414009445":[6,29],"068461":[],"06848116e":27,"068500":[],"068505":[],"068517":[],"068521":11,"068524":[],"068525":[],"068538":[],"068541":[],"068546":5,"068573":[],"06858699":[],"068593":[],"068608":[],"06861085":[],"068619":[],"068622":[],"068629":27,"068642":[],"068643":[],"068654":[],"068659":[],"068664":[],"068676":5,"068689":[],"068701":11,"068724":[],"068727":27,"068735":[],"06875458":26,"068776":[],"068782":[],"068792":[],"068799":[],"068806":[],"068809":[],"068831":[],"068832":[],"068841":[],"068852":[],"068867":[],"068876":[],"068906":27,"068912":5,"068920":[],"068937":[],"068953":[],"068965":[],"068991":[],"068992":[],"068999":[],"069010":[],"069013":[],"069015":[],"069028":27,"069036":[],"069042":[],"069044":[],"069065":[],"069072":11,"069080":[],"069085":[],"069091":[],"069125":[],"069129":5,"069181":[],"069187":[],"069190":[],"069202":[],"069203":[],"069231":[],"069233":11,"069242":[],"069265":[],"069268":[],"069275":[],"069276":[],"069281":[],"069283":[],"069285":[],"069291":[],"069312":11,"069315":5,"069320":[],"069343":[],"069355":[],"069386":[],"069408":[],"069409":[],"069421":[],"06942103":[],"069436":[],"069441":[],"069452":27,"069454":[],"069455":[],"069465":[],"069475":[],"069476":[],"069494":[],"069506":[],"069522":[],"069534":[],"0695405":[],"069578":[],"069583":5,"069584":[26,27],"069612":[],"069617":11,"069621":11,"069626":[],"069640":[],"069651":[],"069654":[],"069667":[],"069669":5,"069672":[],"069681":[],"069685":[],"06969872":[],"069738":[],"069748":[],"069754":[],"069782":[],"069796":[],"069801":[],"069816":[],"069823":5,"069838":[],"069845":[],"069847":[],"069855":[],"069873":5,"069877":[],"069884":[],"069886":11,"069888":[],"069903":[],"069907":[],"069911":11,"069912":[],"069977":[],"06it":6,"07":[6,27],"070018":[],"070028":[],"070034":[],"07003926":[],"070043":[26,27],"070056":[],"070071":[],"070074":[],"070092":[],"070096":[],"070105":[],"070119":11,"070127":[],"070139":11,"070141":[],"070146":27,"070150":[],"07016":9,"070161":5,"070163":[],"07017":9,"07019610e":[],"070213":27,"070216":[],"070222":[],"070241":[],"070245":[],"070276":[],"070288":[],"070302":11,"070305":5,"070333":[],"070339":[],"070343":[],"070364":[],"070370":[],"070379":[],"070381":11,"07039":9,"070419":[],"070428":[],"070429":[],"070443":[],"070450":[],"070453":[],"070457":27,"070466":[],"070486":5,"070491":[],"070492":[],"070494":[],"070518":[],"070562":[],"070576":[],"070577":5,"070582":27,"070585":[],"070601":[],"070612":[],"070618":[],"07062318":6,"070625":5,"070630":[],"070635":[],"070671":[],"070700":[],"070737":27,"070769":11,"070775":[],"070794":[],"070797":5,"070810":[],"070835":[],"070860":[],"070871":[],"07087395e":[],"070874":[],"070887":[],"070889":[11,27],"070911":[],"070936":[],"07093644":[],"070945":[],"070950":[],"070959":[],"070961":[],"070972":[],"070975":11,"070986":27,"07099747918547346":[],"071003":[],"071006":[],"071009":[],"071022":[],"071044":[],"071051":[],"071071":[],"071084":[],"071086":11,"071105":[],"071106":[],"071112":[],"071115":[],"071120":[],"071122":[],"071136":[],"07115":9,"071164":[],"071185":[],"071239":[],"071243":[],"071284":[],"07128434":[],"07129539":[],"0712953943627344":[],"0713":[0,26],"071314":5,"071321":11,"071326":[],"071329":[],"071387":[],"071388":[],"071392":[],"071394":[],"071420":5,"071421":[],"071437":[],"071438":[],"07143876":[],"071441":[],"071445":[],"07145103":11,"071453":11,"071455":5,"071460":[],"071467":[],"071476":[],"071480":[],"0714956":[],"071501":5,"071505":[],"071510":[],"071515":[],"071523":[],"071529":[],"071534":5,"071547":[],"071552":[],"071567":[],"071569":[],"071585":[],"071600":[],"07160048164232467":[],"07160048164232538":[6,29],"0716004816423254":[6,29],"07160048164248561":[],"071604":[],"071614":[],"071619":11,"071632":[],"071641":[],"071659":5,"071666":[],"071698":[],"071705":[],"071741":[],"071751":[],"071767":[],"071804":[],"071836":[],"071852":[],"071853":11,"071897":[],"071901":[],"071906":[],"071908":[],"071920625289855":[],"071921":[],"071922":[],"071935":[],"071942":[],"071946":[],"071949":[],"07195822":[],"071982":11,"071991":[],"072000":27,"072008":[],"072011":[],"072016":11,"07201957":[],"072021":[],"072041":[],"072084":[],"072091":[],"072111":27,"072122":5,"07212695":[],"072127":[],"072132":27,"072145":[],"072147":[],"072150":[],"072176":11,"072183":5,"072184":[],"072186":[],"072198":[],"072216":[],"072222":[],"072240":[],"072242":[],"072254":[],"072290":[],"072334":11,"072342":[],"072344":11,"072348":[],"072352":[],"072361":[],"072364":[],"072365":5,"072370":[],"072384":11,"072398":5,"072435":[],"072442":[],"072452":[],"072468":11,"072471":[],"072474":11,"072476":27,"072483":[],"072492":11,"072493":11,"072515":5,"072521":[],"072522":5,"072523":[],"072529":[],"072530":[],"072554":[],"072558":[],"072569":[],"072575":[],"072597":[],"072598":[],"072611":[],"072617":[],"072618":[],"072620":[],"072621":27,"072638":5,"07264063e":[],"072647":[],"072651":[],"072652":[],"072682":11,"072705":[],"072750":[],"072751":[],"072783":[],"072798":[],"072813":[],"072824":[],"072828":[],"072839":[],"07285":3,"0728785":[],"072898":[],"07291729":[],"072918":[],"072928":[],"072955":[],"072958":[],"072959":11,"072970":[],"072974":[],"07297557":[],"072994":[],"073013":[],"073016":[],"073038":[],"073044":[],"073045":11,"073052":[],"073062":[],"073080":[],"073105":[],"073120":[],"073127":5,"073134":[],"073136":[],"073159":[],"073163":[],"073183":[],"07319349":[],"073206":[],"073225":[],"073230":[],"073236":5,"073238":[],"07325301":[],"073280":[],"073298":[],"073299":[],"073300":5,"073303":[],"073320":[],"073325":[],"073337":11,"07333967":26,"073344":[],"073362":[],"073363":[],"073368":[],"073369":[],"07337358":[],"073374":[],"073380":[],"073388":[],"073391":[],"07339251":[],"073398":[],"073400":[],"073423":[],"073433":[],"073436":[],"073449":[],"07345504":[],"073465":[],"073477":[],"073489":[],"073497":5,"073506":[],"073520":[],"073531":[],"073548":[],"073567":[],"073575":[],"073583":[],"073585":5,"073597":[],"073598":[11,27],"073601":[],"073624":[],"073629":[],"073630":[],"073634":[],"073644":27,"073647":[],"073657":[],"073668":[],"073684":[],"073695":[],"073699":[],"073716":[],"073717":11,"073719":[],"073750":[],"073761":[],"073764":[],"073768":[],"073777":[],"073810":[],"073811":[],"073816":[],"073827":[],"073831":[],"073904":[],"073907":[],"073925":[],"073938":[],"073949":[],"073955":11,"073959":[],"073985":[],"0739975":[],"074004":[],"074006":[],"074007":5,"074019":[],"074034":[],"074046":[],"074050":[],"074061":[],"074067":[11,27],"074084":27,"074102":11,"074111":[],"07413172":[],"074144":[],"074149":[],"074161":11,"074180":[],"074183":[],"074187":[],"074193":11,"074203":5,"074210":27,"07421084":5,"074230":5,"074237":[],"074258":[],"074276":5,"074293":5,"07430684":[],"074310":5,"074323":27,"074336":[],"074346":[],"074375":[],"074401":[],"074418":[],"074420":[],"074453":[],"074480":[],"074488":[],"074495":[],"074513":[],"074530":[],"074552":[],"07456491":5,"074569":[],"074573":[],"074592":[],"074613":[],"074653":[],"074666":[],"074671":5,"074684":[],"074692":11,"074693":[],"074708":[],"074715":[],"074729":[],"074732":[],"074743":[],"074752":11,"074760":[],"074763":[],"074773":[],"074818":[],"074819":[],"074833":[],"074838":[],"074839":[],"074845":[],"074868":[],"074881":[],"074905":[],"07490892":6,"074922":[],"074945":[],"074951":[],"074969":[],"074971":[],"074980":[],"074983":5,"074984":[],"074986":[],"074992":[],"075012":[],"075014":[],"075017":[],"075020":[],"075028":[],"075035":5,"075043":[],"075045":11,"075047":[],"075050":[],"075058":[],"075061":[],"075064":11,"075079":[],"075102":[],"075115":5,"075121":[],"075136":[],"075138":[],"075145":[],"075149":[],"075181":[],"075188":[],"075192":[],"075195":[],"075201":[],"075222":[],"075224":[],"075225":[],"075227":[],"075228":[],"075241":[],"075262":[],"075263":11,"075277":[],"075302":[],"07532297":[],"075330":11,"075335":[],"075356":[],"07535606":[],"075389":[],"075400":[],"075418":11,"075422":5,"075433":[],"075455":11,"075498":[],"075529":11,"075569":5,"07557024":[],"075573":[],"075586":[],"075587":5,"075590":11,"075594":[],"075612":[],"075618":[],"075620":11,"075629":[],"07565581":[],"075727":[],"075728":[],"075760":[],"075770":[],"075779":[],"075782":[],"075797":[],"075816":27,"075820":[],"075829":[],"075840":[],"075863":[],"075887":5,"075889":27,"075899":[],"075911":11,"075952":[],"075958":5,"075959":[],"075982":[],"075987":[],"075996":[],"076007":5,"076022":[],"07602623":[],"076055":[],"076066":[],"076079":[],"07608141":[],"076096":[],"076097":[],"076112":5,"076121":[],"076130":[],"076150":27,"076173":[],"076204":[],"076216":[],"076248":[],"076259":[],"076264":[],"076270":[],"076282":[],"076288":11,"076306":[],"076313":5,"076315":[],"076324":[],"076329":[],"076338":[],"076349":[],"076371":[],"076389":[],"076403":[],"076408":11,"07641937":29,"076454":[],"076491":5,"0764924":6,"076518":[],"076519":5,"076525":[],"076532":[],"07656896":[],"076570":[],"076573":[],"07657669":[],"076588":11,"076589":[],"076604":[],"076612":5,"076619":[],"076628":[],"076632":[],"076637":[],"076665":[],"076697":[],"076726":[],"076729":[],"076760":[],"076775":[],"07678":9,"076780":[],"076783":[],"076795":[],"076804":[],"076818":[],"076819":[],"076821":[],"076843":[],"076862":[],"076866":[],"076867":[],"07687896":26,"076886":[],"076895":[],"076905":[],"07692307692307693":9,"076927":[],"076935":[],"076938":[11,27],"076942":[],"076955":[],"076958":[],"076990":[],"077003":[],"077004":[],"077009":[],"077012":11,"077013":[],"077015":[],"077022":[],"077033":[],"077050":5,"077062":[],"0770789":[],"077079":[],"077083":[],"077091":[],"077107":[],"0771079":[],"077108":[],"077131":[],"077140":[],"077144":[],"077158":[],"077164":[],"077168":27,"077182":11,"077200":[],"077203":[],"077210":[],"077211":[],"077219":[],"077228":[],"077280":5,"077291":[],"077338":[],"077349":[],"07735703":[],"077359":[],"077372":[],"077375":[],"077387":5,"077397":[],"077402":[],"077411":5,"077419":[],"077425":[],"077428":[],"077452":[],"07745809e":27,"077460":27,"077467":[],"077491":11,"077499":[],"077511":[],"077521":11,"07752967":26,"077536":5,"077604":[],"077606":[],"077607":[],"077611":11,"077620":[],"077644":[],"077663":[],"077666":5,"077676":11,"077688":[],"077690":11,"077706":[],"077711":5,"077713":[],"077724":11,"07777777777777778":1,"077785":[],"077793":[],"077822":[],"077825":5,"07783152589039466":5,"077846":[],"077849":11,"07789772e":[],"077910":11,"077921":[],"077930":[],"077948":[],"077949":[],"077954":[],"077970":5,"077978":[],"077986":[],"078033":[],"078040":[],"078042":[],"078082":[],"078101":[],"078110":27,"078133":[],"078143":[],"078157":[],"078158":[],"078165":[],"078167":[],"0782":[],"07820":9,"078211":[],"078253":[],"078254":[],"07826171e":[],"078265":[],"078269":[],"078293":[],"07829878":[],"078299":[],"078300":[],"078314":[],"078323":[],"078326":[],"078333":11,"078336":27,"078426":[],"078432":[],"078436":[],"07844310540708652":[],"078484":[],"078508":[],"078521":[],"078540":27,"078542":[],"078548":[],"078555":11,"07858099596662704":[],"078582":[],"078587":[],"078609":[],"07864":9,"07864140367329474":11,"078646":[],"078651":[],"078652":[],"078667":[],"078672":[],"078697":[],"078704":[],"078707":[],"07871":9,"078710":[],"078715":[],"078719":[],"078725":5,"078749":[],"078759":[],"078768":[],"078771":5,"078776":[],"07878641":[],"078795":[],"078812":[],"078845":[],"078860":[],"078880":[],"078899":[],"078901":11,"078908":[],"078911":[],"078926":[],"078942":[],"078962":[],"078974":27,"07897647347778382":[],"07898165660100093":[],"078986":27,"078987":11,"078992":[],"078995":5,"078999":[],"079001":[],"079021":[],"07903849":[],"079043":[],"079056":[],"079110":[],"079142":[],"079152":[],"079157":[],"079158":[],"079179":[],"079214":[],"079218":[],"079224":[],"079225":[],"079242":[],"07925220e":27,"079260":[],"079264":[],"079268":[],"079273":[],"079278":[],"079281":[],"079298":[],"079306":[],"079319":11,"079333":[],"079347":[],"079368":5,"079379":[],"079383":[],"079389":[],"079405":[],"079406":[],"079424":[],"07943621":[],"079437":[],"079438":[],"07944154":[20,26],"079443":[],"079447":[],"079448":[],"079476":[],"079483":[],"079486":[],"079489":[],"079493":[],"079495":[],"079499":[],"0795449":[],"079546":11,"079553":[],"079558":[],"079580":[],"079609":[],"079615":5,"079618":[],"079622":[],"079631":[],"079649":11,"079671":5,"079676":[],"079680":[],"07968918676726029":[],"0796891867672603":[6,29],"079704":[],"079719":[],"079729":5,"079741":[],"079745":[],"07978179":[],"079786":5,"079814":5,"079819":[],"079823":[],"079827":[],"079839":[],"079847":[],"079856":[],"079858":11,"079860":11,"079882":[],"079893":[],"07989327":[],"079901":[],"079944":[],"079946":27,"079948":[],"07995375":[],"079958":11,"079991":[],"07999999999998":[],"07e":[],"08":[9,23,27],"080026":[],"080028":11,"080048":[],"080056":[],"080060":[],"080061":[],"080070":[],"080098":[],"080106":[],"080120":[],"080137":[],"080168":5,"080200":[],"080249":[],"080256":[],"080288":[],"080295":[],"080297":[],"080305":[],"080312":[],"080319":[],"080332":[],"080334":[],"080343":[],"080347":[],"080370":[],"080374":[],"080384":11,"080395":[],"080406":[],"080407":[],"080410":[],"08041015":[],"080434":[],"08043851":5,"080455":[],"080479":[],"080502":[],"080517":[],"080518":[],"080548":[],"080555":[],"080571":[],"080582":[],"080588":[],"080593":[],"080616":27,"080630":[],"080635":[],"080641":[],"080642":[],"080647":[],"08066746e":[],"080699":[],"080706":[],"080738":[],"080764":27,"080780":[],"080801":[],"080821":[],"080839":[],"08084392":[],"080847":[],"08085812":[],"08086066":[],"080863":11,"080867":5,"080914":[],"080916":[],"080922":[],"080965":11,"080973":[],"080979":5,"080984":[],"080998":[],"081007":[],"081022":[],"081046":[],"081072":[],"081100":[],"081140":[],"081144":[],"081163":[],"081194":[],"081198":[],"081211":[],"081216":[],"081223":[],"081233":[],"081253":[],"081258":[],"081262":[],"081276":27,"08131003":6,"081311":[],"081315":5,"081321":[],"081334":[],"081343":[],"081357":[],"081388":[],"081395":[],"081396":[],"081431":[],"081468":[],"081471":[],"0814721":[],"081481":[],"081483":[],"081503":[],"081520":[],"081533":[],"081540":[],"081547":[],"081559":[],"08156108":6,"081576":[],"081613":[],"081620":[],"081637":[],"08165104":27,"081655":27,"081699":11,"081707":[],"081728":5,"081773":[],"0818000034712947":[],"081827":[],"081832":[],"081837":[],"081848":[],"08188077":[],"081881":[],"081932":[],"081953":[],"081970":11,"081977":5,"081988":[],"081989":[],"081990":[],"082014":[],"082027":[],"082040":[],"082089":[],"082105":[],"082122":[],"082139":[],"082140":[],"082145":[],"082158":[],"082166":[],"082174":[],"082183":[],"082189":[],"082204":[],"082232":11,"082267":[],"082269":[],"082282":[],"082285":[],"082286":[],"082288":[],"082295":[],"082315":[],"0823185":[],"082322":[],"082339":11,"0823594":26,"08237141":[],"082385":11,"082424":[],"082426":[],"082431":[],"082449":11,"082451":[],"082487":5,"082501":[],"082503":[],"082509":11,"08251519":6,"082525":[],"082527":[],"082536":[],"082553":[],"082569":[],"0825971":[],"082604":[],"082620":[],"082642":[],"082653":27,"082683":[],"08271198519070039":[],"082716":[],"082756":[],"082779":[],"082786":[],"082791":[],"08281969e":[],"082839":[],"082846":[],"082852":[],"082874":[],"082875":26,"082877":[],"082896":[],"082900":[],"082921":[],"082939":[],"082990":[],"082991":5,"08299273e":6,"083000":[],"083015":[],"083021":[],"083026":[],"083051":[],"083053":[],"083087":[],"083102":[],"083111":[],"083151":[],"083152":[],"083159":[],"08318298e":1,"083217":[],"083220":[],"08322642264994606":[],"083227":[],"083242":[],"083251":[],"083272":[],"083276":[],"08328216846752691":[],"083300":[],"083319":11,"083320":[],"083321":[],"08333333333333333":[1,9],"083348":[],"083361":[],"08336233266":4,"083380":[],"083399":[],"083404":[],"083417":[],"08343519179767796":[],"083441":[],"083446":[],"083485":[],"083495":[],"083506":[],"083517":5,"083527":[26,27],"08352721390288316":26,"083555":[],"08356774001062162":[],"083573":[],"083579":[],"083600":[],"083627":[],"083629":[],"08363087":[],"083648":[],"083681":[],"083685":[],"083691":[],"083692":[],"083694":[],"08370888":[],"08373167e":[],"08374032367704139":[],"08376632":[6,27],"083766322923899":[6,27],"0837663229239016":[],"0837663229239025":[],"0837663229239043":[6,27],"083772":[],"083779":[],"083799":[],"083829":[],"083832":[],"083834":5,"083849":[],"083859":[],"083899":[],"083913":[],"083935":[],"083977":[],"084000":[],"084006":[],"084008":[],"084017":[],"084018":[],"084019":[],"084024":[],"084028":[],"084032":[],"084076":27,"084079":11,"084086":[],"084092":[],"084096":[],"084101":[],"084107":[],"084124":[],"084147":[],"084151":[],"084157":[],"084184":[],"084187":[],"084199":5,"084223":[],"084224":[],"084226":[],"084230":[],"08426840630693411":[],"08426840630693412":[6,29],"08426840630693413":[6,29],"084269":[],"084277":[],"084278":[],"08428156":[],"084282":5,"084286":[],"084364":[],"084391":[],"084400":[],"084408":[],"084414":[],"084422":[],"084426":[],"084434":[],"084438":[],"084444":[],"084476":[],"084477":[],"084484":[],"08448544e":[],"084487":11,"084523":[],"084536":[],"084541":[],"084549":[],"08455":9,"084570":[],"084604":[],"084624":[],"084629":[],"084636":11,"084645":[],"08464758160254343":[],"084657":[],"084663":[],"084666":11,"084670":[],"084682":[],"084683":[],"084685":[],"084702":[],"084728":[],"08474":9,"084777":[],"084809":27,"084846":[],"084862":[],"084904":[],"084906":[],"084909":[],"084912":[],"084920":[],"084927":[],"084936":[],"084965":[],"084977":[],"084979":[],"085010":[],"085018":[],"08502083":[],"08502185":23,"085027":[],"085044":[],"085104":11,"085105":[],"085108":[],"085113":[],"085154":11,"085163":[],"085165":[],"085167":[],"085184":[],"085185":[],"085210":[],"085224":[],"085242":[],"085249":[],"085252":[],"08526352":[],"085264":[],"085272":[],"085290":11,"08529655":23,"085298":11,"085345":[],"085361":[],"085368":[],"085384":[],"085405":[],"085410":[],"085416":[],"085447":5,"085456":[],"085484":[],"08549564e":[],"08549632935144091":[],"08551306":6,"085571":[],"085603":[],"085655":[],"085716":[],"085764":[],"08576932":6,"085776":[],"085778":[],"085876":[],"085879":[],"08593216":6,"085936":[],"086021":[],"086074":[],"086076":[],"086109":[],"08611111111111111":1,"086112":[],"08612280083325631":[],"086154":[],"086174":[],"086178":[],"086203":[],"086249":[],"086250":[],"086257":[],"086262":[],"086290":11,"086303":[],"08630331":[],"086322":[],"086358":[],"086409":[],"086420":[],"086440":[],"086460":[],"086518":[],"086524":[],"086567":[],"08657894597048958":[],"086608":[],"086610":[],"086624":[],"086647":[],"086665":[],"086679":[],"086690":11,"086705":[],"08673755293381497":[],"086802":[],"086807":[],"086809":[],"086841":[],"08684675":[],"086872":[],"086877":[],"086888":[],"086890":[],"08690":9,"086904":[],"086925":[],"086942":5,"086951":[],"086956":[],"086994":[],"086997":[],"086999":[],"087081":[],"087208":[],"087224":[],"087235":[],"087315":[],"08732305":[],"08734015":[],"087372":[],"087377":[],"087386":[],"087387":[],"08741274":[],"087419":[],"08743287e":[],"087470":[],"087501":[],"087520":[],"087573":[],"08758":9,"08759":[],"087615":[],"087689":[],"087746":[],"087833":[],"087834":[],"087848":[],"087860":[],"087878":[],"087887":26,"087891":[],"087894":[],"087910":[],"08791994":20,"088026":11,"088029":[],"088037":[],"088074":[],"088104":[],"088119":[],"088164":[],"08816688":[],"088176":[],"0881981":5,"088201":[],"088212":[],"08821204":[],"088238":[],"088261":[],"088271":[],"088275":[],"088317":[],"088348":[],"088361":[],"088425":[],"088460":[],"088476":[],"088488":[],"088502":[],"088506":[],"088519":[],"088526":[],"088560":[],"088606":[],"088611":[],"088631":[],"08871404":5,"08873443359350565":[],"08874264":[],"088754":[],"088758":[],"088760":[],"088765":[],"088778":[],"088809":[],"08881497884574564":27,"088816":[],"088825":[],"088853":[],"08887708":[],"08888888888888889":1,"08889302":[],"08891672":5,"088917":5,"088926":[],"088946":[],"088974":[],"088982":[],"088995":11,"08902":9,"089040":[],"089041":[],"089059":[],"089063":[],"089082":[],"08910476e":[],"08917679":[],"089177":[],"08918584":[],"089206":[],"089212":[],"08923158":[],"089233":[],"089240":[],"08924325e":[],"089246":[],"089306":[],"089310":[],"089329":[],"089355":[],"0893679":[],"089369":[],"089374":[],"089425":[],"089441":[],"08945041":[],"089501":[],"089505":[],"089524":[],"0895387":27,"089539":27,"089551":[],"089614":[],"089641":[],"08966600e":[],"089683":[],"0896981":[],"089744":[],"089758":[],"089775":[],"089857":[],"089924":[],"08994018e":27,"08996":9,"089982":[],"08it":6,"09":[1,27],"090014704675496":[],"090028":[],"090039":[],"090128":[],"090174":[],"090229":[],"09023660662586945":[],"090241":[],"090268":[],"090270":[],"09030825":[],"090337":[],"090349":[],"09034902":[],"0903549":[],"090356":[],"090358":[],"090424":11,"090451":[],"090503":[],"090564":[],"090620":[],"090630":[],"090649":[],"090694":[],"090730":[],"090755":[],"090768":[],"0907788":[],"0907831":[],"090828":[],"090830":[],"09083636328656121":[],"090849":[],"09085624":[],"090888":[],"090923":[],"090929":[],"091031":[],"091044":[],"091051":[],"091060":[],"091113":[],"09117221":[],"091209":[],"091224":[],"091236":[],"091266":[],"091311":[],"091315":[],"091340":[],"091349":[],"091363":[],"091369":[],"091376":[],"091414":[],"091416":[],"091418":[],"091426":[],"091440":[],"091445":[],"091477":[],"09149881":[],"091579":[],"091620":[],"091630":[],"091647":[],"091659":[],"09166666666666666":1,"091696":[],"0917":9,"09170751":[],"09172408":[],"09172409":6,"091772":[],"091891":[],"091896":[],"09189927":[],"091996":[],"092007":[],"092066":[],"092126":[],"09216046":[],"09218919":20,"092249":[],"092290":[],"0923":[],"092409":[],"092478":[],"09251":9,"09252364":[],"092524":[],"092621":[],"092724":[],"092769":[],"092879":[],"09289204":[],"09297039":[],"092998":[],"093078":[],"09309965420717024":[],"09318696260700278":[],"09322328e":27,"093247":[],"0934597075922044":[],"093497":[],"093556":[],"093593":[],"093624":[],"0938":[],"093844":[],"093930":[],"093979":[],"09404851e":[],"094082198961999e":6,"0940821989624095e":[],"0940821989643363e":6,"094082198966615e":[],"0940821989673748e":[],"094139":[],"094227":[],"094252":[],"094362":[],"09440475":[],"094404754965417":[],"09444444444444444":1,"094472507965532":[],"094477":[],"094610":[],"094619":[],"094657":[],"094722":[],"09480900e":[],"094812":[],"094848":[],"094867":[],"094925":[],"09497154":26,"095171":[],"095184":[],"095313":[],"095315":[],"09533574":[],"095510":[],"095522":[],"095617":[],"095702":[],"0957156":[],"09575221":[],"095871":[],"095902":[],"095935":[],"09609807":5,"096173":[],"09631112":[],"09634375e":[],"096472":[],"096516":[],"09652672":[],"096623":[],"09662862":[],"09672929714683368":[],"09677394":[],"096833":[],"096845":[],"096872":[],"096893":[],"096894":[],"097":[],"097111":[],"097360":[],"09739330e":[],"097396":[],"097435":[],"09744":9,"09744272":[],"09759873":[],"097602":[],"097634":[],"097710":[],"09772449":[],"09780":9,"09781386":[],"09789363e":[],"09791":9,"098069":[],"098135":[],"09846636e":[],"09848429e":27,"09849763":[],"098498":[],"09856879":[],"09861229":[20,26],"09862064":20,"09873132":[],"098802859381565":[],"099":[],"09903804":8,"0991919894927399":[],"09919198949274803":[6,29],"099191989493334":[],"09951287404314545":1,"09963157":13,"099648":[],"09964817":[],"099777":[],"09989861":[],"0n":[0,26],"0s":[],"0x11da42d90":[],"0x11db14850":[],"0x11e031e50":[],"0x11e09b370":[],"0x11e77b070":13,"0x11ecf9850":13,"0x128ee8850":[],"0x12946f2e0":[],"0x1336a6040":[],"0x133c2e1c0":[],"0x138098070":[],"0x1387541c0":[],"0x13b240850":[],"0x13b8c72e0":[],"0x7fad10f9a280":[],"0x7fad20f69be0":[],"0x7fd098df3280":[],"0x7fd0a9063be0":[],"1":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,22,23,24,25,28,29],"10":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,22,23,24,26,27,28,29],"100":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,20,23,24,26,27,28,29],"1000":[0,1,2,4,5,8,11,13,14,19,23,26,27],"10000":[2,5,6,10,11,13,23,28,29],"100000":8,"10001":10,"1001":[9,23],"1002":23,"1003":23,"10030":9,"100303":[],"100358":[],"10044225464078282":[],"1005":23,"10056192":20,"100670":[],"1007":28,"10077114273548984":[6,29],"1009":23,"10097956":[],"1011":23,"101292":6,"1013":23,"1013904243":23,"10141413e":6,"10148978":20,"1015":23,"10188442":[],"102":27,"10207647":[],"1022233262115424":[],"10222333":[],"10222868":[],"1023":23,"10230":9,"1023111":[],"10235180e":[],"1023858":[],"1024":3,"102401":[],"102449":[],"10254846":20,"1026":23,"1027":23,"1028":[],"103":1,"1030":23,"10302062":[],"10308387":[],"103257":[],"10326551":20,"10340":9,"1037":23,"10378326e":1,"1038":23,"103822":[],"10391807":6,"10398646080125035":[],"10398646080125036":[6,29],"10398646080125037":[6,29],"1040":23,"10405456":11,"10427270e":[],"10430":9,"104411":[26,27],"10441496e":[],"10453697e":27,"10455924":[],"1047":23,"10486183":[],"104862":[],"10505137":[],"105161":[],"105169":[],"10516924":[],"10520":9,"10555555555555556":1,"1056":[],"10572":29,"10589577":5,"106":[],"106095":[11,27],"10620135":[],"10635754":26,"106431":26,"10659682":[],"10686259":[],"106971":[],"107":[],"10712245e":[],"1078":29,"10790125813226321":27,"108":[6,9],"108359":[],"10835935":[],"10880921":[],"109":[],"10913":6,"10931453":6,"10950766":[],"10951119":26,"10960":9,"1096767776832326":[],"10967678":[],"1098906":[],"10it":[],"10th":9,"10x":[0,26],"11":[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,20,21,22,23,24,25,26,27,28,29],"110":[],"1100":23,"1101":23,"11022302e":[],"11029323":[],"11033412":26,"11046407e":27,"11046771":[],"111":[1,7,12],"11100":9,"11167579":[],"11202337":29,"1122558214":[],"112319":[],"11231934":[],"112383":[11,27],"1124":9,"1125":[],"11276576e":[],"11279984e":27,"11283168":[],"11297834":[],"1133":[],"11341291":26,"11352543":[],"1136":[],"11362930e":[],"11366595":[],"11388888888888889":1,"1139":[],"11390":9,"114":[3,4,14],"11402309":[],"11430166e":27,"114437":[],"114550":29,"11462415":5,"11470382":20,"11481199":[],"11482289e":6,"11499517":[],"115":[3,4,14],"11507992e":1,"11520281":[],"11547777218875695":[],"11547777218875696":[],"11547777218876518":[6,29],"11547777218940905":[],"11547777218940906":[],"11551515":20,"115822":6,"11582635":23,"11590":9,"11598862":[],"11598862273198":[],"116":[3,4,14],"11604844":[],"116048442683864":[],"11660":9,"11666666666666667":1,"11674401539815678":[],"117":[3,4,8,14],"117430":26,"11744554e":6,"11780":9,"11790868":[],"11793498e":[],"117986":[],"118":2,"1180677":[],"118068":[],"1182":29,"1183":[],"118318":[26,27],"1184":[],"11840":9,"1185":[],"11850274":[],"1186":[],"11868087":[],"118681":[],"11890":9,"119":[2,3,4,14],"11909685e":[],"119625":[],"11962537":[],"1199":[],"119936":2,"119999999999976":[],"11it":[],"12":[0,1,2,3,4,5,6,8,9,11,12,13,20,23,25,26,27,29],"120":[2,3,4,14],"1203":9,"1203284":8,"12044974":27,"120450":27,"1206":8,"12069773":[],"12086283":[],"121":[8,9,10],"12107137e":[],"12152272":[],"1215pm":[24,26],"12160173":[],"12167821":[],"12178678":[],"121787":[],"12182967":6,"12196674":[],"122":[2,8,9,10],"12211324e":[],"12222222222222222":1,"122282":26,"122439":[],"1224392":[],"12288563":[],"122886":[],"12289498e":[],"123":2,"12302855":[],"12318726e":6,"12330033":27,"12330280229368":[],"12333649":6,"12345260617257443":[],"123459876":[],"123711":6,"12380":9,"12387751":[],"12394241e":[],"124":[0,26],"12400":9,"12427889":[],"12434807":[],"12446443":20,"1245596497124075":[],"12455965":[],"125":[],"1250":[],"125000":[],"12539459":[],"12552073e":6,"12591227":27,"126":9,"1261":9,"12611267":20,"12618549":5,"12630042":[],"1265":9,"12669602e":27,"127":4,"1270":[],"127043":[],"1271":6,"1277":6,"127773":[],"12777777777777777":1,"127812":26,"12788968":[],"12790":9,"128":[3,4,13],"12803365e":[],"12804129e":27,"1280756":26,"128664":6,"12871842":27,"12885048":[],"12890007e":[],"12898627064868978":[],"129":2,"129099":[],"12909917":[],"12933164":[],"129332":[],"12934015":[],"12937662":[],"12945452":[],"1297":9,"1297300314822336":[],"1298":9,"129963":29,"12adb44b1c20":[],"12it":[],"12m":[],"12pm":[24,26],"13":[0,2,5,6,9,11,12,13,20,23,26,27,29],"130":9,"13003291":6,"13006658e":[],"13019959":[],"130200":[],"13030182":[],"13055555555555556":1,"130694":27,"13069442":27,"13083543199018746":[],"131":9,"13117061":[],"13120126":20,"13131825":[],"1314":[],"13151043e":27,"132":9,"13208672e":[],"13220608e":6,"13228567":[],"13229545716505003":[],"132360":[],"1323603":[],"1326":9,"13260331":[],"13261136":[],"1326197715":26,"13280":9,"13281267":[],"13288373":[],"133":7,"13314597157635316":26,"13320535":[],"13333333333333333":[],"13375152e":[],"13392097":[],"13393141":[],"1345":29,"1346":29,"135":9,"13519106":[],"13535942":6,"135365":[],"13536532":[],"136":[],"1360":[],"13609760e":[],"1361":[],"13618516e":[],"1362":[],"1363":[],"1364":[],"13646574":5,"13655438":[],"13661243e":6,"13679863":6,"13687085e":[],"1371":6,"13734823":[],"13740":9,"137400784702911":27,"137652":[11,27],"1378":[],"1382":[],"1383":[],"1384":[],"13847924":[],"1385":[],"1386":[],"13865173":5,"13876586436927824":[],"138775":[11,27],"1388888888888889":1,"13890":9,"1392559581788775e":[],"1392559584983597e":[],"1392559585048734e":6,"1392559591206444e":6,"13925595925919e":[],"13it":[],"14":[0,2,3,4,5,6,8,9,10,11,12,13,14,20,23,25,26,27],"140":[2,9],"14010994":26,"14021063":6,"1404":[],"14042769":[],"140428":[],"14045566":[],"140525":[],"14052537":[],"14061742":[],"14072037":[],"141":2,"14100":9,"1416398":6,"14174062":[],"14174745":6,"1418":9,"142":9,"14231548":[],"14248744":[],"14250":9,"14252712":[],"14266332e":[],"14277718e":27,"142857":[],"14298603":[],"143":[2,3,4,7,14],"1435666":[],"14360598":[],"1437":1,"144":[3,4,14],"14400":9,"1440501043841336":1,"14432948":[],"14440":9,"14451625":[],"1446729567":4,"14484695":[],"14491591":20,"144993":26,"145":[2,3,4,14],"14561980e":[],"1457774":[],"146":2,"14617985":[],"14623399":26,"14629156":29,"14648477":[],"146489":5,"14648903":5,"147":2,"14710":9,"1472032":[],"14722222222222223":1,"14724135e":[],"147400":[11,27],"14741468":[],"147420":[11,27],"14783702":[],"1479":9,"147934":6,"14793475":26,"148":2,"14812206":6,"14818":[],"14822021080294":[],"1484256":[],"148463":[],"14846308":[],"148564":[],"14859":6,"14868784":5,"148688":5,"148734":[],"148768":[],"148771":6,"148854":[],"148895":29,"149":[],"14900336":[],"149213":[],"14932651":[],"149452":[],"149655":[],"149864":[],"14988578":[],"149886":[],"14995486":[],"149955":[],"149991":[],"14g":[6,29],"14it":[],"15":[0,2,3,4,6,7,8,9,12,13,14,15,16,17,18,23,26,29],"150":[4,8,9],"15005476":5,"150184":[],"15060311":[],"15084741":[],"15089627":[],"150920":[],"15092012":[],"1509778":[],"15098090e":6,"151":[],"15130074e":6,"151348":[],"15172236":13,"151986":[],"1519922":23,"152":9,"15200":9,"15254784":20,"15258907":[],"152696":[],"15269628":[],"15273101":23,"1527777777777778":1,"153036":[26,27],"153106":[],"15313054":29,"15320771e":[],"15321987":[],"1533795":[],"153760":[],"15384615384615385":9,"15384615384616":[],"154":[],"15410688":[],"154107":[],"15442554":[],"15454301":[],"15456415e":[],"154720":[],"15483121":[],"15484848":26,"15485374225770068":[],"154911":[],"1549258803307933":26,"15499979":5,"155":9,"155000":5,"15503141":[],"15508406":[],"15545946e":27,"155491":[],"15550086":[],"155687":[],"155883":[],"156":[],"1562":[],"15626984":[],"156270":[],"15629539":[],"15680777":[],"156956":5,"15696119":26,"157":[],"15714797":[],"15717291":[],"15746031e":[],"1575":9,"15752533e":[],"158":9,"1582408":[],"158241":[],"1583767":[],"15843769515580663":[],"1586300629904382":[],"1587":9,"15891336":[],"15899253":[],"159":[],"1590":9,"15939618":20,"15990":9,"15g":[6,29],"15it":[],"15pm":26,"16":[1,2,3,4,5,6,8,9,10,14,23,26,27,28,29],"160":[],"16006961":[],"16026316":20,"1603":3,"1604":[],"16049171e":[],"1605":[],"160539":[],"16054190e":[],"1606":[],"16061448":[],"1607":[],"16105443":[],"16111111111111112":1,"1612":[],"1614891":[],"16167387":[],"16171326":26,"16211139":5,"16220":9,"162246":5,"16231451":4,"16232481":[],"1625":9,"1628":9,"162999":26,"163":[],"16304863":27,"163049":27,"1630775253":1,"16342407":5,"16343471":6,"16350368":[],"163504":[],"16356503":29,"16384":3,"16397496":[],"16499193":26,"16500":9,"1655226":[],"16575256":[],"166":9,"16660817":[],"166667":[],"167":[],"16740002":[],"16749353e":27,"16754123":26,"167787":5,"168":9,"168044":[],"16804444":[],"16805821e":6,"16807":[],"16827044":[],"16832385":[],"16840081":[],"16961682":[],"16981204":[],"16b8e3cda33a":[],"17":[1,2,3,4,5,6,8,9,14,18,23,26,27,29],"17001895":[],"170019":[],"17009925":26,"17012048":[],"17074761":26,"17084902":[],"1709":9,"171525":27,"1715252":27,"17174962e":1,"17222222222222222":1,"172405":[],"17240522":[],"1726":9,"17297886491529482":[],"17300":9,"1731":9,"17339342":[],"173439546289586":[],"17428293":[],"17446471":6,"1752":9,"175300":[26,27],"175652":11,"17565241":11,"17606773":[],"17641709":6,"17647619":6,"17676228":[],"176880142835407":[],"17709473":[],"17764878e":[],"17774184e":[],"17777777777777778":1,"17796101e":[],"17801022":5,"17826349":[],"17861098":6,"17897671":[],"17917768":5,"17928538":[],"17930649":[],"17938393":[],"17942122":[],"17949575":5,"17953942":11,"1797":[1,3],"17983246":23,"18":[2,3,4,6,7,8,9,10,13,14,23,29],"180":[],"18029127":5,"18036103":[],"1807":4,"1809":9,"181":9,"1810":[],"1812":9,"18156717":[],"18166474":[],"1821":9,"18217743e":[],"18221254":[],"18264218":[],"18266178":[],"182662":[],"18276924":[],"18306165":[],"183062":[],"18327677":[],"18333333333333332":1,"1836":29,"18383522":27,"18393678":[],"184":9,"18409479":23,"184094793957168":23,"184519":[26,27],"18460525e":27,"184895":[],"185":[],"1851":[],"1852":[],"18526":[],"185278747229417":[],"1853":[],"1854":[],"1855":[],"185584177293881":11,"18573652":20,"186":[],"1860":9,"18603611e":[],"1861":[],"18611111111111112":1,"18613217e":6,"18632646":20,"18655084":5,"186551":5,"18657863":[],"18660":9,"18670072e":27,"18673098":11,"18682538":[],"18695705":[],"187268182169147":[],"18780801":26,"18790439176058887":[],"188":[],"18826299":[],"188263":[],"188499":[],"18849928":[],"1887":6,"18906854":[],"18912963":[],"18931988":[],"189367":26,"189377":[],"1893772":[],"189496":[26,27],"189622":[26,27],"1897755":20,"18998208":[],"18it":[],"19":[2,3,4,6,9,13,14,23,29],"19003":6,"19023414e":[],"19032152e":[],"19096968":[],"190970":[],"19105386":[],"19121744":20,"191262820314401":[],"19144544":[],"19158446":[],"19166666666666668":1,"191963":26,"19207979":5,"19213479":[],"19220":9,"19237495":[],"19264104e":[],"19284552e":[],"193":[],"19322111":26,"19354258":[],"19379506":[],"19394283":[],"194":[],"1940":[0,27],"19404282648955e":[],"194042826653172e":[],"194042826815498e":[],"1940428268204826e":6,"194042827197027e":6,"19425617":[],"1943":12,"194325":[],"19432526":[],"1944":[],"19444444444444445":[],"19466812":[],"19468989":20,"1950915150":[],"19528375":[],"1954":[],"19542686939553625":[],"1956":9,"19569961":[6,27],"1961":[],"1962":[],"19627430151896047":[],"1963":[],"1964":[],"1965":[],"19650622e":[],"19677483e":[],"19696497e":27,"1970":[20,26],"197104":[],"19710439":[],"1973":9,"197370":[11,27],"19740":9,"197738983259782":[],"1979":[6,29],"19800":9,"19853775e":[],"19866124":[],"1989":[],"19890453":[],"199":[],"19910037e":[],"19937":[],"19983530":6,"1999":29,"19994371":[],"1999902359008225":23,"1_1":12,"1_2":12,"1_3":12,"1cm":[0,8,10,23,26],"1d":[1,2,3],"1e":[1,2,4,13,14],"1e10":14,"1e4":6,"1f":1,"1k":20,"1n":[0,26],"1s":[],"1x":[0,26],"2":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,22,23,25,28,29],"20":[0,1,2,3,4,6,7,8,9,13,14,20,23,24,26,27,28,29],"200":[0,2,3,4,8,9,10],"2000":[0,27],"200000":[],"20015436":6,"20038136":[],"2004":13,"2006":25,"2008":26,"20098424e":[],"2010":1,"2011":1,"2014":4,"2015":1,"2016":[0,26],"20174121":[],"2018":[0,3,4,6,14,27,29],"2018906331":[],"2019":[8,9],"2020":[3,4,14],"2021":[6,14,27,28],"2022":26,"2023":[15,16,17,18,26,28,29],"20255857":[],"2027":9,"20277777777777778":1,"20299677":[],"203":[],"20313883":[],"20316225":[],"20320502":[],"2034747":23,"20355156":29,"20375361":[],"203753611274545":[],"20436207e":[],"20461136":[],"20500":9,"20509391":[],"20517254e":[],"205231":26,"20561803":[],"206":[],"2060":9,"20609082":[],"206091":[],"20661216":[],"206640":[],"20664007":[],"20669787e":[],"2069":9,"20708879":[],"207089":[],"207545":[11,27],"20756638":[],"207888":26,"20793824":[],"20833333333333334":1,"2086376":[],"20867052175003364":[6,29],"20867052175006306":[],"20867052175109335":[],"20893188":[],"20893514":[],"20916295":27,"20965182":[],"209789":26,"20980":9,"20999563":[],"21":[0,1,2,3,4,5,6,7,9,12,13,14,15,20,26,27,29],"210151":[],"2101511":[],"210340":[11,27],"21037404":26,"21058097":5,"21089449":20,"21110005":29,"21130":9,"2116753732":4,"21169159e":6,"21171751e":[],"21186069":26,"21265216":[],"21266166e":[],"21296315":[],"213":[],"213103":[11,27],"21328866":[],"21350883":[],"213509":[],"21369083e":[],"213743":[11,27],"214241":11,"21424133":11,"21435514e":[],"21467644":[],"21472683":[],"2147483647":[],"21493572":[],"21503438e":[],"21519063":[],"21561586":20,"21576737e":[],"21582995e":[],"21596432":6,"21601417":26,"216290":[11,27],"21639245":13,"21654926":[],"216683":[11,27],"21692581":23,"21786964":[],"2188":[],"21909039":[],"21913656":[],"21919813":[],"21922061":[],"219222942590059":[],"219249":[],"21924917":[],"21927988":[],"21944444444444444":[],"2195":[],"21985165":[],"21997099":[],"22":[0,1,2,3,4,5,6,9,12,13,14,20,26,27,29],"22014758":[],"220148":[],"22016672":[],"22044605e":[5,27],"220506557673408":[],"22090974":[],"22094791":[],"220968":[],"221":8,"22105472e":27,"221180":[26,27],"22134069":[],"22138768e":27,"2216":9,"2218":9,"221805":2,"221921":5,"2223808":[],"222400":[26,27],"2225":[],"2232822462117919":[],"22335823":[],"22403397554517984":[],"22442417":[],"2246674023625205":[],"22467274":29,"225":4,"22519792e":27,"22524236":26,"22566121":[],"22567203":[],"2259440937":[],"226124657522696":[],"22612466":[],"226856":[],"22685646":[],"22690428":5,"22722021":[],"22772039":[],"22821344":[],"22828694":[],"2284246870217162":[6,29],"2284246870217459":[],"22842468702288576":[],"22842468702288582":[],"22847924":5,"22887861":[],"22893751":[],"22929905e":[],"2299":[],"22it":[],"23":[1,2,3,4,6,7,9,12,13,14,20,26,29],"23002365e":6,"23014274e":[],"23056012":[],"23076923076923078":9,"23095558e":[],"231":29,"23103285":[],"23106527":26,"231224729143838":[],"2315033":[],"23167717":5,"23168292":[],"231683":[],"232":29,"2321528":[],"232153":[],"232435":[26,27],"23297056":[],"23309661":26,"23319332":20,"23333333333333334":1,"23333773e":27,"233528":[],"23365145e":[],"2338675":27,"233868":27,"23399553e":27,"234":6,"23405814":20,"23408962e":5,"23470374":[],"234723":[],"23472346":[],"23477043":[],"23483916":[],"23496851":[],"234969":[],"23522201":[],"23528337":29,"2357089093":[],"23593659":23,"2360682191515046":[],"2361111111111111":[],"2364":9,"23643365":29,"23659936":[],"2369833":26,"237038":[],"23703839":[],"23732849":[],"2379":6,"23792491":[],"237925":[],"238":29,"23816792":[],"238168":[],"238574":[],"23857423":[],"23875722":[],"23962594":[],"2397":9,"23971032":27,"24":[0,1,2,3,4,6,9,13,14,20,23,26,29],"240670854503034":[],"24085321":29,"241252":[],"24125217":[],"24140":9,"24159785":27,"2416":9,"24175744e":6,"24181044e":[],"2419":9,"24276315":[],"24279008":[],"2430":9,"24382528e":27,"24390":9,"24444444444444444":1,"24479698":26,"244858":[],"24485843":[],"24501973e":[],"24512498":[],"24533566":[],"24566623":20,"246":2,"2460037":[],"24611674":[],"24629107e":[],"24672263":23,"24688732":[],"2470":[],"24712854":[],"2476536":[],"247654":[],"24770094":[],"24782786":[],"24784064":[],"24801014":[],"24829908":5,"24844778e":[],"24848907":[],"24906604e":6,"24908246":[],"24924624":[],"24929083":[],"249302":[],"2493023":[],"2495":[],"25":[2,3,4,5,6,7,8,9,11,13,14,15,21,26,27,29],"250":[2,4,7,9],"2500":[],"25000":[0,27],"250000":[],"25006757":[],"250154":[],"25028968":[],"25033968":26,"250636":26,"25070907":[],"25115233":[],"25116851":[],"25137033":26,"2517560119":[],"25186426":26,"251879":[26,27],"25201286e":[],"25216021":[],"252436":[26,27],"25253892":[],"25259666":[],"2526":[],"252866":[],"25286618":[],"25328924":[],"25366775":[],"253668":[],"253775":[],"25385387":[],"253854":[],"2544356":11,"254436":11,"25455904":[],"25485457":[],"255":3,"255001":[26,27],"25522012":[],"25580178e":27,"256":[2,4],"25617654e":6,"25617657e":[],"25617658e":6,"256962":[],"257":[],"2572":9,"25724702e":[],"2572495066":[],"2572e3a4b38d":[],"2575":9,"25781559":[],"257816":[],"25789255":[],"25792767":[],"25845e8df859":[],"25881631":[],"25902112":[],"259153":[11,27],"2597":9,"25976336":[],"26":[2,3,4,6,9,13,14,23,29],"26015755":[],"26022420e":27,"260225":[],"26022531":[],"26079358":[],"26115367":[],"261154":[],"26153846153846":[],"26181254":[],"261813":[],"26185107":[],"26186844":[],"2619":27,"26193455e":[],"26223899":[],"26265627":[],"26290036":[],"26291585":[],"262916":[],"26292364":29,"26294938":[],"2629493813057833":[],"26297455":27,"26301436":5,"26306244":[],"26350283":23,"264":[],"26409315307910025":6,"2640931530791003":[],"26409315307910036":6,"2640931530791005":[],"26409315307910053":[],"264421":26,"264540221699101":[],"264666":[],"26466619":[],"265":[],"2650":9,"26506981e":[],"265109911":4,"26512016":0,"26521761e":[],"26531223e":[],"2654":9,"26549135":[],"266":[],"26610075":[],"26624267e":27,"26662339374864535":[],"26666667":13,"26678445":[],"26710969":5,"26780278":5,"268":9,"26804287":[],"268043":[],"268227":[],"26822717":[],"268484":[],"26848435":[],"26896710e":[],"269":[],"26995402":27,"27":[0,1,2,3,4,6,13,14,27,29],"270":[],"27011455":[],"2703":[],"27050214":[],"27090196":[],"27091656":[],"270917":[],"27092897":[],"27092910":6,"271082":5,"27108228":5,"273817":[],"27381733":[],"27404277":[],"27412395":[],"2745333":[],"27498198":[],"2750":9,"27524966e":[],"27525917":[],"275341":[],"27547877":[],"27552531e":[],"27562809e":[],"27580794":[],"27621662e":[],"27625974":[],"276263":[11,27],"27660123":[],"27674224":20,"27692307692308":[],"27700":9,"2772":[],"2775623201":[],"27760":9,"27801122":[],"278036":[],"27803645":[],"27809912":[],"27865175":23,"27898999":20,"27924636":5,"27932015e":[],"27961873e":[],"27n_":23,"28":[1,2,3,4,6,9,13,14,16,27,29],"28001319":[],"280179":26,"28047021":[],"280573":5,"280647":[11,27],"28065343":[],"28067036":[],"28097861":[],"280979":[],"28102554":[],"2812":[],"28186376":[],"281930":26,"28205578e":27,"28213741":[],"28219089":[],"282191":[],"282259":26,"28230186":26,"28266792e":[],"282727":[11,27],"28284417e":[],"283":[],"2830637392":4,"2831603":[],"28336218e":6,"2836":[],"28390":9,"28391978":27,"28443039":29,"28457442":[],"28475098":8,"28500583e":[],"28506507":[],"28587856":20,"28590743":[],"2861":23,"28634473":[],"286345":[],"2871":9,"2873":9,"28744729":[],"2882":23,"28836053":[],"288455813429108":[],"2886":23,"2886847885377843":[],"28868479":[],"28875373":[],"2890":[0,26],"2891298":[],"2892":23,"28948521":[],"2895238":[],"28967287":[],"28it":[],"29":[3,4,6,7,9,14,29],"29082851":[],"290829":[],"29083183":[],"290832":[],"29115152e":[],"29148545e":[],"29149329":[],"2915":23,"29167186":5,"29229741":[],"2923076923077":[],"29238177":26,"29276615":[],"29294618e":[],"2931":26,"294277":[],"2942772":[],"29512284":[],"295123":[],"2954":[],"29546690e":[],"2955":[],"2956":[],"2957":[],"2958":[],"29588901":[],"29592539":[],"296247":[],"296414":26,"29650918":[],"29661191":[],"29662593e":[],"29672381":[],"2968":26,"297":[],"2971492148":[],"2972":[],"297248":[],"29724806":[],"2980":26,"29822833":6,"298273":[],"29830651":13,"298375":[],"298836":[],"29883607":[],"299":[3,4,14],"2990":26,"299267190588216":[],"29938171e":[],"29972182":[],"299748":[26,27],"29it":[],"2_":12,"2_1":12,"2_2":12,"2_3":12,"2_i":12,"2_m":[6,23,29],"2_t":13,"2_x":23,"2b":23,"2cm":8,"2d":[1,3,11,12,19,26],"2e":[6,29],"2f":[0,7,9,10,11,12,26,27],"2ff97f4bf03b":[],"2g":2,"2g_i":2,"2k":3,"2m":[6,29],"2n":[0,2,3,26,27],"2nd":9,"2p":23,"2pm":[24,26],"2pt":4,"2s":[],"2x":[0,3,8,13,26],"2x_ix_jy_iy_j":8,"2x_j":8,"2y_i":10,"2y_j":8,"3":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,19,20,21,22,23,24,28,29],"30":[0,1,3,4,6,7,9,10,13,14,20,23,24,26,27,29],"300":[3,4,14],"30000":[0,26],"30006341e":27,"30010":9,"30035718":[],"3005849":[],"301":[3,4,14],"30119421":8,"301475":[],"3014751":[],"3017":[],"3018":[],"3019":[],"301927":[],"302":29,"3020":[],"3021":[],"30232456e":[],"30258509":[20,26],"302m":[],"303":6,"30313128":[],"30337713":[],"3037":[],"3038":[],"30384239":[],"3039":[],"303m":[],"3040":[],"3041":[],"30442964":27,"30466214e":6,"304m":[],"305":[0,26],"30535506":[],"30552077":[],"30559547":[],"30577095":26,"305m":[],"306":[0,3,4,14,26],"30626515e":27,"3063":[],"30638512e":[],"306m":[],"307":[0,3,4,14,26],"30700873e":[],"30702075":[],"3072":3,"3073":29,"30761405e":[],"3076923076923077":9,"30787294":6,"3079975":[],"307m":[],"308":[0,3,4,14,26],"30813073":[],"30839676":[],"30844205":[],"308763":[],"30891693":[],"308917":[],"308m":[],"309":[0,3,4,14,26],"30936179":[],"309362":[],"30940":9,"30984244e":[],"30993886e":[],"30b53504a633":[],"30it":[],"31":[3,4,6,12,14,20,23],"310":[0,3,4,14,26,28],"310579":27,"3105791":27,"311":[0,26,28],"31120247":[],"31138734e":[],"31193395e":[],"311m":[],"312":[],"3120271598582915":[],"31211671":[],"31228042":[],"31229747":[],"3123314713548606":[6,29],"31276579e":6,"312m":[],"313":28,"31311243":[],"31318084":5,"31331481":[],"313315":[],"31347523":[],"31377492":[],"314":28,"3141149":[],"31424359":[],"314244":[],"31447174":[],"31457796":5,"315":[6,28],"31507591":[],"31519573":[],"31531613":[],"3155":[0,5,6,28,29],"315746":[],"31574634":[],"315977":[],"31597731":[],"316":[],"31605061":[],"31614188e":[],"31644071":[],"31650694":6,"31658671e":[],"31681097":[],"316811":[],"317":[],"31718909":11,"31719317":[],"317367":11,"31768693":[],"31782593":[],"317825933186352":[],"3178411477108273":[],"317m":[],"31816439":26,"31835835":[],"31866499":[],"3189":29,"31896852":8,"319803":[],"31980301":[],"31988705":[],"319m":[],"31it":[],"32":[3,4,6,12,13,14,20,23],"3200":1,"32039227e":[],"32047562":[],"32066545e":5,"32068012":[],"3208":2,"320m":[],"3211480515121596":23,"32149601703519115":[6,29],"3214960170351912":[6,29],"3215":9,"32158804":[],"32179365":[],"32183101":[],"32183405":[],"32190363":[],"32305813":[],"32309075":[],"32320052":[],"32360769":[],"323608":[],"324":2,"32450054":27,"32458459":[],"32459186":[],"32474290e":[],"324m":[],"3250":[1,6],"32512":[],"32541271":[],"32542732":26,"32566092":[],"32609560e":[],"32611807":[],"326238":[26,27],"3263505":[],"3267061":[],"32695433e":27,"326m":[],"32714903e":5,"32721178":[],"327212":[],"32750531e":[],"32766274":[],"32784259e":[],"327883798133277":[],"32816737":[],"32903042":[],"32946226e":27,"329492":26,"32968167":23,"3297":[],"32992274":[],"329923":[],"32it":[],"33":[2,3,4,9,12,14,20,24],"330":[3,4,14],"3303":[],"33066907e":[5,27],"33078983":[],"33079801":[],"331":[3,4,14],"3310":[],"33113182":26,"3313155":[],"33135331":[],"33142087e":27,"33166055e":[],"33174939e":27,"3318361":[],"331939":[26,27],"33196524":13,"33199489":26,"331m":[],"332":[3,4,14],"332018":[],"33201844":[],"3322326":[],"332331":[],"333":[3,4,7,14],"3331":[],"33327369":[],"333274":[],"33333333":13,"3337":[],"33370315e":[],"334":[3,4,14],"335":[3,4,14],"33589475":[],"336":[3,4,14],"33611111111111114":[],"33645986e":[],"33646268e":[],"336801":[],"33680101":[],"33697405":[],"337":[3,4,14],"33711888":[],"33736485":[],"338":[3,4,14],"33819480e":[],"33823452e":[],"33829607":26,"3384":[],"33874172":20,"33900713":[],"33939645e":[],"33941551":[],"33956555":[],"33it":6,"33m":[],"34":[2,3,4,9,14,20],"340071371496255":27,"3403":9,"34037032e":[],"34040204e":[],"340583":26,"340782":[11,27],"34114547":5,"34158665":[],"341m":[],"34212496":[],"342680":[26,27],"34270689":23,"34305928":[],"3431023":26,"3435384":26,"3436":[0,26],"3437":[0,26],"34378387e":[],"34412923":27,"34442654e":27,"34498451":[],"34498455e":27,"3456":[],"34561469":[],"345687771875474":[],"34568872":[],"34569596":5,"3457":[],"34578026":[],"3458":[],"34585355":[],"3459":[],"34591885":[],"345919":[],"3460":[],"34626037":[],"34652523":[],"34678929":[],"346810":[],"34695383":[],"346954":[],"3469819128513336":[],"3473884":[],"34740615":[],"34852463e":[],"34914442e":[],"3498":[],"34989509":[],"34it":6,"35":[0,2,3,4,6,9,14,17,21,24,26],"35030572":[],"35074881":[],"350749":[],"35123041":[],"35128114":20,"3514":[],"35140":9,"35147135":[],"35147532":[],"351636":[11,27],"35182854":5,"35203688":[],"35207264":[],"352073":[],"3521478":26,"3522":[],"3525":[],"3528556":[],"352856":[],"3529":[],"3530606977":[],"35309734":[],"35319678":[],"35346808":[],"3538":[],"35396404e":[],"35401056e":[],"35412147":29,"3543":[],"35438626":[],"3544313922":6,"3546":[],"35470445e":[5,27],"35473061":[],"3548":[],"355":[3,4,14],"3551":[],"35533773":6,"35533774":[],"35559283":20,"3556":[],"35562617e":[],"35568525":[],"3558":[],"355906":[],"35590603":[],"356":[3,4,14],"3560":[],"356399":[26,27],"35665906e":27,"3567":[],"357":[3,4,14],"35700089":26,"35704388":[],"357044":[],"3572":[],"35724288e":[],"3575":[],"357508":[26,27],"35759569e":[],"3576":[],"3577":[],"35770742":[],"35771826":6,"35771842":[],"35776087e":[],"35793003441520066":[],"358":[3,4,14],"3581341341":4,"3581571":[],"35819312":[],"35821426":[],"35846425":29,"3585":[],"3587":[],"35892474":[],"35894575":[],"359":[3,4,5,14,28],"3593":[],"35947075":[],"3594821":[],"3595":[],"3598":[],"359999999999985":[],"36":[0,2,3,4,5,6,14,21,23],"360":1,"36028477e":27,"3604":[],"3605":[],"3606":[],"360688":[],"36074377":26,"3609":[],"36102113":[],"36117602":[],"3613":9,"3615":[],"361556":[26,27],"3617":[],"3621311":5,"36227629e":[],"3624":[],"3627":[],"3628":[],"363295916323784e":6,"363295916414895e":[],"3632959170548605e":[],"3632959215700067e":6,"363295924430451e":[],"36336962":20,"363834":[],"36383443":[],"363936":[],"36393643":[],"364244035362507":[],"36424404":[],"3643":[],"3644017":[],"364402":[],"364418e97433":[],"3645":[],"36452684":26,"3646":[],"36462687e":[],"3647":[],"365350":[],"36535019":[],"3655":[],"3655222":5,"36570403":[],"3659":[],"36593124":13,"3665263":[],"3668":[],"36689784":[],"366898":[],"3669":[],"36692356":[],"367":2,"3672":[],"3673":[],"3674":[],"3679":[],"36809474":[],"3684":[],"368537":[],"36853738":[],"3687":[],"3688":[],"368m":[],"36904006e":[],"369139":[11,27],"36it":[],"37":[3,4,6,9,14,21,26],"3702755":26,"3704":[],"37071826e":27,"370782966":4,"3711":[],"37114298":[],"37126183":23,"3713":[],"37157712":[],"3718":[],"3721":[],"3722":[],"37239927e":27,"3725":[],"3729":[],"37307168":[],"3739":[],"37396662":6,"37415316":[],"37419664":[],"374197":[],"374291":[],"37429133":[],"3744637":5,"374464":5,"3748":[],"3749":[],"3753":[],"375351":[],"37535105":[],"375694":26,"37585988":[],"3759":[],"3760":[],"37622356":26,"37624303":[],"3765":[],"376547":26,"376559":[],"37655936":[],"376834":[],"37683438":[],"37692363":[],"37703055":[],"37713991":[],"3772":[],"3773":[],"37732":[],"377372":27,"37737221":27,"3776":[],"3777801602":6,"3779":[],"37816901":[],"37835429e":[],"3784":[],"378664":[],"37866422":[],"37885034":[],"37900111":6,"3791":[],"379203":[],"3794":[],"379647":[],"37964744":[],"37992857":[],"37it":[],"38":[3,4,9,14,23],"380":9,"38019139":[],"3802":[],"3803":[],"3804":[],"3805":[],"380739":[],"38073947":[],"38098948":26,"3811":[],"38135733e":6,"3815":[],"381627865854956":[],"38162787":[],"38168549":[],"3817475779":[6,29],"3818":[],"3819":[],"3820":[],"382187":26,"38222896":[],"3823886":[],"382389":[],"38246359":[],"382672":[],"38267217":[],"3827":[],"382951":[],"38295101":[],"38299798":[],"3830":[],"38302314":[],"3833":[],"38336316":[],"3834":[],"3837":[],"3838":[],"383878":[],"3838783":[],"3839":[],"383942":[],"38394242":[],"3842":[],"38432672e":[],"38461538461538464":9,"38469441":[],"3850":[],"3851":[],"38533184":[],"38533185":6,"3854":[],"3858":[],"386":9,"38605872":[],"386059":[],"3861":[],"3862":[],"386294":[],"38629412":[],"38629436":[20,26],"38629844":[],"38636467":[],"38637915":[],"3864":[],"38644889":[],"3867":[],"3869":[],"387":28,"3871":[],"3872":[],"38730917e":[],"387482":[],"38748219":[],"3876":[],"38763091":[],"38788697":[],"388":[],"3881":[],"38816656":[],"388167":[],"3882":[],"3885":[],"3886":29,"3886211842840971":[],"38868469":[],"3888":[],"3889":[],"389":[],"38901478":[],"38906684":[],"389067":[],"3891":[],"38916861e":6,"38962192e":6,"3898":[],"39":[0,3,4,9,14,24,26],"390":[],"3906":[],"39074222e":[],"39095416":[],"39116042":[],"3914":[],"3915":[],"3916":[],"391602":[],"3917":[],"39197698":27,"391977":27,"39214397":27,"392144":27,"3922":[],"3925515884752442":[],"39272691":[],"3928":[],"3929303193461995":[],"3931":[],"39311435":[],"39340081":[],"39384255":[],"393843":[],"39417707e":[],"3943":[],"3944":[],"39444293":[],"39456996":[],"39493843":[],"3950":[],"39548035e":[],"3955":[],"39550083":[],"395501":[],"39560937":[],"3957482":[],"39579407":5,"3958":[],"39595799":[],"3960":[],"39600607":[],"39614347e":[],"3962":[],"39626502":[],"3970":[],"39706038":5,"39707147":[],"39724390e":[],"39730396":21,"3975":[],"39769983":26,"397700":[11,27],"39789527":[20,26],"3979":[],"39794864e":[],"3980313467":6,"3983":[],"39864527":[],"39892216e":[],"39920034":[],"39924562":[],"399246":[],"39931051e":27,"3994":[],"39943328":[],"3996":[],"39969826":23,"399836":[26,27],"39it":[],"3bdkrb":[],"3d":[2,3,4,6,13,21,29],"3f":[1,3,9],"3n":20,"3x":[2,8],"3x_i":2,"3y":8,"3yk470mj5p931p9dtkk0y6jw0000gn":[1,6,13,21,26,29],"4":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,23,26,28,29],"40":[1,3,4,6,9,14,21,24,26,29],"400":4,"4000":[9,26],"40026896":13,"4005":[],"40062254e":[],"4011":[],"40116777":[],"401168":[],"4012":[],"4017":[],"4017308":[],"40183706":[],"401842":[11,27],"40214433":[],"4022":[],"40236659e":[],"40236901e":[],"40264255":[],"4027":[],"4030":[],"40340043":[],"4034668048":[],"40355028":23,"4038":[],"40389922":[],"4039":[],"404":27,"4045":[],"40474117e":27,"4048":[],"4050":[25,26],"40506137":[],"4055":[],"405890":[11,27],"4059":[],"40599799":[],"40620066":27,"406201":27,"40672425":[],"4068":[],"40684484e":27,"4069":[],"4075":[],"40754621":[],"40768645e":[],"4077":[],"40789038":23,"408":2,"40805354":26,"4082":6,"40828861e":[],"40829683":[],"408297":[],"40837841":26,"4084":[],"4087":9,"4087793":5,"4088":[],"4089":[],"40893740e":27,"409":2,"40927184e":6,"40947888":20,"40961677":[],"40980089":[],"409801":[],"40984037":20,"40a38ad763f1":[],"41":[0,2,3,4,9,14,20],"410":2,"4100":[],"41050604":20,"4107":9,"410815":[],"41081513":[],"41083418e":[],"41091395e":[],"411":2,"4115":[],"41165766":13,"4117":[],"41170104":[],"412":2,"41211579":[],"412116":[],"4128":[],"4131":[],"41335011":[],"41340248":[],"41351287":[],"413513":[],"41357064":[],"41369795e":[],"41386599":[],"4143":[],"41433969":5,"4144":[],"41446721":[],"4146":[],"41486312e":[],"41511965e":1,"415201":26,"41542567":[],"4155":[2,15],"415534701258823":[],"41566661":[],"4162706317":6,"41646135":[],"4166666666666667":9,"4167":[],"41718305":20,"4176":[],"4177":[],"41816468e":[],"418506":[11,27],"4185877":26,"4186":[],"41876267":[],"418763":[],"41882036e":[],"41882037e":6,"41923956":[],"4192423635":[],"41958102e":[],"4198":[],"41990268":[],"41it":[],"42":[1,3,4,8,9,10,14,20],"420":2,"42028578":[],"42037468e":[],"420543":[],"42054337":[],"42078103":[],"42090072":26,"42092776e":27,"421":2,"4212":[],"42130182":[],"42190899e":[],"422":2,"422275":[],"4224":[],"422480":[],"4224801":[],"4229538":11,"422954":11,"423":2,"4230769230769231":9,"42340403e":[],"42352791":[],"42394972":[],"424":[],"42441033":5,"42449643":[],"42450":9,"42451192":[],"425":[],"4255":[],"4258989918":[],"426":[6,7],"42615374e":[],"42619859":[],"42631342":[],"42694395":[],"427017":[],"42729886e":[],"42732954":[],"42745728579187":26,"4275":[],"42790075e":[],"42794448":[],"4284066":[],"42882114e":[],"429":29,"42902155":[],"42915006":[],"42976761":[],"43":[0,1,2,3,4,7,9,14,20],"43001767":[],"43007002e":[],"4300745":[],"43012535e":[],"43036842":[],"4304167":[],"43054282":5,"4310":26,"43113284e":[],"43135183":[],"43185254":[],"43294696e":[],"433":29,"43330971e":6,"43333886":[],"43379387":23,"43459933e":27,"43471448":20,"43482628":[],"43490889":23,"434932":[],"43493232":[],"435163":[26,27],"43525003e":27,"43533824":20,"43552433":[],"435567":[],"43556723":[],"43560678":[],"43570786e":27,"43579948e":6,"43639284e":27,"436462435":4,"43656628":[],"43693489":[],"43761347":[],"43766686":11,"437731":11,"43773101":11,"4378299":20,"4379":[],"43794243e":[],"43795846":[],"438060758":6,"438111004052204":[],"438136":[26,27],"43876695":[],"438767":[],"439230":6,"43975616":26,"4399427":[],"439943":[],"44":[0,1,2,3,4,14,20],"44021223":[],"44065609":[],"44089210e":[5,27],"4410":[],"4411":[],"441264":26,"44152248":[],"44190304":[],"44198566":[],"4421034379901165":[],"44210344":[],"44214614":26,"442600":[11,27],"442701":[],"44270138":[],"443217":[26,27],"443538568902246":[],"44353857":[],"444":9,"44463741":[],"446033":29,"44624525e":27,"446453":26,"44651551":[],"44688507e":[],"44792888":20,"44830642":[],"44838453e":[],"44879019":23,"44894073":[],"44906236e":[],"44970586e":1,"44993812":26,"44997121":[],"44it":[],"45":[2,3,4,9,14,24,26],"450":9,"45020332":23,"450257":[11,27],"4504":9,"45112054":[],"45112450e":[],"45143018":[],"45146007e":[],"45220770e":[],"45241645e":[],"4524458":23,"452553":26,"45268835e":[],"45290829":[],"4533029":5,"453303":5,"45359479":26,"45372697":[],"453727":[],"45393214e":[],"454027":[],"45402701":[],"4543859":[],"45464765":[],"45494697":20,"455173":29,"45571233e":[],"4557763":11,"455947":[26,27],"456":9,"456418966187335":[],"45643708e":[],"457":2,"45720968e":[],"45739087":[],"45741697":[],"457770268480242":[],"45797073":[],"458027":[26,27],"458078":[11,27],"45840226e":[],"45865699e":[],"45915671e":27,"45917334":20,"45951629e":[],"45960079":5,"45979894":[],"45985488":[],"459855":[],"46":[2,3,4,9,14,24,26],"4601":9,"46023985":[],"461":[],"461175":[],"46153846153846156":9,"46164895":26,"46184816e":27,"462":7,"46284227":[],"46312791":[],"46369869":[],"46383925e":6,"463861":26,"46400080e":[],"464424":[],"4644244":[],"46508305":[],"46543658e":27,"46567887":[],"46573851":[],"46575843e":[],"466":[],"46602982":[],"46650488":[],"466505":[],"46696223":[],"4676059":27,"467606":27,"46794929":[],"469":[],"46932688":[],"4694":[],"46984697e":6,"469868":[],"46986815":[],"47":[2,3,4,9,14,24,26],"47008799e":[],"4703":[],"47042744":5,"47057708":23,"470714":[26,27],"47075725":6,"47082726e":[],"47099765e":[],"47116868e":6,"47125748":5,"47132891":5,"47158029e":[],"47202234":20,"47202442":29,"47245463":[],"472455":[],"47248297":[],"47313680":29,"47355725e":[],"47356208":26,"47387858":[],"47400238":[],"47432993":[],"4744":[],"4744219":[],"47478057":[],"47482507":[],"47485224":[],"47513129":20,"475311":[],"47531107":[],"4756968":20,"47610036":6,"47649841":[],"47726407e":[],"47732343e":[],"47756853":20,"47815203":11,"47920156":[],"47942814":[],"479465113":4,"47977394e":[],"48":[3,4,9,14,29],"480":26,"48005485":11,"480055":11,"48089797":[],"48133064":[],"48179077":23,"481979":6,"48212873":[],"48217977":[],"48241947":[],"48257387":[24,26],"48259891e":[],"48316523e":[],"483257001":13,"4837":[],"48420165":[],"48442336":[],"48442771":[],"48476997":11,"48574149":[],"486873":[],"48687342":[],"487000566129411":23,"48739546":[],"48783291":11,"487833":11,"48783370e":[],"48802709":26,"48871288":[],"488713":[],"48877744e":[],"489":[],"489522":[],"48952201":[],"48971452":[],"48974963":[],"48994188":5,"48it":[],"49":[3,4,5,6,9,11,14,21,27],"490":[],"49001938e":[],"49052317e":[],"491":[],"49152":3,"49197115":20,"492":[],"49242688":[],"49282737":[],"493":[],"49352496":[],"493525":[],"49372893":[],"49385454e":[],"4940954":[0,26],"49468927e":[],"49545491":[],"495708":[],"49570819":[],"4959161509356135e":[],"495916150936645e":6,"495916150936654e":[],"4959161509377256e":6,"495916150938325e":[],"49614357":[],"496337":[],"49633743":[],"49636583":[],"49666527e":[],"49672291":[],"49680475019226":[],"497":[],"49730046e":[],"498":[],"49838167e":[],"49862065e":[],"49887949e":[],"499":[],"4990":23,"499012":[],"49901244":[],"4992":23,"49932184":26,"49932427e":5,"4997":23,"49974854":[],"49992743e":[],"4b5d3mupqma":[],"4c4c7f":[9,10],"4d":3,"4f":6,"4pm":[24,26],"4y":8,"4y_i":10,"5":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,20,21,23,26,27,28,29],"50":[1,2,3,4,6,7,8,9,10,13,14,26,27,29],"500":[1,3,4,6,9,10,13,29],"500000":[],"50000455":5,"50000553":5,"50000718":5,"50000855":5,"50000969":5,"50001063":5,"50001142":5,"50001207":5,"50001261":5,"50001306":5,"50001343":5,"50001374":5,"500014":5,"50001414":5,"50001422":5,"50001439":5,"50001454":5,"50001466":5,"50001476":5,"50001485":5,"50001492":5,"50001498":5,"50001502":5,"50001506":5,"5000151":5,"50001512":5,"50001515":5,"50001517":5,"50001518":5,"50001519":5,"50001521":5,"50001522":5,"50001523":5,"50001524":5,"50001525":5,"50032818e":27,"501":[],"501049":[],"50104946":[],"50105159":[],"50173752e":[],"5018":23,"50219982":[],"50227564e":6,"5031474174499113":[],"50314742":[],"50321091":5,"504167":[],"50416731":[],"50427787":29,"50453066":26,"50456256":5,"504563":5,"504881":[],"50488131":[],"50504743":[],"50584526":[],"506":[0,27],"506553":27,"50655336":27,"50680321e":[],"50727059":[],"50733669e":[],"50754416":[],"50769230769231":[],"50799086":[],"507d50":[9,10],"50812169":[],"50835355":[],"508354":[],"50846111e":[],"50846112e":6,"50894343":[],"50931378":[],"50it":[],"50j":13,"50x10":1,"51":[3,4,10,14],"510":1,"51000420e":27,"51004249":[],"51013506":26,"51054961e":[],"51075117":[],"51108791":[],"51126895e":[],"51131471":[],"511315":[],"51146442":[],"51150176":[],"51174395":[],"51178587":23,"511888":5,"51190686e":27,"51191552":6,"511977":[],"51197747":[],"512132":[],"51214899":[],"51220693":26,"51265232":[],"51289697":[],"512897":[],"51374050":29,"5138":[],"51389553":[],"51400374e":[],"514219":[26,27],"5146617":26,"51507839":[],"51508374":[],"51561271":[],"51578522e":[],"51656406":[],"51664729":[],"51719273":5,"517193":5,"51732028":[],"517350858882083":[],"5174581":[],"51749473":[],"5177783846":4,"517823":[],"51782322":[],"517931":5,"51793145":5,"518030":[],"51803019":[],"51850390e":[],"5188328":[],"518833":[],"518895":26,"51900727e":[],"51905011":[],"51933924":[],"519750":5,"51975002":5,"519842":[],"52":[3,4,14],"52033394e":27,"520931":[],"52093124":[],"52113415":[],"5216048821598704":[],"521719":[],"52171908":[],"52191849":[],"521986":[],"5222222222222223":1,"522758":[],"522836":26,"52307692307693":[],"52362157e":27,"52428467":[],"52512084":[],"52580392":[],"525804":[],"5260627":[],"52606351e":[],"52606392":11,"526744":[11,27],"52678301":[],"52713812":[],"52723079":[],"52744746":[],"52773051":[],"52773891":[],"52775466":29,"52799362":[],"528115":[],"52811546":[],"52812155":23,"52819229":[],"52839842":[],"52872817":20,"52874252":5,"529":[],"52902847e":[],"52911941":[],"52945186e":27,"52945798e":[],"53":[3,4,9,14],"5303329":11,"5305555555555556":1,"5312":[],"531280":[26,27],"5316382":26,"53189647":[],"53219937":[],"53254035":[],"532643":[],"53264317":[],"53312754":[],"533142":20,"5333734":[],"533941":[],"53394148":[],"53423784":[],"5342745":26,"534362":26,"53462883":26,"53476918":20,"53479276":[],"534793":[],"53502096e":[],"53603432":[],"53611562":[],"53617977":11,"536180":11,"53623614":[],"53697476":[],"53703498":6,"53738247":29,"53752634":[],"5378811":11,"53794784":[],"537948":[],"53815559":[],"53822787":[],"5384615384615384":9,"53846153846155":[],"53861506e":27,"539261":[11,27],"5393681":20,"53938383":[],"539384":[],"53952479":[],"539525":[],"53952664e":[],"539683":[],"53968309":[],"53971260e":[],"53it":[],"54":[3,4,6,9,14,23],"540":9,"54016188":[],"540162":[],"54039921":5,"54041041e":[],"54041662":[],"54078623":[],"541605":[26,27],"54163136":[],"54219245":[],"5424657":13,"54302039":[],"543939":27,"54393936":27,"544439":[26,27],"54445121":[],"54526769":[],"54537329":[],"546166676":[],"54632096":26,"54637219":29,"54644646e":[],"546712":[],"54671213":[],"5472246386316972":[],"54722464":[],"54780216":[],"548110376991839":[],"54845458":[],"548455":[],"54861905":20,"54886137":[],"54894095":[],"54956992":[],"54957609e":[],"54969168":[],"54it":6,"55":[1,3,4,9,14],"5501":[3,4,14],"55026099":[],"550321":[],"55051175":[],"55070893e":[],"55141095":[],"55202922":[],"55220861":[],"55229041":23,"552731":[],"55273102":[],"5538822":20,"5539":[],"55505907":[],"555187":[],"55518724":[],"555496":[],"5554964":[],"55553537":[],"5555555555555556":1,"55571665":[],"55578041":[],"55626454":[],"55640013":[],"55640387e":[],"5566":[],"55687658":13,"55765644e":[],"557795":[11,27],"55805262":26,"558080":[],"55808001":[],"558241":[],"55824107":[],"55854694":11,"55907387e":[],"55914617e":[],"55936475e":[],"5594":6,"55965719":[],"56":[1,9],"56033697":5,"56071752e":[],"561":[],"5611":[],"5615":[],"56183518":[],"56198284":5,"5625":[],"56302854":[],"563167":26,"56343926":[],"56383695":23,"5639":[],"564":9,"564374":[11,27],"56437897":[],"564379":[],"56447742e":[],"56481272":[],"565":9,"565006":[],"56500639":[],"56536":[0,26],"56542321e":[],"56556315":[],"565651":[],"56565106":[],"56589683":29,"566":9,"566074":[],"56607416":[],"56636616e":6,"567":9,"568":[9,29],"56807589e":[],"56821272":20,"56848886":[],"569":[1,9],"56912044e":6,"56939714":5,"56953559":26,"56965674":29,"56992937":[],"57":[0,8,9,24,26],"570":9,"5700":[],"57031517e":27,"57081718e":27,"571":[5,28],"571092":[],"57109235":[],"571105947979336e":[],"571105947979352e":6,"571105947979394e":6,"571105947979395e":[],"571105947979439e":[],"57113133":23,"57154252":[],"57174058":[],"57201944e":6,"572069":26,"57266138":[],"57286194":[],"57350724":[],"57361898":[],"57403561":[],"57445493":[],"574465":[11,27],"57488231":23,"57505712":[],"576":29,"576431":[],"57643113":[],"57673618":[],"5769230769230769":9,"57776818e":[],"57807896e":[],"57810065":[],"57810898e":[],"5783575":[],"57842073e":[],"57867898":[],"578889":[],"57888946":[],"57947706":[],"57it":[],"58":[2,9,10,24,26],"58039072":[],"58042237":[],"58084359":[],"58104393":[],"581044":[],"58112299":[],"581123":[],"58171189":[],"58187211":23,"58187347":[],"58312967":[],"5833333333333334":9,"583595":[26,27],"58395707":[],"58428804":[],"58474054":[],"584804":[],"58492636e":27,"58574143":[],"58590795":[],"58666142":[],"58671946":[],"58738148":[],"5888888888888889":1,"589":[],"58908017":26,"58912508e":[],"58986647":[],"59":[2,9],"590":[],"59095724":20,"591":[],"59118929":[],"591317992":4,"591594":[],"59159438":[],"59164202":26,"5916751":20,"59182949":[],"59190877":[],"592":[],"59203799":20,"59277468":[],"593":[],"593040":[],"59304008":[],"59323905":[],"59366475":20,"59367954":[],"59428035e":27,"5944444444444444":1,"59445979":[],"594460":[],"59468796":[],"59567145e":[],"59591979":[],"595920":[],"59592669":[],"59609892":[],"59616454":[],"596165":[],"59648068e":27,"59654302":[],"597084":11,"5970842":11,"59766":[],"597660":[],"59813773":[],"59833875":[],"598392":[],"59839245":[],"59914165":[],"599142":[],"59950055":[],"59955801":[],"59981487e":[],"59990347":23,"59it":6,"5cm":23,"5dd54edf2138":[],"5f":8,"5x":8,"5y":8,"6":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,23,24,26,27,28,29],"60":[1,2,3,6,9],"60000":4,"60056803e":[],"60081665":[],"60083846e":[],"60133562":[],"60158334":[],"601583341274718":[],"60164827e":[],"60176806":[],"6019067271":4,"601942":[],"60194224":[],"60236782":[],"60236938":[],"60268567e":[],"60293962":5,"603494":[],"60349429":[],"603636":26,"60378214":26,"60420593":5,"60515083":[],"6052136":[],"60522226":[],"60538875":[],"60543038":27,"60619654":[],"6063219":[],"606439":[],"60651016":[],"60673226":11,"606760":5,"60685864":[],"60707801":[],"60715594":[],"607384":11,"60738434":11,"60791699":[],"6079169911277265":[],"608028":[],"60802823":[],"60815105":6,"60820168e":27,"60826013":[],"6084611325305795":[],"60938882":[],"609389":[],"60943791":[20,26],"60975366":[],"60980325":[],"60it":6,"61":[2,7],"61018347e":27,"61031939":[],"61033524":[],"6111111111111112":1,"61123608":[],"612939":[26,27],"613579":[26,27],"61399851":[],"613999":[],"61406871":[],"614133":[],"61413333":[],"61463451":[],"614808":[11,27],"6149274949460215":[],"61520194e":[],"61536670e":[],"6153846153846154":9,"61576236e":[],"61605739":[],"61634314":[],"61702282":6,"617572":[],"61757228":[],"61775176":29,"618":[],"61808351":[],"61885405":23,"61893735":[],"618982":[26,27],"61899355e":[],"61970354":20,"6199381169260247":[],"62":[],"62083489":20,"621102":26,"62123034":[],"62168613":[],"62170669":[],"621707":[],"62223681":5,"622237":5,"62249103":[],"62364974":[],"62373464":11,"62389761":[],"62405149":[],"62420578":20,"624220":[],"62422045":[],"62432074":[],"62451626e":[],"62470815e":[],"625":7,"62535124e":27,"62571673e":[],"6257439":26,"62620724":[],"626635268":[6,29],"62683307":[],"62686358":26,"62759057":[],"62761922":[],"62767384":[],"62769765":[],"62806444":[],"62818572e":27,"62819534":20,"62894215":5,"62896882":[],"62914124":[],"629961":[],"63":[0,1,6,7,27,29],"6300745149331701":27,"63012429":[],"630224":[],"63025821e":6,"63112036":[],"63115792":[],"63162342":27,"63180447":29,"63207808":[],"63249532e":6,"63258368e":[],"63262164":[],"63264564":[],"63270833":[],"63281351":[],"63332882e":[],"63367582":[],"633676":[],"633949":26,"63395187":[],"63430285":[],"6343356":[],"63498144":5,"63550707":[],"6367311":[],"63675140":[],"637104":[],"63710438":[],"637129335071195":27,"6371293350711955":[],"63734811e":[],"637545":[],"63754524":[],"63860687":[],"63862189":[],"63901111":[],"63933182":[],"63it":[],"64":[1,2,3,4,7,13,20,26],"64008343":5,"64012627":5,"64033527":[],"64064128":[],"64113381e":[],"64141716e":[],"642380":[],"64238022":[],"64293754":[],"64295978e":27,"64298419":[],"6431453":[],"64316192":29,"64316482":29,"64425009":[],"64452882":[],"645":[],"64507671e":27,"64528459":[],"645285":[],"64615384615385":[],"646283":[11,27],"647":6,"64736858":[],"64741194":23,"64742912e":6,"647473":[11,27],"64858537":[],"6489862":[],"649339":[],"64933923":[],"649382":[11,27],"64972151":[],"64x50":1,"65":[1,2,7,8,9],"65011691":20,"65045925":26,"65136857":[],"65246006":[],"65280103":[],"6530742540053943":[],"65335632":[],"653578701904388":[],"65359027e":[],"6536392":27,"65442354":27,"654424":27,"65444431":[],"6546329613199204":[],"65522261":[],"65534935e":[],"65554133":[],"65571174e":27,"65578316":[],"65599456":27,"6562":[],"65626043":[],"65628853":[],"65643756e":[],"65693278e":[],"657041":26,"65743689":27,"65746064":[],"6577093":[],"65795279":[],"658":[],"65833132":27,"65885453":5,"65891389":27,"658914":27,"659":[],"65907898":[],"659306":[],"65it":[],"66":2,"660":[],"66001778":[],"66054752":[],"66064822":27,"66080313":27,"66087937":[],"66111673":[],"66124684":5,"66204648":6,"66219404":6,"66226149":[],"66247212":[],"66250644":[],"662854":[],"66285417":[],"66285434":[],"6628996975186953":27,"66292841":[],"66294408":27,"66295776":[],"662958":[],"66310422":[],"66351927":[],"6638":9,"66410989":[],"66443521":[],"6652177":27,"66560":9,"66599246":[],"66614025":[],"66619972":[],"666200":[],"66620847":[],"66638078":[],"666597":26,"66665241":[],"6669838269597004":[],"667":9,"667239":26,"66726905":[],"66758934":[],"668172":[26,27],"668186":[],"66818635":[],"66934291":[],"66950317e":[],"66973261":[],"66981186":[],"66it":[],"67":[2,9],"67047975e":6,"67054028":[],"671089":[],"67142502":20,"67148008e":27,"67160204":[],"67177934":[],"67180442e":27,"67236038e":[],"67258699":[],"672721":[26,27],"67314874e":[],"673457751724773":[],"67354673":[],"67392051":5,"673921":5,"67442363":[],"67554302":[],"67588315":[],"67591616":[],"67655635":[],"67697934":[],"67743788":23,"67783138":[],"67838309":[],"67882608":[],"67970864":[],"679709":[],"67973263":[],"68":2,"68029036":[],"68040392e":[],"68075712":[],"6808538439837775":[],"68127341":[],"68176047e":[],"68192193":5,"68232235":23,"68246089":[],"682464":[],"68246434":[],"68279358":[],"68282072":11,"68282653":[],"68286725":27,"68322599":[],"6834195":[],"68347072":[],"68372271":[],"68415876e":27,"68485505":[],"68534263e":6,"68542204":5,"68545647e":[],"68592431":29,"68626148e":[],"68650491":[],"68664664":[],"6869":9,"68719389":[],"687194":[],"68731669e":[],"6887363571":4,"68886089e":[],"689230294669661":[],"68929213e":6,"689345":26,"68944595":[],"689519":[11,27],"68965135":[],"68996179":[],"68it":[],"69":[2,7,9,23],"690":9,"69023787":[],"69051742":20,"690569639355314":[],"69061276":[],"690617":[26,27],"69069n_":23,"69106251e":27,"69115646":[],"6914791":[],"69154964":[],"69178261e":27,"692":[1,29],"6923076923076923":9,"6924788":5,"692479":5,"69295955":[],"69313663e":[],"69356940e":[],"69363975e":[],"6943316601792833":[],"69481746":[],"69484813e":[],"69487891e":[],"69493539":[],"69504801":6,"6960326":[],"69634577e":6,"69695259":5,"69764852":[],"69826476e":[],"6984511994530214":[],"69871367e":27,"69896355":[],"698964":[],"69908626":6,"69915639":[],"69965466":26,"69981939e":[],"69985355":[],"69987647":23,"6999536":11,"69it":[],"6e75736fdab1":[],"6f7a6bd7d79f":[],"6m":[],"6n_":23,"6pm":26,"7":[0,1,2,3,4,5,6,7,8,9,11,12,13,14,15,20,21,22,23,25,26,27,29],"70":[1,6,7,9],"70037324e":27,"70073443":27,"70127680":[],"701370":5,"70179437":[],"70205195":[],"70208087e":[],"70233292":[],"70244042":[],"70311568":[],"70354373":[],"703716d317a7":[],"70421972e":[],"70434005":[],"70465373e":[],"70473799e":[],"70482138":[],"70493970e":27,"7050":[],"70501534":[],"705698":[],"70569833":[],"70598996":[],"70653767":4,"70710678":[5,27],"70721787":[],"707218":[],"70727837":[],"70761641":20,"70769586":[],"70827971e":[],"70831425":[],"70832814":5,"70850263":20,"70856363":26,"7086067479626619":27,"70911476":26,"709698":26,"70980493":[],"70it":[],"71":1,"71075825e":[],"71112140e":27,"71131626":[],"71135895":20,"71142161":27,"711422":27,"7119":9,"712018":[11,27],"71287025":[],"713163":26,"71316703e":27,"7133574":20,"71335815":23,"7134":[],"713487":[],"71348713":[],"71350226":[],"71441535":[],"7151":[],"71721168":[],"71737253":27,"71760245":[],"71761101":27,"718165":5,"71822416":[],"71846708":26,"718697":[],"71869727":[],"71921767e":[],"72":[],"72057967":[],"72097126":[],"72101205":[],"72117282":11,"721173":11,"72139847e":27,"72174172":11,"72199579e":[],"72201591":[],"722047011333792":[],"7221662":[],"72228205":26,"7225597":[],"72261919":[],"72271878e":6,"72299619":26,"72347283":[],"7236674":5,"72373129":[],"72394787":[],"724":3,"72509099":[],"72522848":[],"72567956":[],"725680":[],"72594302":[],"7265799":[],"72671218":[],"72706640e":27,"72855434":[],"72859758":5,"72904541e":[],"7293182":[],"72981762":8,"72986500e":[],"73":[6,29],"7300372":[],"73005463":[],"7306310662842432":[],"731000":[26,27],"73133982":26,"7316348":[],"73174557":[],"73185363e":[],"73287103e":[],"733096":[26,27],"73406033e":[],"734540":[],"73454039":[],"73484667":[],"73509675":20,"73565195":[],"735738558766299":[],"73573856":[],"736115211426478":[],"73628359e":[],"73667253":11,"73704762":20,"737386":[],"73738602":[],"73752910":[],"73772319":26,"7386068":[],"73947737":[],"73it":[],"74":[2,6,29],"740":9,"74081822":8,"741264":[],"74126425":[],"74199799":20,"74260356":26,"742975":[],"74297504":[],"74368436":[],"74384949":27,"743u":[],"74409239":26,"74433499":[],"7448615806559786":[],"74577306":20,"74592114e":[],"74607851":27,"7469898175164704":[],"747224":[],"74722409":[],"74829661":27,"74840212":5,"7484669886413582":[],"74935095":[],"74943449":[],"749765":[26,27],"74997869e":[],"75":[2,5,6,8,9,11,21,26,27,29],"7501749450963715":[],"750445":[26,27],"75050135":27,"7506246":26,"750u":[],"75106135":[],"75118364":[],"751275":[],"75127504":[],"751699":[11,27],"75170092":5,"7522":[],"7525":[],"75269037":27,"75320666e":[],"75350216":[],"75352505":23,"75357263":[],"753573":[],"75357959e":[],"75361646":[],"753846153846155":[],"75390337":[],"75408649":[],"75453081":[],"75460639":[],"7546383166870465":[],"75492292":[],"75505597":[],"75517445":[],"75576555":27,"75621895":[],"756232":[],"75623206":[],"75628425":[],"756352":[26,27],"75638425e":27,"75670484":[],"756705":[],"75703556e":27,"75703965":[],"757040":[],"75747652":20,"75770568":27,"758193":[],"75819326":[],"75838233":[],"75853556e":[],"75934097e":27,"75it":6,"76":[2,9,24,26],"7600498":20,"7601030735620569":[],"76060096":[],"76063234":[],"76066069":[],"76196895e":27,"763u":[],"76400155":[],"7643536":[],"765":7,"7651899":[],"76529528":[],"76598941e":[],"76674796":[],"76707201":[],"76731400e":[],"76802186":[],"76834212e":[],"768813":[],"76881337":[],"76892410e":27,"7692307692307693":9,"76923076923077":[],"76936315":5,"7694444444444445":1,"7696835316227453":[],"7697":29,"77":[2,9,24,26],"77034458":[],"770345":[],"770356":20,"7705527590466072":[],"77070756":[],"770708":[],"77081499e":27,"77085375":[],"77144798e":[],"77152076":5,"77176703":23,"7718":9,"77288474":20,"773329728649545":[],"77332973":[],"77333117e":[],"774300":[],"77494715":[],"775":[],"77533038e":[],"7754499790100834":[],"77553346e":[],"77559332":[],"77618224":[],"77627886":[],"77632220e":[],"77636e":13,"77640601":[],"77662945":[],"7767978193240488":[],"77711437":[],"77714169":8,"77754132e":[],"77756468":20,"7780841853755783":11,"77816763e":[],"7782028952":4,"77836133e":[],"77859437":[],"77865169":[],"77954956e":5,"77it":[],"78":2,"78009660e":27,"78011544":[],"78018374e":[],"78027207":20,"7803076":26,"78035464":[],"78104004e":27,"78156479e":[],"78184120e":6,"78263078":26,"78272242e":[],"78286771":[],"78329097":[],"7843182645426894":[],"78459267":[],"7846153846154":[],"785061":26,"7851533175757255":[],"78515702":23,"785157020098348":23,"78561443":[],"78596961":[],"7860324":[],"7866024":[],"78674758":[],"787349":[],"7873493":[],"78738205e":27,"78752269":[],"78941903":5,"78972773":[],"78980112":[],"78988962e":[],"78990036":[],"79":2,"79000942e":[],"79009329":[],"790818":[],"79081838":[],"79093776":[],"79106527e":[],"79111643":5,"791123":[],"79135075":[],"79145214e":[],"79190435e":[],"79224555":[],"7923086060375724":[],"79230861":[],"79285683e":[],"79313278":26,"793167":[26,27],"79339933e":27,"79345488":[],"79394867":[],"7939646":[],"79419162":[],"794282":[11,27],"79441434":[],"79449156":[],"79512696":[],"79544237":26,"79552752":[],"79559972":[],"79577619":[],"79589065":13,"79602861":[],"79700882":26,"79725436":26,"79754246":[],"797e":6,"79827506e":[],"79856831e":[],"798761":[],"79876149":[],"79934594":23,"79974359":[],"79994469":26,"79it":6,"7bsq":[],"7c394b1e8b71":[],"7d7d58":[9,10],"7f2b3a6174c2":[],"7jexgh5soo":[],"7m":[],"7odqolophta":[],"8":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,20,21,23,26,27,28,29],"80":[0,1,2,5,6,8,9,27,28,29],"800":[4,7],"800615":[],"80088524":20,"80108096":[],"80162359":[],"802796":[],"80279641":[],"80280228":[],"80315282":[],"803153":[],"80320028e":27,"80344720e":[],"80354994":6,"803804":[],"80380438":[],"80413812e":27,"80422999":[],"80460179":29,"80462258":[],"80469739":5,"8049181":[],"80515205":20,"8055555555555556":1,"80561191":[],"80609615e":6,"80609616e":6,"80638926e":[],"8064":[],"80659468":11,"80661365":[],"806614":[],"80728853":[],"80734875":[],"807349":[],"80738685":[],"8074242":[],"80747253":[],"80763641":23,"80768431":[],"80788856":26,"80842254":[],"808423":[],"80847477e":6,"80947943":[],"80961969e":[],"80994952":[],"80it":[],"81":1,"81007682":[],"8104687":[],"810479":11,"8104791":11,"81048318e":6,"81051173":[],"81065392":[],"8108619":[],"81160425":5,"81168086":[],"81187794":[],"81333804":6,"81333805":[],"81343612":27,"81388724e":[],"814":[7,11],"81409445":20,"8143782":5,"81447915":[],"814860456269878":[],"81486046":[],"815563":[],"81592628":[],"81597834":29,"815am":[24,26],"81633628":11,"816454":[26,27],"81656508":[],"81671821":[],"81681019":[],"816847":[26,27],"81709481":20,"8175":[],"8177581":23,"81784072":20,"818059":[],"81805914":[],"81874188e":[],"81912272":[],"8192798":[],"81955926":[],"81974332":[],"82":2,"82027884":[],"82045496":[],"82073684":[],"82092612e":[],"82113506":20,"82187419":26,"8219235992494145":[],"8219236":[],"82198978":5,"82202765":[],"82250097e":[],"82253259":[],"8225325933960339":[],"82358522":[],"823585220707946":[],"82365318":20,"82404007e":[],"82454365":[],"82484316":[],"82592":[],"8260149":20,"82650876":[],"826509":[],"8265786":5,"82747653":[],"827477":[],"82750687":[],"82771329e":[],"82773778":[],"828190347382744":[],"82819035":[],"828940":[],"82894017":[],"82949718e":[],"829648415811072":[],"82964842":[],"82975422e":[],"82985482":13,"82997013e":5,"83":[2,26],"8305555555555556":1,"83146596":29,"83170827":23,"83303886":26,"83320794":[],"833208":[],"8338786":[],"834027":[],"83402742":[],"83512277":5,"83516719e":[],"83632853":[],"836874":[],"83687444":[],"83698677":[],"83723539":[],"83724637":[],"83757843":[],"83760781":[],"83770135":27,"83858458":23,"838584583553216":23,"839818":26,"83982725":[],"83999999999999":[],"83it":6,"84":2,"84031027e":[],"84062065":[],"84150451":[],"84203538":[],"84228957e":[],"84232163":[],"842436":[26,27],"84291049e":27,"84292394":[],"842u":[],"84310736":[],"84317815":[],"84355903e":1,"8437169762110144":[],"84371698":[],"84443254e":1,"84569271":[],"84614066":[],"8461538461538461":9,"84643086":11,"846431":11,"846471":20,"84653115":[],"84696315e":[],"84708672":[],"84723978":20,"84780262":6,"84854738":[],"84858":28,"848618":[],"84861838":[],"84886051":[],"848861":[],"84923989e":6,"84924359":20,"84924834":[],"84942942":[],"84955363":11,"849554":11,"84977962":[],"84994524":5,"85":[1,2,9],"850164":5,"8503720991789538":[],"85044503":11,"85051211":[],"85065653":[],"85071429e":[],"85086629":[],"85091337":[],"85097049":[],"85102768":[],"85104809":[],"85120833":[],"85212606":[],"85263220":6,"8527777777777777":[],"85278920e":[],"85299536":[],"853241":[],"85324115":[],"853835":26,"853u":[],"8548082":[],"8557822":[],"85581869":[],"85595734":[],"85601992":5,"85630009":[],"85688243e":[],"85694362":[],"85754328":26,"85758696":[],"85785878e":[],"858":9,"858185":26,"8583333333333333":1,"85834701":[],"85843684":[],"85895314":20,"85902603":[],"85953586":[],"85959007":[],"859628021353642":5,"85979474e":[],"85982436":[],"85it":[],"86":2,"86014003e":[],"86058560e":[],"8608479":[],"860848":[],"861":[],"86117291":5,"86134827":5,"86144422":[],"86145244":11,"861676":26,"86171505":[],"8617319":20,"86181269":[],"8620652":[],"86208133":[],"86252988":6,"86265211":23,"8634474":20,"86361743":[],"86387759e":[],"8638888888888889":1,"86416637e":[],"86425056":[],"864251":[],"86478158":[],"865158":[],"86515815":[],"86546962":[],"86550074":[],"86572911":[],"86597540e":[],"86623151":[],"86630":9,"86635201e":[],"8666666666666667":1,"86810":9,"86811137":[],"86858926":[],"86879198":[],"868792":[],"86893619":[],"869":[0,26],"86905621":[],"86925797":[],"869258":[],"87":[0,9,27],"870":[0,26],"8702784034":4,"8705211":[],"87072815e":27,"87099757":[],"871":[0,26],"87131961e":[],"87136737":[],"87191952":[],"871920":[],"872":[],"8722222222222222":1,"87243817":[],"873":[0,6,26],"87327414":[],"87327422":[],"87379129":[],"87381451":5,"874":[0,26],"874951":[],"87495119":[],"875":1,"87532289e":[],"87556429":26,"8759":13,"876":6,"87602519":20,"87618586":[],"87625493":[],"87627342":29,"87647541":[],"876664":[],"87666423":[],"87667593906086":[],"87667594":[],"8768":[],"878123":26,"878297":[],"87836018":[],"87849647":[],"878891":11,"87889126":11,"87890188e":[],"87902877":[],"8792323":[],"879271":[],"87927126":[],"87927601":[],"87940752":[],"87it":[],"88":[2,13],"88001352":[],"88017651":[],"880177":[],"88029792e":27,"88031314":[],"88046261":5,"8805555555555555":1,"88063413":[],"88118231":[],"88137798":[],"88168312e":6,"881788":[],"88182591":27,"8826033":[],"88274486":[],"883":[],"88323026":27,"88335933":[],"88336879":5,"88352503e":[],"884":[],"88450476":[],"884669":27,"88477619":[],"885":[],"8850944":[],"88512489":[],"88529063e":6,"8855417382991412":[],"88554174":[],"88560514":[],"88595926":[],"886":[],"88613493":27,"8866190885623907":[],"88661909":[],"88679947":[],"88690267e":27,"887":[],"887366":[],"88736621":[],"88739247":[],"887392472315955":[],"88746795":[],"88778045":20,"88778091":[],"8879":[],"88871662":[],"8888888888888888":1,"889080":[],"88908038":[],"8894":[],"889686":[],"88981061":20,"88it":[],"89":2,"89045167":[],"89126914e":27,"8916666666666667":[],"89249454":[],"89261458":[],"89274639":[],"89288636":11,"89298533":[],"89335182":[],"893352":[],"893489":[],"89348922":[],"89396164e":[],"89410423":5,"89421873":[],"8944444444444445":1,"89482188":26,"89543736":[],"89545727":[],"89550839":[],"89563529":0,"89565264":[],"89582298":[],"89591553e":[],"8962476":[],"896248":[],"89658408":13,"8970168":20,"89778737":23,"89779302":20,"89842298":[],"89869600e":[],"89873772":[],"89876748":[],"89881026":26,"8990231":26,"8991514":[],"89944994":[],"899450":[],"899734":[],"8f":[6,29],"8g":[6,29],"8n":20,"8x8":1,"9":[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,22,23,24,26,27,28,29],"90":[1,2,6,9],"90034533":[],"90044211":11,"900449":[],"9005135872087155":[],"90054363":[],"90066122":[],"9007164":[],"90075537":5,"90092179":26,"9011":6,"90189742":[],"90220243":5,"90252116":[],"90266948":5,"90268858":[],"9027777777777778":1,"9033":[],"90342093e":[],"903824":[],"90382431":[],"90383424":[],"904":[],"9040":9,"904221":[],"90475506e":6,"90491568e":[],"9054":29,"90540529":[],"9055555555555556":1,"90559087":[],"905591":[],"90595152":[],"906298":[],"90629812":[],"90643349":[],"9067362":[],"906747":5,"90694878":[],"90699099":[],"90793026":20,"90813514":[],"9083333333333333":[],"90884627":[],"90886410e":27,"90895045":[],"90940378":[],"9096305360355206":[],"90999452e":[],"91":[24,26],"910":9,"91094169e":[],"9111111111111111":1,"91124889":[],"91128596":5,"91236018":26,"912u":[],"913":[],"91346044":[],"91379157":[],"913791573406831":[],"91383439":[],"91396388":[],"914":[],"91410101":[],"91416375":[],"91479093":[],"91492986e":6,"915":[],"91511388":[],"91549644":[],"915549":[],"916":[],"91616374":[],"916164":[],"9166666666666666":9,"91682433":[],"9169":6,"917":[],"91760278":5,"918":[],"91811160e":[],"91812702":5,"918992":[26,27],"919":[],"91936525":[],"9196398947631166":[],"91it":[],"92":[6,9,24,26],"92087142":[],"9212905":[],"921291":[],"92143477":[],"921435":[],"92169784e":[],"92173899":26,"92208477":[],"922085":[],"92250132":26,"92272314":[],"92275042":[],"922u":[],"9230769230769231":9,"92322465e":[],"92337385e":[],"92343595":[],"923602":26,"92373266":[],"924018":[],"92461411":[],"92463154e":[],"92472578":11,"924726":11,"92477093":[],"924e":6,"925":1,"92507116e":1,"9252772":[],"92578916":5,"92603747":[],"92626212":[],"92631966":[],"926320":[],"92651068":29,"926583":[],"92658312":[],"92738529e":[],"92754397":[],"927544":[],"9277777777777778":1,"92811987":[],"92817032":[],"92819235":[],"92857143":7,"92863147":[],"92888885":[],"92916438e":[],"9292":[],"92921648":[],"929612":5,"92968793":[],"92995253e":[],"92it":[],"93":[],"93023603":[],"93034176e":[],"93037171":[],"930372":[],"9305555555555556":1,"930583":[],"930683":[],"93082503":[],"931":[0,26],"93155188":5,"93158979":5,"93166184":26,"93188452":[],"931885":[],"93229640e":[],"93248252":[],"932483":[],"93265898":20,"932u":[],"933":[5,28],"93325497":[],"93371943e":[],"93386901":[],"93414191":[],"93492130e":6,"93503":11,"93518534":[],"93522293":[],"93579127":[],"93586895":[],"935u":[],"93601008e":27,"9361111111111111":[],"936762":[],"93676229":[],"936856":[],"93685631":[],"937":23,"937082":[26,27],"93725291":5,"93792215e":[],"93799826":5,"938":23,"9383327":[],"9387":9,"939":[0,23,26],"93969474":20,"93it":6,"94":[7,23],"9400":[],"94015598":13,"94054854":[],"940549":[],"94073014":20,"94076632":[],"940776":[],"94077605":[],"94107596":[],"9410965":23,"94117118":[],"941866404575299":[],"94212937":[],"94226022e":6,"94227931":[],"9424":[],"9426":29,"94271589":[],"94273542":27,"94284104":5,"94304103":27,"94320205":5,"94321297":[],"94390673":23,"94404545":20,"9440887":[],"94413158":[],"9444444444444444":1,"945":[],"94506946":23,"94548496":[],"94591015":[20,26],"946":[],"94639099":11,"946393":[],"94640767":[],"94659383":[],"946957":5,"947":[],"9472222222222222":1,"94735055":[],"9474044318317307":[],"947903":[],"94790323":[],"948":[],"94822514":6,"9482527":5,"94861749e":[],"948641":[],"94867516":[],"948675162607389":[],"949":[],"94905663":[],"949087":[],"949107":11,"94910711":11,"94915262":[],"94917559e":[],"94925462":13,"9492866":[],"94966662e":27,"94986593":[],"95":[1,7,9,11,29],"95008046":6,"95011994e":[],"95054046":[],"95079224":[],"95079764":[],"95136998":[],"9518":6,"95231424":5,"952387":27,"9524":[],"9527777777777777":1,"95284275":5,"95302":[],"953065564":1,"95324615":[],"95351665":5,"9538239":[],"954":23,"9543538503067644e":9,"95446837":[],"95456434e":27,"9549351910143222":[],"954988":[],"954u":[],"95511792":27,"955118":27,"95539206":26,"9555555555555556":1,"95569422":[],"955820c21e8b":4,"95594445e":[],"95604667":20,"95645365":[],"956454":[],"956563":[11,27],"95684892":5,"95703":13,"95714723":[],"957147232685324":[],"957421":[],"95742107":[],"9577775794806141":[],"958194":[],"95819409":[],"958228616652075":[],"9582286166520774":5,"95824108":[],"958476":[],"95911004":[],"95921014e":27,"95927495":[],"9594":[],"959533":[],"95953339":[],"95976303":[],"95994091":[],"95it":[],"96":[6,7,11,29],"960":23,"9601304850018328e":[],"960130485007504e":6,"960130485007934e":[],"9601304850213484e":[],"96013048502692e":6,"96024953":5,"96032148":[],"96081861":[],"96084663":5,"961":23,"9618":6,"962":23,"96251494":[],"96257580e":[],"96261297":[],"9626883":[],"962990":26,"9631064":[],"96317459":[],"9635449873404844":[],"9637117593816477":6,"96390357":[],"9640435":5,"964268":[],"96426825":[],"96431281":[],"964735":[],"96473528":[],"96489434":[],"9649652536":4,"96496641":[],"96504592":[],"96514849":[],"96527903":[],"96534137":[],"96542994e":[],"96551427e":[],"96552318":[],"965548":[26,27],"96562567":20,"965838":11,"96583833":11,"96601782":[],"96606158":[],"96620033":[],"96631321":27,"96653373":[],"96667129":[],"96670977":[],"96686324e":[],"96688672":5,"966899":26,"96735081e":[],"9674916":5,"967536":[],"9675364":[],"96769675":[],"9678":[],"967809":[11,27],"96793569e":27,"96804878":23,"96810393":[],"96812218":[],"96863851":[],"96941962":[],"9698377974429513":[],"96987657":[],"96992454":[],"96995249":[],"96it":[],"97":7,"97005689":5,"970057":[],"970477":[],"97047717":[],"97065296":[],"97087408":13,"97108e":13,"97117751":[],"971688":[],"97186457":11,"971865":11,"97200305e":[],"97202":[],"9722222222222222":1,"972286":[],"97228638":[],"97230501":[],"97243128":5,"972643":5,"97264311":5,"97300836":5,"97305528e":[],"97409408":23,"9740940821517547":23,"97421755":5,"974218":5,"97434186":[],"97467734e":[],"97477238":[],"97497404e":6,"975":1,"97507735":5,"97509292":[],"97533203":[],"97547354":[],"97547354476579":[],"975510299261579":[],"97558913":26,"9760832":[],"97656972":[],"97665553e":[],"97690235":29,"97705827":5,"97708897":[],"977089":[],"97713364":[],"97731700e":[],"977418":[],"97758848":5,"9777777777777777":1,"977880":[],"97788031":[],"978":28,"9780387310732":25,"9780387848570":25,"9781492032632":25,"97840267e":[],"97840468":[],"978553":5,"97898392":[6,27],"97906022e":27,"97926491":5,"97926642":[],"97936584":20,"97952271":[],"97955570e":[],"97980332":[],"97it":[],"98":[0,1,7,9],"980":9,"98005862e":[],"98017611":[],"98018358":[],"98036405":[],"9805555555555555":1,"98060869":[],"98091621":5,"98129668":[],"981321":[26,27],"98139097":5,"98153145":[],"98248312":[],"98248681":5,"982487":5,"98249059":[],"982491":[],"98266587":[],"98275501":5,"982758":[],"98275836":[],"98316352":[],"983164":[],"98328759":[],"983310":[26,27],"98346748":[],"9835443722554817":[],"98399675":[],"98400474":[],"98413059":5,"984525":[],"98453971":[],"98454786":5,"98467494":[],"98496212":26,"985":23,"98566191":5,"98581925":[],"9859105":26,"986":23,"98604688e":[],"9861111111111112":1,"986699":5,"98680716":5,"98699753":[],"98716878":5,"98740124":[],"98750794e":27,"98756882":[],"98794823":[],"9879924":[],"98808176":5,"98822371":6,"98842948e":[],"98880065e":[],"9888888888888889":1,"989":23,"9890348":5,"98919197":13,"9893149172528393":[],"9893405":[],"9893447":5,"989393825676568":11,"98947894":27,"98950411":[],"989645675273948":[],"9898254753574576":[],"98982548":[],"9898ff":[9,10],"99":[6,7,9,11,13,29],"990":9,"99009525":5,"9902552771282336":[],"99035178":26,"99049330":[],"99051150":6,"9906601091977738":[],"99088801":5,"9909606":[],"991":23,"99115119":5,"99157584":[],"99176998":5,"99180856e":[],"992":23,"9920084":26,"9920099299023244":[],"99218609":20,"99242921":5,"992504":[],"99255249100346":27,"99265097":5,"99274513":[],"993":23,"99316252":5,"99353454":[],"993535":[],"993537":[],"99353748":[],"99371056":5,"99374388":[],"993865":[],"99389612":5,"99397049":[],"993972":[],"99397245":[],"9940672992288855":[],"9943201":5,"99435648":[],"9947613223847728":[],"9947756":5,"99492986":5,"9949347794612255":[],"99498108":[],"99519225":[],"9952495547161103":[],"9952714635635723":27,"99528218":5,"99539415":5,"99541140030306":[],"9954205504035075":[],"9954441212986554":27,"9954911579937146":[],"9955282554647219":[],"9956582":[],"99566069":5,"9957763445154583":[],"99578809":5,"9959033816551833":[],"9959293935368551":[],"99594988":[],"995950":[],"9959898232423614":[],"996":[5,28,29],"9960039497353536":[],"99608161":5,"9960913291755783":[],"9962074631757628":[],"9963202597920238":[],"9963525271588923":[],"9963864072152893":[],"9963961":5,"99650061":5,"9965048209369255":[],"996535469511469":[],"9966080206050006":[],"996694948530336":[],"9967458":5,"9967912002709148":[],"99700706":5,"99709215":5,"99724883":[],"99729756":5,"9973102337301051":[],"99751458":5,"99758326":5,"99760425":13,"99775587":5,"99775949":[],"997830":11,"99783004":11,"99793613":5,"99799099":5,"99813653":5,"99820454":[],"99828624":5,"99829953":[],"9983295":5,"99845267":5,"99854557":27,"998577":5,"99861053":5,"99862019":27,"99869482":27,"99871521":5,"99876945":27,"99881845":5,"99883628":27,"99884384":5,"99884409":27,"99891093":27,"99891873":27,"99893323":5,"99898558":27,"999":[9,23],"99901131e":[],"99901896":5,"99903755":5,"99906023":27,"99911427":5,"99912709":27,"99913489":27,"99918546":5,"99919837":5,"99920175":27,"99926459":5,"99927642":27,"9993237":5,"99933188":5,"9993511":27,"99938942":5,"99941797":27,"9994385":5,"99944272":5,"99948754e":[],"99949266":27,"99949306":5,"99952195":23,"99953381":5,"99953475":5,"99956735":27,"99957911":5,"99961294":5,"99965056":5,"99967865":5,"99970894":27,"99970988":5,"9997332":5,"99975913":5,"99977416":[],"99977849":5,"99978365":27,"99980002":5,"9998161":5,"99984732":5,"99987324":5,"99989476":5,"99991263":5,"99992698":[],"999927":[],"99992746":5,"99993978":5,"99995":5,"9999555851685968":[],"999955585168597":6,"9999840939906267":[],"9999858320366368":[],"9qcu8vcxxru":[],"9x":[6,21],"9y":[6,21],"\u00f8yvind":[6,27,28],"abstract":1,"boolean":4,"break":[0,4,6,11,14,26],"byte":[20,26],"case":[0,1,2,3,4,5,6,7,11,12,13,14,16,19,20,21,26,29],"catch":[0,26],"char":[],"class":[0,1,3,4,6,7,8,9,11,12,13,23,26,29],"const":[],"default":[0,1,2,4,6,7,13,16,20,21,26,27,28],"do":[0,2,3,4,5,6,8,9,10,11,12,13,14,15,16,20,21,27],"ekstr\u00f8m":4,"eng\u00f8i":[],"enum":[3,4,14],"export":9,"f\u00f8470":[24,26],"final":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,16,18,21,22,23,24,26,29],"float":[0,3,4,5,9,11,13,14,20,26,27],"function":[2,3,4,5,9,14,15,16,17,18,19,20],"import":[0,1,2,3,4,6,7,8,9,10,11,12,13,14,16,17,21,23,29],"int":[0,1,2,3,4,5,6,11,13,14,20,23,27,29],"long":[0,1,3,4,12,13,26],"m\u00f8svatn":[6,21],"new":[0,1,2,3,5,6,7,8,9,10,11,13,14,17,20,21,26,27],"null":26,"public":[0,3,4,14,19,26],"return":[0,1,2,3,4,5,6,7,8,9,11,13,14,20,21,23,26,27,28,29],"s\u00f8rli":[],"s\u00f8rlie":[],"sch\u00f8yen":[6,27,28],"short":[4,5,21,28,29],"steinsv\u00e5g":[],"super":[5,27,28],"switch":0,"throw":[3,6,23,29],"true":[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,20,21,23,26,27,28,29],"try":[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,19,20,21,23,26,27,28,29],"var":[1,5,6,10,11,13,18,21,23,26,27,28,29],"while":[0,1,3,4,5,6,7,8,9,11,12,13,23,26,27,28,29],A:[2,3,5,6,7,10,11,12,13,16,18,19,20,21,22,23,24,25,27],AND:2,And:[0,3,4,5,6,9,13,19,21,23],As:[0,1,2,3,4,5,6,8,10,12,13,16,20,21,23,26,27,28,29],At:[0,4,6,13,21,26],BE:[0,26],Be:[2,19,26],Being:13,But:[0,1,2,3,5,6,9,10,23,27,28,29],By:[0,3,5,6,12,13,17,20,26,27,28,29],For:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,25,26,27,28,29],IF:[6,28],IN:25,If:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,15,16,19,20,21,23,26,27,28,29],In:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,25,26,27,28,29],Is:11,Ising:[5,12,27],It:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,23,26,27,28,29],Its:[1,2,4,11],NO:[7,11],No:[6,9,26,28],Not:[0,1,5,6,26,27,28,29],OR:23,Of:23,On:[0,3,15,22,23,24,25,26],One:[0,1,3,4,5,6,7,8,11,12,13,17,21,23,27,29],Or:[0,1,6,21,26],Such:[0,6,12,16,23,29],That:[0,5,7,10,11,12,14,21,23,26,28,29],The:[4,10,13,14,15,16,18,20,21,22,23,24,25],Then:[0,1,6,8,9,10,11,12,13,14,20,21,26,27,29],There:[0,3,4,5,6,8,9,11,12,14,20,21,22,23,24,26,27,28],These:[0,3,4,5,8,9,10,11,12,13,14,15,16,20,21,23,24,26,27,28],To:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,23,28,29],With:[0,5,6,8,9,10,11,12,14,18,20,21,23,26,27,29],_0:[5,8,10,11,13,27],_1:[2,5,6,8,10,11,12,13,14,20,27,28],_2:[2,5,8,11,12,13,20,27],_3:20,_4:20,_9:13,_:[0,1,2,4,5,6,7,8,9,10,11,12,13,17,18,20,21,26,27,28,29],_________________________________________________________________:[],__call__:[],__class__:10,__doc__:[6,29],__future__:[8,9],__getattr__:28,__getitem__:[],__init__:[1,2,3,4,14,28],__main__:2,__mosek:[],__mul__:[],__name__:[2,10,28],__rmul__:[],__traceback__:[],__version__:[3,4,14],_api:[3,4,14],_asarrai:[],_auto10:[6,12],_auto11:6,_auto12:6,_auto1:[2,3,4,5,6,7,12,13,20,23,27],_auto2:[2,3,4,5,6,12,13,20,23],_auto3:[3,4,5,6,12,13,20],_auto4:[4,6,12,13,20],_auto5:[4,6,12,13,20],_auto6:[4,6,12,20],_auto7:[4,6,12,20],_auto8:[6,12],_auto9:[6,12],_ax:[],_base:8,_broadcasting_shape_rul:[3,4,14],_build:[0,17,19,21,25,26],_build_call_output:[],_c:1,_call:[],_call_flat:[],_check_1d:[],_check_optimize_result:[7,11],_compon:11,_config_pb2:[],_coordinate_desc:6,_copy_docstring_and_deprec:[],_cpgg0jyh8m:[],_decor:[0,26],_delta:[3,4,14],_depth:9,_dilate_shap:[3,4,14],_ey:[3,4,14],_fraction:9,_get_lin:[],_getitem_multilevel:[],_handl:[],_i:[0,1,2,5,6,7,8,11,12,13,15,16,17,21,26,27,28,29],_inference_funct:[],_interpolatefunctionerror:[],_is_primit:[],_j:[0,1,2,3,5,6,8,13,17,21,27,28,29],_jit_compil:[],_k:13,_l:12,_lambda:[6,26],_leaf:9,_logist:[7,11],_m:10,_make_vjp:[2,13],_maybe_define_funct:[],_multilayer_perceptron:1,_n:[2,5,8,11,13,27],_node:[2,9,13],_not_impl:[3,4,14],_notokstatusexcept:[],_np:2,_num_output:[],_one:[3,4,14],_p:[5,8,27],_plot_arg:[],_process_traceback_fram:[],_r:[],_ratio:11,_reduce_and:[3,4,14],_reduce_max:[3,4,14],_reduce_min:[3,4,14],_reduce_or:[3,4,14],_reduce_sum:[3,4,14],_reduce_window_max:[3,4,14],_reduce_window_min:[3,4,14],_reduce_window_prod:[3,4,14],_reduce_window_sum:[3,4,14],_sampl:9,_select_forward_and_backward_funct:[],_split:[6,9,21],_src:[3,4,14],_stateful_fn:[],_stateless_fn:[],_sy:[3,4,14],_t:13,_test:[6,21],_trace:[2,13],_tri:[3,4,14],_upcast_fp16_for_comput:[3,4,14],_valu:[2,13],_varianc:11,_weight:9,_xla_comput:[3,4,14],_zero:[3,4,14],a0:3,a0faa0:[9,10],a1:[0,26],a2:[0,26],a3:[0,26],a4:[0,26],a_0:[0,26],a_1a:[0,26],a_2a:[0,26],a_3:[0,26],a_3a:[0,26],a_4:[0,26],a_4a:[0,26],a_:[0,1,16,20,26,27],a_h:1,a_i:[0,1,2,12,26],a_j:[1,12],a_k:[0,1,12],a_ndim:2,aaron:25,ab:[0,2,5,13,14,26,27,28],ab_channel:19,abandon:1,abbrevi:22,abid:23,abil:[0,10,26],abl:[0,1,4,5,6,7,10,12,13,16,21,27],abort:[],about:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,19,20,21,24,28,29],abov:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,25,26,27,29],abovement:[6,29],abscissa:13,absolut:[0,2,5,6,13,26,27,28,29],absorb:[27,28],acceler:13,accept:[0,3,6,9,21,26,27],access:[0,3,11,23,27],accid:[4,6,29],accident:[3,4,14],accmod:[],accompani:[0,26,27],accomplish:[8,9,13],accord:[0,1,2,5,6,9,12,13,14,15,16,23,26,28,29],accordingli:11,account:[0,3,5,13,15,23,26,28,29],accumul:[12,13,23],accur:[0,3,4,6,10,13,29],accuraci:[0,1,3,4,5,6,7,9,10,11,12,26,27],accuracy_scor:[0,1,10,26],accuracy_score_numpi:1,achiev:[0,1,5,6,8,12,20,26,28,29],aco:23,acquaint:[19,26],acquir:[1,19,26],acr:[0,27],across:[1,3,6,9,19,26,29],act:[1,3,20],action:23,activ:[0,2,3,4,9,22,24,26,29],actual:[0,1,4,5,6,8,11,16,20,23,26,27,28,29],ad:[1,3,4,5,8,13,14,15,16,20,28,29],ada_clf:10,adaboostclassifi:10,adadelta:13,adagrad:[],adam:[1,3,4,24,26],adapt:[4,6,13,25,27,29],add:[0,1,2,3,4,5,6,8,10,11,12,15,16,17,21,23,26,27,28,29],add_lin:[],add_outgrad:2,add_subplot:[1,7,12,14],addendum:5,addit:[0,2,3,5,6,7,8,9,10,12,13,15,17,19,20,21,23,24,25,26,27,28,29],addition:[12,13],address:[1,9,11,13,26],adjac:[3,12],adjoint:[5,27,28],adjust:[0,5,12,13],admir:[0,26],advanc:[4,6,12,25,26,29],advantag:[1,3,5,6,10,13,20,28,29],adversari:26,afecionado:26,affect:3,affin:[0,3,8,11,27],afford:3,aficionado:26,aforement:14,african:[0,27],after:[0,1,2,4,5,6,9,11,12,13,15,19,20,21,23,26,27,28,29],afterward:[0,26],ag:[0,7,22,26,27],ag_0:2,again:[0,1,4,5,6,7,8,10,11,12,13,15,16,21,23,26,27,28,29],against:[1,4,7,10],agegroup:7,agegroupmean:7,aggreg:[9,10],agorithm:10,agre:[5,6,23,28,29],agreement:13,ahead:9,ai:[0,25],aid:11,aim:[0,1,4,6,7,11,14,15,16,19,20,21,27,29],ainv:5,airplan:3,aka:[5,28,29],al:[0,2,4,15,16,17,25,26,27,28,29],alarm:[5,7,28,29],algebra:[0,3,5,13,19,27,28,29],algo:[],algorithm:[0,1,2,4,5,6,7,8,13,14,19,20,21,23,25,26,28,29],align:[0,2,5,6,7,8,13,21,23,26,27,28,29],all:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,15,19,20,21,22,23,24,25,26,27,28,29],all_gath:[3,4,14],all_to_al:[3,4,14],allevi:[1,13],alloc:[3,20],allow:[0,1,2,3,5,6,8,10,13,15,16,19,20,21,26,27,28,29],almost:[0,1,6,8,11,13,23,26,29],alon:[2,9],along:[2,3,4,5,6,9,10,11,19,20,26,27,28,29],alpha:[0,1,2,3,4,6,7,8,9,10,13,14,23,26,27,29],alpha_0:3,alpha_1:3,alpha_2:3,alpha_:10,alpha_i:[3,13],alpha_k:13,alpha_m:10,alpha_n:3,alpha_opt:13,alreadi:[2,3,4,5,6,10,12,19,20,23,26,27,28,29],also:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,26,27,28,29],alter:1,altern:[0,1,4,5,6,7,8,9,11,13,20,21,26,27,28,29],although:[0,1,5,6,8,10,13,16,26,28,29],alwai:[0,3,5,6,12,13,16,23,26,27,28,29],am:[4,27],ame2016:[0,26],american:[0,27],among:[0,3,5,9,10,12,20,26,27,28],amongst:[5,28,29],amount:[0,1,3,4,6,8,10,14,19,29],an:[1,2,3,5,6,7,8,9,11,12,13,14,15,16,17,19,20,21,23,24,25,27,28,29],an_:23,anaconda3:[],anaconda:[0,1,15,19,21,26],analog:13,analys:[6,28,29],analysi:[1,3,4,7,14,15,16,17,20,25],analyt:[2,3,5,6,7,12,13,15,19,21,26,27,28,29],analyz:[0,1,3,4,5,6,14,16,17,21,23,26,27,28],andrew:1,angl:[0,3,9,27],anharmon:3,ani:[0,1,2,3,4,5,6,7,8,9,10,12,14,18,23,26,27,28,29],anim:[4,12],ann:12,annot:[0,1,3,7,8,26,27],announc:26,anoth:[0,1,3,4,5,6,7,8,10,11,12,13,20,21,23,26,27],anp:2,ans_vspac:2,ansatz:[0,26],answer:[0,1,3,5,6,20,21,24,26,28,29],antialias:[2,6,21],anticip:4,anymor:[1,8],anyon:[4,8],anyth:[1,23],anytim:[24,26],apach:[1,3,4,14],apart:[11,13],api:[1,3,4,14,19,26],appar:2,appear:[0,1,3,13,16,20,23,26],append:[1,3,4,8,9,13,26],appendcon:[],appendix:21,appendvar:[],appli:[0,1,2,3,4,6,7,8,9,10,11,12,13,15,16,21,23,25,26,27,28,29],applic:[0,1,3,4,5,6,7,9,12,13,20,21,23,25,26,27,29],apply_gradi:4,approach:[1,2,4,5,6,9,10,11,12,13,17,19,23,25,27],appropri:[2,6,9,12,13,19,23,29],approv:26,approx:[0,2,3,6,10,11,13,23,26,29],approxim:[0,1,2,3,4,5,6,7,10,11,13,18,21,23,26,27,28,29],apt:[0,15,19,21,26],aq:23,ar:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,22,23,24,25,26,27,28,29],aragorn:26,arang:[1,3,4,6,7,9,10,12,13,21,26],arbitrari:[1,4,6,8,12,13,23,29],arbitrarili:[0,1,11,26],arc:[6,21],architectur:[3,4,12],area:[0,3,6,9,21,25,26],arg:[0,2,13,26],argc:[],argmax:[1,11],argmin:[4,10,14],argnum:[2,13],argnum_0:2,argnum_1:2,argsort:11,argu:[1,13],argument:[0,2,3,5,6,11,12,13,21,26,27,28,29],argv:[],argval:2,aris:[0,6,12,13,23,26,29],arithmet:[0,13,20,26],arm:[3,4,6,14,28],arma:[],armadillo:20,around:[0,1,4,5,6,11,23,26,28,29],arr:2,arrai:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,19,21,23,27,28,29],arrang:[3,26],arraybox:13,arriv:[0,6,9,11,20,23,26,29],arrow:12,arrowprop:8,art3d:13,art:[0,1,15,19,26],articl:[0,3,4,6,10,18,26,27,28,29],artifici:[0,2,7,12,25,26],artificialneuron:12,arug:13,arxiv:[3,4],as_fram:27,asap:26,asarrai:[0,2,6,9,27,28],asc:[],ascii:[],ashrafi:[],asid:27,ask:[5,6,11,12,28,29],aspect:[0,6,19,21,26,27,28],assembl:[0,3,26],assert:4,assess:[0,6,21,26,27,28,29],assici:4,assign:[0,7,8,9,12,13,14,22,24,25,26,27],associ:[0,6,9,12,14,23,26,29],associative_scan:[3,4,14],assum:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],assumpt:[0,3,5,6,9,11,18,21,23,26,27],ast:[0,5,6,18,21,26,28,29],astronomi:[26,27,28,29],astyp:[4,9,10],asymmetri:[0,26],asymptot:[4,6,29],async_wait:[],athbf:[],atla:26,atoi:[],atom:[0,26],attempt:[0,4,6,7,8,10,26,28],attend:22,attent:[0,20,26],attr:28,attract:[0,10,26],attribut:[0,3,4,9,13,14,26,28,29],attributeerror:[3,4,13,14,28],audi:[0,26],audio:[3,4],august:[15,16,26],aurelien:[0,15,22,25,26],austfjel:[6,21],author:[0,1,3,4,10,14,23,26],authour:26,auto:[6,9,10,23],autocor:23,autocorrelation_tim:23,autocorrelform:23,autocovari:23,autoencod:[4,19,26],autoencond:19,autograd:[19,26],autograph:[3,4,14],autom:[0,19,25,26],automac:20,automag:26,automat:[0,1,2,3,4,11,16,19,20,26],automobil:3,autonom:4,avail:[0,1,4,6,10,11,15,19,20,21,22,24,25,26,29],averag:[0,1,3,6,9,10,13,14,23,24,26,27,28,29],avg:[],avoid:[0,3,4,5,6,9,11,13,14,20,26,27,29],avx2:[],avx:[],awai:[2,3,6,27,28],awar:[2,10],award:[24,26],ax:[0,1,2,3,4,6,7,8,9,10,11,12,13,14,20,21,26,27,29],axes3d:[2,6,13,21],axes_grid1:6,axessubplot:27,axhlin:8,axi:[0,1,2,3,4,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],axiom:[5,28,29],axlabel:[0,26],axvlin:[4,8],axvspan:4,b1:8,b2:8,b3:8,b:[0,1,2,3,4,5,6,8,9,10,12,13,14,15,16,23,24,26,27,28,29],b_0:0,b_1:[0,2,12,13],b_2:[0,13],b_5:13,b_:[0,1,20],b_group:9,b_i:[0,1,2,12,26],b_ia_:[0,26],b_ia_i:0,b_index:9,b_is_vec:2,b_j:[1,12],b_k:[0,1,12,13],b_m:12,b_meta:2,b_score:9,b_valu:9,bachelor:[22,24],back:[0,3,4,5,6,8,9,10,15,20,23,26,27,28],backbon:20,backend:[1,4],backend_specific_transl:[3,4,14],background:[25,26,27],backpropag:1,backtrack:9,backup:20,backward:[1,2,4,12,20],backward_pass:2,bad:[6,27,28],badli:23,bag:[9,19,26],bag_clf:10,baggin:26,baggingboot:10,baggingclassifi:10,baggingtre:10,balanc:[6,29],band:20,bandwidth:20,bar:[0,6,11,15,16,21,26],barber:25,bare:[4,10],base:[0,1,3,4,5,7,8,9,10,14,19,23,24,25,26,27,28],basi:[5,7,8,10,11,12,13,20,27,28],basic:[2,6,8,12,13,14,19,21,23,26],batch:[3,4,11,12,13,14],batch_shap:4,batch_siz:[1,3,4],batchnorm:4,bay:7,bayesian:[5,19,25,26,28,29],bbupeevmxtg:[],bc298b802fe2:[],beam:[26,29],becattini:24,becaus:[0,1,2,3,4,5,6,8,9,12,13,14,26,28,29],beccatini:26,becom:[0,1,2,5,6,7,9,12,13,23,26,27,28,29],been:[0,1,2,3,4,5,6,11,12,13,19,20,21,26,27,28,29],befor:[0,1,2,3,4,5,6,7,8,12,13,14,20,21,23,26,27,28,29],beforehand:[0,23,26],begin:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,21,23,24,26,27,28,29],behav:[1,6,13,29],behavior:[0,1,13,26],behaviour:12,behind:[0,1,6,8,13,26],behnoosh:[],being:[0,1,2,3,4,5,7,8,10,11,12,13,17,23,26,27,28,29],believ:[9,20],belong:[7,8,9,13,14],below:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],benchmark:10,bendik:[],benefici:[1,13],benefit:[0,1,4,11,13,15,19,26],bengio:[1,22,25,26,27],benign:[1,7,9],bennosh:[],besid:[4,5,28],bessel:[5,27,28,29],best:[0,1,2,3,4,5,6,7,8,9,10,12,13,21,24,26,27,29],beta:[0,1,3,5,6,7,10,11,13,16,17,18,21,26,27],beta_0:[0,1,3,5,6,7,13,26,27,28],beta_0x_:[0,26,27],beta_1:[0,1,3,5,6,7,10,13,26,27,28],beta_1x_0:[0,26],beta_1x_1:[0,7,26],beta_1x_2:[0,26],beta_1x_:[0,26,27],beta_1x_i:[7,13,27],beta_2:[0,3,13,26,27],beta_2x_0:[0,26],beta_2x_1:[0,26],beta_2x_2:[0,7,26],beta_2x_:[0,26,27],beta_2x_i:27,beta_3:3,beta_3x_i:27,beta_4x_i:27,beta_:[0,3,6,7,13,26,27,28],beta_i:[0,3,5,17,26,27,28],beta_j:[0,5,6,13,18,21,26,27,28,29],beta_k:13,beta_linreg:13,beta_m:10,beta_mg_m:10,beta_n:3,beta_p:7,beta_px_p:7,betavalu:5,better:[0,1,2,3,4,6,9,10,11,12,13,26,27,28,29],between:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,17,21,23,26,27,28,29],beyond:[0,1,5,6,8,13,21,26,27],bf:[13,14,20,23],bgd:13,bia:[0,1,2,3,5,8,9,10,12,13,26,27,28],bias:[1,2,3,5,6,9,12,28],big:[0,1,2,5,6,14,18,26,28,29],bigger:[1,6,27,28],bigr:12,bike:9,bilbo:26,bilek:[],billion:[3,12,19],bin:[0,7,23,27],binari:[0,3,5,7,9,10,12,26,28,29],binarycrossentropi:4,bind:[0,27],binomi:[19,23,26],binsboot:[6,29],bioinformat:[0,26],biolog:[1,12],bios1100:[19,26],bird:[0,3,26],birth:26,bishop:[22,25,26],bit:[1,4,20,23,26],bitwis:[3,4,14,23],bivari:2,bk:[0,13,27],bla:[20,26],black:[8,9,14],bledso:26,blob:[21,26],block:[6,10,19,20,23,26,29],blockingavg:[],blockingstd:[],blockingvar:[],blocksiz:[],blocksizemax:[],blocksizemin:[],blogpost:4,blue:[0,3],bluntli:26,bm:27,bmatrix:[0,1,3,5,7,8,11,13,20,26,27,28],bmi:1,bodi:[0,1,4,12,26],bold:1,boldfac:[0,5,16,27,28],boldsymbol:[0,1,2,3,5,6,7,8,10,11,13,14,15,16,17,18,21,26],boltzmann:[12,19,26],book1:25,book:[21,25,26,28],boost:[1,9,19,26],boostrap:10,bootavg:[],bootstd:[],bootstrap:[1,13,19,21,26],bootvar:[],bootvec:[],borrow:26,boston_dataset:[0,27],bot:8,both:[0,1,4,5,6,8,9,10,13,14,15,16,19,20,21,23,24,26,27,28,29],bottl:7,bound:[0,8,12,27],boundari:[2,4,8,11,12],boundkei:[],box:[4,9],boxed_arg:2,boyd:[8,13],bracket:[4,23],brain:[1,7,12],branch:9,breast:[5,7,11,28,29],breviti:13,brew:[0,15,19,21,26],brg:8,brief:[21,27],briefli:[0,26],bring:[0,5,6,10,26,27,28],broad:[0,26],brought:[13,19,26],brownle:4,browser:26,brunton:[],brute:[3,5,11,27,28],bs:[8,9,10],buffer_s:4,bui:4,build:[0,4,5,6,10,20,23,26,28,29],built:[0,1,3,4,6,27,29],bunch:11,busi:[0,27],bx:[],bzl:[],c1:[8,11],c2:[8,11],c95af3df0cdd:[],c:[0,1,2,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,23,24,25,27,28,29],c_0:23,c_1:12,c_2:12,c_3:12,c_4:12,c_:[0,8,9,10,13,23,27],c_i:[12,13],c_k:23,ca:[1,26],cabc613b8702:[],cach:10,cal:[0,8,10,12,13,15,16,26],calcul:[0,1,2,4,5,6,8,9,10,11,12,13,14,15,16,18,20,23,26,28,29],california:27,call:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,24,26,27,28,29],callabl:[],callback:[],calor:[0,27],cambridg:[13,25],can:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,25,27,28],cancel:[0,13,26,27],cancellation_manag:[],cancer:[5,10,28,29],cancerpd:7,candid:[8,9,10],cannot:[0,1,4,5,6,7,8,9,22,23,26,27,28],canopi:[0,15,19,21,26],canva:[21,26],cap:[5,28,29],capabl:[0,1,8,13,19,26],capac:[2,24],capita:[0,27],caption:21,captur:[4,11,12],captured_input:[],car:[3,4],card:[0,7,26],cardin:1,care:11,carefulli:13,carlo:[0,6,19,23,25,26,29],carri:[2,6,7,21,29],cart:10,carvalho:[],casella:25,cast:1,cat:[3,4],categor:[0,1,3,9,11,26],categori:[0,1,3,7,10,12,14,26,27],categorical_crossentropi:[1,3],caus:[0,5,6,23,26,27,28,29],causal:0,causat:[0,26],cax:1,cb:[6,26],cbar:1,cbook:[],cc:[0,1,5,13,26,27,28,29],ccc:[5,12,28],cd_fast:6,cdf:23,cdot:[0,2,6,12,13,14,20,23,26,29],celebr:13,cell:[0,2,3,4,6,9,10,13,14,15,21,26,28],center:[0,1,6,7,8,9,11,14,21,23,26,28,29],centr:[],central:[0,3,5,6,8,16,20,26,27,28],centroid:[14,23],centroid_differ:14,centuri:3,certain:[0,3,6,7,9,23,26,27,29],certainti:29,cg:13,cha:[0,27],chain:[0,1,13,19,23,26],chanc:[1,5,13,23,28,29],chang:[0,1,2,3,4,5,6,8,9,11,12,13,14,20,21,23,26,27,28,29],channel:3,chapter3:[0,21],chapter:[0,6,10,11,18,20,21,25,26,27,28,29],charact:[0,3,5,8,26,27],character:[8,9,10,12,23],characterist:[0,1,3,10,13,26],charg:[0,26],charl:[0,27],chase:4,chat:26,chd:7,chddata:7,cheap:[5,27],cheaper:[1,13],check:[0,1,3,4,5,6,11,13,20,26,27],checkmark:3,checkpoint:[3,4,14],checkpoint_dir:4,checkpoint_prefix:4,chemic:[],chen:10,chiaramont:2,choic:[0,1,2,3,4,6,9,12,13,14,20,26,27,29],choleski:[5,20,27],choos:[2,3,6,9,10,11,13,14,21,29],chosen:[0,1,2,6,8,9,10,13,16,23,26,29],chosen_datapoint:1,christian:25,christoph:[22,25,26],cifar10:3,cifar:3,cin:[],circ:[1,12],circl:[0,8,12,27],circuit:3,circumfer:9,circumv:[1,5,13,27],ckpt:4,clariti:23,class_nam:[3,9],class_val:9,class_valu:9,class_weight:[],classic:[7,9,13],classif:[0,3,5,6,7,8,11,12,19,21,25,26,27,28,29],classifi:[0,1,4,7,9,10,11,26],classificaton:1,classifii:10,clean:1,clear:[1,5,10,12,13],clearli:[0,3,5,6,7,8,18,23,27,28,29],clever:[1,10],clf3:0,clf:[0,6,8,9,10,26,27],clf_lasso:6,clf_ridg:[6,26],clip:[3,23],clone:24,close:[0,1,2,4,6,8,9,11,12,13,14,21,23,25,26,29],closer:[3,5,13,27],closest:[8,11,13,14],closur:[19,26],cloud:[19,26],cluster:[0,1,4,6,11,19,26,29],cluster_label:14,cm:[1,2,3,6,8,9,13,21],cmap:[0,1,2,3,4,6,8,9,10,21,26],cmap_arg:6,cmath:[],cmb:22,cmd:9,cmu:27,cn_:23,cnn:12,cnn_kera:3,cntk:[19,26],co:[0,2,3,6,9,13,26,29],code:[3,4,6,7,8,15,16,17,19,20,21,23,25],coef0:8,coef:[0,26],coef_:[0,5,6,8,9,13,26,27,28],coeff:5,coeffici:[0,3,5,6,7,8,9,13,15,16,20,26,27,28,29],coerc:[0,6,26,29],coin:[10,23],coin_toss:10,col:[0,11,26,27],colab:[19,26],cold:9,colinear:[0,27],collabor:21,collaps:8,collect:[0,2,6,10,11,19,23,25,26,29],collinear:[5,27,28],color:[0,3,4,6,8,9,10,21,23],color_channel:3,color_cod:6,colorbar:[1,6,21],colsample_bytre:10,colsaobject:10,colspec:[0,26],column:[0,1,2,5,6,7,8,9,11,12,17,20,26,27,28,29],columntransform:9,com:[3,4,6,14,19,21,25,26,28],combin:[1,2,5,6,7,10,23,28,29],come:[0,1,3,4,5,12,13,14,15,26,27,28,29],command:[0,1,26],comment:[0,4,5,6,15,16,21,26,27],commerci:[0,15,19,21,26],commod:[0,26],common:[0,1,3,5,6,7,9,11,13,14,21,23,26,27,29],commonli:[0,1,4,6,7,9,13,14,27,29],commun:[0,12,21],commut:3,commutatitav:3,compact:[0,1,3,5,6,7,9,11,12,13,14,26,27,28,29],compair:0,compar:[0,3,4,5,6,11,13,15,16,20,21,26,27,28,29],comparison:[2,4,13],compat:[3,4,7,14],compet:[0,26],competit:10,compil:[0,1,3,4,13,15,19,20,26],complet:[0,2,3,4,9,12,26],completenn:12,complex:[1,5,8,9,11,12,13,15,16,26,29],complic:[0,1,9,13,26,29],compoment:27,compon:[0,1,3,4,5,6,7,9,14,16,19,26,27,28,29],components_:11,compos:[9,12,13,14,19,26],compphys:[0,6,17,19,21,22,24,25,26,27],compress:[0,26,27],compris:6,compromis:[5,27],compulsori:[19,26],comput:[0,1,2,3,4,5,6,7,8,10,11,12,13,15,16,17,19,20,21,22,23,25,26,27,28,29],computation:[0,3,6,9,13,23,26],computationalscienceuio:26,computerlab:21,con:[],concat:26,concaten:[2,4,6,14],concav:[1,9,13,27],concentr:[0,10,27],concept:[0,2,19,26,27],conceptu:[12,13],concern:[0,1,4,7,26],concic:26,conclud:[0,5,13,28,29],conclus:1,cond:[2,3,4,14],conda:[0,1,15,19,21,26],condis:27,condit:[0,2,4,5,6,8,9,11,13,23,26,27],conduct:[19,26],condwav:2,coneqp:[],confid:[0,5,6,7,8,18,21,26,27,28],config:[3,4,14],config_pb2:[],configur:3,confirm:[5,12,28,29],confus:[5,6,7,10,20,27,28,29],confusion_matrix:9,congruenti:23,conjug:[4,8],conjugaci:13,conjunct:3,connect:[0,1,3,4,9,11,12,13,20,26,27],consequ:[5,6,8,10,12,13,27,28,29],conserv:[5,14,27],consid:[0,1,2,3,5,6,7,8,9,10,12,13,16,20,21,23,26,27,28,29],consider:[0,1,5,13,26,27,28,29],consist:[0,1,2,3,4,6,12,13,21,23,27,29],constant:[0,2,3,4,5,6,8,12,13,14,23,26,27,28,29],constitu:[0,26],constitut:[2,6,29],constrain:[1,3,5,7,11,28],constraint:[5,6,8,13,17,27,28,29],construct:[0,1,2,3,5,6,7,8,9,10,11,20,23,26,27,28,29],contact:[0,26],contain:[0,2,3,4,5,6,7,8,9,11,12,13,14,17,20,21,23,25,26,27,28,29],contemporari:26,content:[1,19,20,26],context:[6,10,13,21,29],contigu:20,continu:[0,1,2,3,4,5,6,7,8,9,10,12,13,17,18,20,21,23,26,27,28,29],contour:[9,10,13],contourf:[8,9,10],contrast:[1,4,9,10,12],contribut:[0,3,5,13,23,26,27,28],contributor:[0,21,26],control:[0,1,3,9,13,15,19,26],control_flow:[3,4,14],conv2d:[3,4],conv2dtranspos:4,conv:[3,4],conv_general_dilated_patch:[3,4,14],convei:26,conveni:[0,5,6,12,13,20,21,26,28,29],convent:[12,27],converg:[1,2,4,5,6,7,8,11,13,14,27,28],convergencewarn:[1,6,7,8,11],convert:[0,1,3,4,5,9,11,13,14,20,26,27],convert_phas:[3,4,14],converter_error_data_pb2:[3,4,14],converttomatrix:4,convex:[4,5,7,27],convinc:13,convolut:[1,4,19,26],cool:[4,9],coolwarm:[6,21],coordin:[5,12,14,27,28],coorel:[0,27],copi:[0,1,2,14,27],copyright:[3,4,14,29],core:[2,3,4,10,13,14,26],corel:26,coronari:7,corr:[0,5,7,11,27],correalt:[11,19],correct:[0,1,2,3,4,5,7,13,20,23,26,27,28,29],correctli:[1,2,6,7,10],correl:[0,1,3,5,6,7,10,12,13,19,23,26,28,29],correlation_matrix:[0,5,7,11,27],correspond:[0,3,5,6,8,9,11,12,15,16,17,19,20,21,23,26,27,28,29],cortex:12,cosin:[3,6,29],cost:[0,2,3,5,6,7,8,9,12,13,16,17,21,26],cost_deep_grad:2,cost_funct:2,cost_function_deep:2,cost_function_deep_grad:2,cost_function_grad:2,cost_grad:2,cost_sum:2,costol:13,could:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,17,20,21,23,26,27,28,29],coulomb:[0,26],count:[0,9,22,23,24,26],countor:13,coupl:[4,5,6,28,29],cours:[0,1,3,5,11,21,22,24,27,29],courvil:[22,25,26,27],cout:[],cov:[5,6,11,20,23,26,27,28,29],cov_xi:[5,11,27],cov_xx:[5,11,27],cov_yi:[5,11,27],covari:[0,7,19,20,26,28],covariance_matrix:[5,11,14,27],cover:[0,5,17,19,24,25,27,28],covert:[0,26],covxi:23,covxx:23,covxz:23,covyi:23,covyz:23,covzz:23,cpj4mh7m9no:[],cpu:[1,3,4,14],cpu_feature_guard:[],craft:3,crawford:26,creat:[1,2,3,4,5,6,9,10,11,12,13,19,21,26,28],create_biases_and_weight:1,create_convolutional_neural_network_kera:3,create_neural_network_kera:1,create_x:[5,11,27],credit:[0,7,24,26],crim:[0,27],crime:[0,27],criteria:[0,4,9,10,14,23,26],criterion:[9,10,13],critic:[6,21,26,27,28],critiqu:21,cross:[0,1,3,7,9,10,13,19,23,26,27,28],cross_entropi:4,cross_val_scor:[6,29],cross_valid:[7,10],crossvalid:[6,29],crucial:[1,23],cs231:3,cs:[22,24],csr_matrix:[20,26],cstdlib:[],csv:[0,4,6,7,9,26,29],ctnk:1,ctx:[],cubic:0,cumbersom:[5,28,29],cumsum:[10,11,26],cumul:[7,10,23],cumulative_heads_ratio:10,cup:[5,28,29],current:[1,2,3,4,6,13,14,21,25],curs:[0,27],curv:[6,7,10,12,21],curvatur:13,custom:[6,14,21],custom_cmap2:[9,10],custom_cmap:[9,10],cutpoint:9,cv:[6,7,10,29],cvxbook:13,cvxopt:[5,8,27],cyber:[],cycl:[1,3,4,12,14],cyclotron:[27,28],d1:[],d2:[],d2_g_t:2,d3:[],d670a873ab0c:[],d985fb40c43d:[],d:[1,2,3,4,5,6,7,8,9,10,11,13,14,20,23,24,26,27,28,29],d_f:13,d_g_t:2,d_net_out:2,da:3,dagger:[5,20,27],dai:[1,9,19],dalen:[],damp:3,daniel:[24,26],darget:9,darkr:23,dat:[0,26],dat_id:[0,6,7,9,26,27,29],data1:14,data2:14,data3:14,data4:14,data:[2,4,5,8,10,12,13,14,17,18,20,25,29],data_handl:[],data_id:[0,6,7,9,26,27,29],data_indic:1,data_modul:27,data_panda:26,data_path:[0,6,7,9,26,27,29],data_url:27,databas:1,databookuw:[],datafil:[0,6,7,9,21,26,27,29],datafram:[0,4,5,7,9,11,26,27],datapoint:[1,5,6,7,11,13,27,29],dataset:[0,4,6,7,8,9,10,11,13,14,15,16,21,26,27,29],datatyp:4,date:[15,16,17,18,21,26,27,28,29],daughter:10,david:25,davison:29,dbh:1,dbo:1,dcomposit:20,ddot:2,dead:1,deadlin:[16,17,18,22],deal:[0,1,3,5,6,8,11,13,14,17,18,20,21,23,26,27,28],dealt:0,debt:7,debug:[0,5,6,28,29],decad:[0,3,26],decai:[0,13,23,26],decemb:[22,24,26],decent:10,decid:[0,2,3,5,6,9,21,28,29],decim:[0,26],decis:[0,1,8,11,19,25,26],decision_funct:8,decision_tre:9,decisiontreeclassifi:[9,10],decisiontreeregressor:[0,9,10],declar:[0,4,20,26],decompos:[5,6,20,27],decomposit:[0,6,12,21,26,28],decompost:[5,27],deconvolut:3,decor:[0,26],decorrel:[10,13],decreas:[1,2,4,5,6,10,11,13,28,29],deduc:[0,26],deep:[3,7,12,13,19,22,25,26,27],deep_neural_network:2,deep_param:2,deep_tree_clf1:9,deep_tree_clf2:9,deep_tree_clf:[9,10],deepen:[5,19,26],deeper:[0,3,4,26],deeplearningbook:25,deer:3,def:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,21,23,26,27,28,29],def_covari:23,def_funct:[],default_tim:4,defect:[5,27],defici:[5,27],defin:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,27,28,29],definit:[1,2,5,6,7,8,10,11,12,13,17,20,23,27,28,29],defint:23,defun:[],defvjp:2,degre:[3,5,6,8,9,10,11,15,16,17,21,23,26,28,29],del:1,delet:6,delimit:4,deliv:[22,26],delta:[0,2,3,6,8,12,13,14,26],delta_0:3,delta_1:3,delta_2:3,delta_3:3,delta_4:3,delta_5:3,delta_:[1,20],delta_h:[0,1,26],delta_j:[3,12],delta_k:12,delta_l:[1,3],delta_momentum:13,delta_n:[0,3,26],delug:19,delv:[0,26],demand:13,demonstr:[0,3,5,6,7,11,12,19,26,27,28,29],den:4,denomin:[1,5,28,29],denot:[1,2,6,7,13,23],dens:[1,3,4],dense_1:[],densiti:[0,2,6,23,26,29],depart:[24,26,27,28,29],depend:[0,1,2,4,5,6,7,8,11,12,13,15,19,20,21,23,26,27,28,29],depict:23,deploy:[0,15,19,21,26],deprec:[2,6,13,21,26,27],deprecate_nonkeyword_argu:[0,26],depth:[0,3,9,10,20,29],deriv:[0,1,2,6,7,8,10,11,13,16,17,19,21,26],derivati:13,derivative_fn:13,descend:[5,9,11,27,28],descent:[0,1,3,7,8,12,26,27],descr:27,describ:[0,2,4,5,6,8,10,11,12,13,18,20,21,26,28,29],descript:[0,8,9,26],design:[0,1,3,4,5,6,7,10,11,12,13,15,16,17,18,21,26,28,29],designmatrix:[0,26],desir:[0,2,4,5,13,14,26,27],despit:[1,12],destroi:20,det:[5,20,27],detail:[0,6,11,13,14,20,21,27],detect:[3,8,12],determin:[0,2,3,4,5,6,8,9,10,11,12,13,16,20,23,26,27,28,29],determinist:[7,13,23],dev:1,develop:[0,3,5,8,10,11,12,19,20,21,26,27,28,29],deviat:[0,1,2,4,5,6,21,23,26,27,28,29],device_nam:[],devicearrai:[3,4,14],devis:12,df1:26,df:[4,8,11,13,26],di:[0,27],diag:[5,8,27,28],diagnost:[1,10],diagon:[0,5,7,13,18,20,21,23,26,27,28],diagonaliz:[5,27],diagram:10,diagsvd:6,dice:[6,23,29],dict:[6,8],dict_kei:27,dictionari:[0,27],did:[0,1,5,6,7,10,11,14,21,26,28,29],die:1,diff1:2,diff2:2,diff:2,diff_ag:2,diffeent:8,differ:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,15,17,19,20,21,23,25,26,27,28,29],differenti:[0,3,16,19,20,26,27],differential_oper:[2,13],difficult:[0,1,6,10,13,21,23,26,29],difficulti:[0,1,13,26],diffonedim:2,digit:[0,1,3,4,6,21,22,24,26],dilemma:13,dilut:1,dim:[4,11,14,20],dimens:[0,1,2,3,4,5,8,9,11,14,16,17,20,26,27,28],dimension:[0,4,5,6,9,11,13,14,19,20,26,28,29],dimensionless:[0,3,26],diment:20,dimnsion:4,diod:3,direct:[0,1,2,4,11,12,13,14,26,27],directli:[1,4,5,6,23,27],directori:9,disabl:[3,4,14],disadvantag:[0,26],disappear:[3,6,29],disc_loss:4,disc_tap:4,discard:[6,11,29],disciplin:[0,3,12,26],disclaim:23,discord:26,discourag:[13,27],discov:[0,26],discover:[5,28],discret:[1,3,5,7,13,28,29],discrimin:[4,7,10,11],discriminator_loss:4,discriminator_loss_list:4,discriminator_model:4,discriminator_optim:4,discuss:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,25,26,27,28,29],diseas:7,disguis:[6,27,28],disord:[1,7],displai:[0,1,3,4,5,6,7,8,9,10,11,12,14,21,23,26,27,29],displaystyl:[0,5,17,26,27,28],displot:27,disregard:[0,26],dissimilar:[11,14],dist:14,distanc:[0,8,9,11,14,23,27],distance_list:9,distinct:[3,7,8,9,10,14],distinctli:8,distinguish:[0,4,7,8,23,26],distplot:[0,27],distribut:[0,1,4,6,7,10,11,13,14,15,16,18,19,20,21,26,27],distrubut:[0,15,19,21,26],div:[],dive:[0,8,20,26],diverg:[1,13],divid:[0,1,3,5,6,7,8,9,11,12,23,26,27,28,29],divis:[6,8,9,13,20,23,29],dna:7,dnn1:4,dnn2_gru2:4,dnn:[0,1,2,4,12,26],dnn_kera:1,dnn_model:1,dnn_numpi:1,dnn_scikit:[0,1,26],doamin:[28,29],doc:[0,6,17,19,21,22,24,25,26,27],document:[4,7,11,13,27],doe:[0,1,2,3,4,5,6,8,10,11,12,13,16,20,21,23,26,29],doesn:[3,9,12],dog:[1,3,4],domain:[5,8,13,21,28,29],domin:[0,26],don:[0,1,3,5,6,8,11,13,15,19,21,26,27],done:[0,2,3,4,5,6,9,10,11,13,17,20,21,26,27,28,29],dot:[0,2,3,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],doubl:[3,4,20,26],doubli:1,down:[0,3,6,9,11,12,13,26],download:[0,1,3,5,6,20,21,25,26],downsampl:3,dozen:1,dq:[6,29],drag:13,dramat:11,drastic:4,draw:[4,6,10,13,29],drawback:[0,1,3,13,27],drawn:[1,4,6,7,11,23,26,29],drive:[3,4],driven:3,drop:[0,1,5,6,11,13,23,26,27,28,29],dropna:[0,6,26,29],dropout:4,ds:[],dt:[2,3,13,23],dtype:[0,1,2,3,4,14,20,26,27],dualiti:6,dub:[0,26],due:[1,2,5,6,8,10,12,13,24,26,27,29],dummi:[0,27],dure:[0,1,3,4,8,9,11,17,19,21,26,29],dwell:[0,27],dwh:1,dwo:1,dx:[2,3,8,23],dx_1:23,dx_1p:[6,29],dx_2p:[6,29],dx_mp:[6,29],dx_n:23,dxp:[6,29],dy:[1,8,23],dynam:4,dysth:[],dz:8,e7cu:[],e:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,18,23,24,26,27,28,29],e_:[0,2,26],each:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,19,20,22,23,24,26,27,28,29],eager:29,eapprox:[0,26],earli:[1,13],earlier:[0,5,7,8,9,11,12,13,26,27],earthexplor:[6,21],eas:[6,9,14,29],easi:[0,5,6,7,8,9,10,11,12,13,18,19,20,26,27,28,29],easier:[5,6,8,9,13,23,26,27,29],easiest:13,easili:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,21,26,27,28,29],eastern:[24,26],ebind:[0,26],eblock:9,econometr:26,economi:5,ecosystem:[19,26],ect:22,edg:3,edgecolor:[6,29],edu:[13,27],educ:[0,21,26,27],eface79dac2c:[],eff:23,effect:[1,4,10,13,23],effic:1,effici:[0,3,10,13,19,20,23,26],efron:[6,29],egrad:13,eig:[5,11,13,20,23,26,27],eigen:23,eigenpair:[5,11,27],eigenvalu:[0,5,8,11,13,20,26,27,28],eigenvector:[5,11,13,17,27,28],eight:[20,26],eigval:[20,23,26],eigvalu:[11,13],eigvec:[20,23,26],eigvector:[11,13],eispack:[20,26],either:[1,5,6,7,8,9,10,11,13,15,16,21,23,26,27,28,29],ekstrom:[],elabor:23,elarn:3,electr:[0,3,12,26],electron:26,electur:[],eleg:11,element:[1,2,3,4,5,6,7,8,11,12,13,17,18,19,20,21,22,25,27,28,29],elementari:[10,13,20],elementwis:[3,13],elementwise_grad:[2,13],elessar:26,elif:[2,14],elim:20,elimin:[3,8],els:[1,2,3,4,7,9,12,13,20],elu:1,elus:[0,26],email:[22,24,26],embed:[0,11,27],embodi:[6,21,29],emit:23,emner:25,emphas:[0,10,19,26],emphasi:[0,19,25,26],empir:[1,11,23],emploi:[0,1,5,6,11,13,21,23,26,27,28,29],employ:[0,26,27],empti:[6,10,29],emul:12,en:[19,25],enabl:11,enable_check:[3,4,14],enbodi:[6,29],encapsul:[3,4,14],encod:[0,3,5,9,11,14,17,26,27,28],encompass:[0,21,23,26],encount:[0,1,5,6,7,13,23,26,27,29],encourag:21,end:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,20,21,23,24,26,27,28,29],end_box:[2,13],end_nod:[2,13],end_valu:[2,13],endl:[],endpoint:[3,6],energi:[0,4,6,27,29],enet_coordinate_desc:6,enforc:12,eng:25,engin:[0,1,3,4,19,26],english:21,enorm:3,enough:[0,6,13,26,29],ensembl:[1,9],ensur:[0,1,2,3,5,6,11,13,23,27,28,29],ensure_initi:[],entail:26,enter:[5,6,17,27,28],enthought:[0,15,19,21,26],entir:[1,3,7,9,19,23,26],entiti:[9,12,20,26],entri:[0,5,8,11,12,20,26,27,28,29],entropi:[1,3,7,10,13,26],enumer:[0,1,2,3,4,6,8,26,27,28],env:[0,1,2,3,4,6,7,8,11,13,14,23,26,27,28],environ:[2,19,21],eo:[0,6,26,29],eol:[0,26],eosfit:[0,26],epoch:[0,1,3,4,12,13,26],epsilon:[0,5,6,7,13,21,26,27,28,29],epsilon_0:[0,26],epsilon_1:[0,26],epsilon_2:[0,26],epsilon_:[0,26],epsilon_i:[0,26,27],eq:[3,13,14,20,23],eqnarrai:[3,5,6,28,29],equal:[0,1,2,3,4,5,6,8,9,11,12,13,14,15,20,21,23,26,27,28,29],equat:[1,3,4,5,6,7,8,9,10,11,13,14,16,17,18,20,21,23,29],equilibrium:[2,12],equiv:[3,13,20,23],equival:[0,1,5,7,8,11,13,19,20,26,27,28,29],erf:23,eriador:26,eridg:26,err:[0,10],err_:[6,29],err_sqr:2,errat:13,errno:9,erron:2,error:[1,2,4,5,6,7,9,11,12,13,15,16,17,18,19,20,21,23,28],error_estimate_corr_tim:23,error_handl:[],error_hidden:1,error_output:1,escap:13,especi:[1,3,9,12,13,21],essenti:[0,5,6,9,10,12,14,21,23,27],establish:[0,6,10,11,15,21,26],estim:[0,1,5,6,7,10,11,13,19,23,26,27],estimated_mse_fold:[6,29],estimated_mse_kfold:[6,29],estimated_mse_sklearn:[6,29],et:[0,2,4,15,16,17,25,26,27,28,29],eta0:[8,13],eta:[0,1,3,8,12,13,26],eta_:13,eta_t:13,eta_v:[0,1,3,26],etc:[0,1,3,5,7,8,9,11,12,13,14,15,19,20,21,23,27],ethic:[19,26,27],etsim:29,euclidean:[0,14,27],evalu:[0,2,3,4,5,6,9,13,21,23,26,27,28,29],evaluationform:21,evaluationgrad:21,evalut:13,even:[0,1,3,4,5,6,8,9,10,11,12,13,14,19,20,23,26,27,28,29],evenli:4,event:[3,4,5,7,10,14,23,28,29],eventu:[0,5,6,11,12,13,21,24,27,28,29],everi:[0,1,2,3,4,5,6,9,10,11,12,13,14,19,23,24,26,27,28,29],everyth:[4,12],everywher:[4,13],evolv:[0,26],exact:[0,2,5,11,12,13,20,23,26,27],exactli:[0,3,4,6,12,19,21,27,29],exam:26,examin:[6,29],exampl:[5,11,12,13,15,16,19,20,21,23,25],exce:[1,12,13],excel:[0,1,4,5,10,26,27],except:[3,4,6,8,9,14,20],excess:[0,26],excit:[0,26],exclud:[1,6,12,21,27,28,29],exclus:[0,1,3,6,23,26,29],execut:[2,5,13,27],executing_eagerli:[],exemplifi:13,exercic:[24,26],exercis:[5,19,21,22,24,28],exercisesweek35:17,exhaust:[6,29],exhibit:[0,5,6,8,26,27,29],exist:[0,1,2,3,5,6,7,8,9,13,18,20,21,26,27,28,29],exit:[5,20,27],exp:[0,1,2,5,6,7,8,10,11,12,13,15,16,21,23,26,27,28,29],exp_term:1,expand:[5,7,11,13,27],expans:[0,3,5,8,10,12,13,26,27],expect:[0,1,5,6,7,11,12,13,15,16,19,21,26,27],expectation_value_of_h_wrt_p:23,expens:[6,10,13],experi:[0,1,6,8,13,19,21,26,27,29],experiment:[0,3,4,6,9,14,23,26,29],experimental_get_tracing_count:[],expert:[1,9],explain:[0,6,9,10,11,13,21,26],explained_variance_ratio_:11,explanatori:[0,26],explicit:[0,3,6,13,20,21,26,27,28],explicitli:[0,4,26],explod:1,exploit:[0,3,12,13,26],explor:[1,4,6,8,13,19,21],expon:1,exponenti:[0,1,5,6,10,13,21,23,26,28,29],export_graphviz:9,export_text:9,exporttext:9,expos:[19,26],express:[0,2,3,5,6,7,10,12,13,16,20,21,23,28,29],exptmean:23,exptvari:23,extend:[0,2,7,11,13,17,19,26],extens:[0,12,15,19,26],extent:[0,1,6,25,26,29],extern:[3,6,9],extra:[1,3,5,24,26,27],extract:[0,3,5,6,7,8,11,13,20,26,27],extrapol:[0,26],extrem:[0,1,4,5,6,7,8,9,13,16,20,27,28,29],extremum:13,extrins:11,ey:[0,5,6,13,14,20,26,28],f11:[0,26],f12:[0,26],f13:[0,26],f1:13,f1_grad:13,f1d:13,f2:13,f2_grad_x1:13,f2_grad_x1_analyt:13,f2_grad_x2:13,f2_grad_x2_analyt:13,f3:13,f3_grad:13,f3_grad_analyt:13,f3ckh:[],f401:[3,4,14],f4:13,f4_grad:13,f4_grad_analyt:13,f5:13,f5_grad:13,f6:13,f6_for:13,f6_for_grad:13,f6_grad_analyt:13,f6_while:13,f6_while_grad:13,f6d7a289d493:[],f7:13,f7_grad:13,f7_grad_analyt:13,f8:13,f8_grad:13,f9:[0,13,26],f9_altern:13,f9_alternative_grad:13,f9_grad:13,f:[0,1,2,3,4,5,6,7,8,10,12,13,14,16,18,20,23,24,26,27,28,29],f_0:[3,10],f_1:[10,13],f_2:[12,13],f_3:12,f_:10,f_d:23,f_grad:13,f_grad_analyt:13,f_i:[0,6,12,16,29],f_m:[3,10],f_n:3,f_raw:2,f_vec:2,f_wrap:2,face:13,facecolor:[6,8,23,29],facil:[0,15,19,26,29],facilit:12,fact:[0,1,3,5,9,11,12,13,26,27,28,29],factor:[0,1,3,5,6,9,10,11,13,20,23,26,27,28,29],factori:13,fade:6,fafab0:[9,10],fahimeh:[24,26],fail:[0,6,7,8,11,13,24,26,29],failur:7,fairli:[1,2,23],fake:4,fake_loss:4,fake_output:4,fall:[8,9,22],fals:[0,1,2,3,4,5,6,7,9,10,14,20,21,26,27,28,29],famili:[0,7,8,23],familiar:[0,3,5,6,8,15,19,20,21,23,26,28,29],famou:[6,12],far:[0,3,4,5,6,8,11,12,13,14,26,27],fashion:[0,9,10,26],fast:[1,3,6,10,12,13,19,23,26,29],faster:[1,11,13],fastest:[13,20],favor:7,favorit:23,fc:3,fdf8a5d7c717:[],fdfcc778e1f8:[],fe5b9d300cc0:[],featur:[0,1,3,5,6,7,8,10,11,12,13,18,19,21,23,26,28,29],feature_nam:[0,1,7,9,27],feautur:9,fed:1,feed:[0,2,3,11,19,26],feed_forward:1,feed_forward_out:1,feed_forward_train:1,feedback:[4,26],feeddorward:4,feedforward:[1,4,12],feel:[0,5,6,11,13,15,16,19,21,24,26,28],feet:[0,27],fei:26,felt:21,fetch:[6,21,27],fetch_california_h:27,fetch_openml:27,few:[1,3,4,5,9,23,26],fewer:[0,9,11,26],ffnn:[1,12],fft2:[3,4,14],fft:[3,4,14],fft_batching_rul:[3,4,14],fft_p:[3,4,14],fftfreq:[3,4,14],fftn:[3,4,14],fftshift:[3,4,14],fgakzw6k1qq:[],field:[0,3,6,12,19,26],fifth:[0,6,15,16,21,26],fig:[0,1,2,3,4,6,7,12,13,14,21,26],fig_id:[0,6,7,9,26,27,29],figaxi:23,figsiz:[0,1,2,3,4,6,7,8,9,10,26,27,29],figur:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,19,21,26,27,28,29],figure_id:[0,6,7,9,26,27,29],figurefil:[0,6,7,9,26,27,29],file:[0,2,3,4,5,6,7,9,13,14,21,26,27,28,29],file_prefix:4,filenam:[26,27],filenotfounderror:9,fileout:[],filepath_or_buff:[0,26],fill:[5,9,27],filter:[3,4],filtered_flat_arg:[],filtered_tb:[],financ:[0,26],find:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,19,21,23,26,27,28],find_top_boxed_arg:[],fine:[0,14,15,26],finish:2,finit:[3,5,6,12,13,17,23,27,28,29],first:[0,1,2,3,5,6,7,8,9,10,11,13,14,15,16,17,20,21,23,24,25,29],firsteigvector:11,fit:[1,3,4,5,6,7,8,9,11,12,13,15,16,17,21,23,27,29],fit_beta:[6,27,28],fit_intercept:[0,5,6,27,28,29],fit_mod:9,fit_transform:[0,6,8,9,11,29],fiti:[0,26],five:[0,9,17,21,26,27],fix:[0,3,4,6,10,11,12,13,21,26,29],fixedformatt:6,fixedloc:6,fkkt:[],flag:4,flake8:[3,4,14],flat:[12,13],flatbuff:[],flatten:[1,3,4,5,20],flexibl:[1,6,8,10,12,26,27,29],flip:[24,26],float32:[4,9],float64:[4,20,26,27],flop:[5,20,27],flow:[1,4,12],fluctuat:[5,28],fly:11,flyvbjerg:[],fm:[0,26],fma:[],fmax:3,fmesh:13,fn:7,focu:[0,3,4,5,6,15,19,25,26,27,28,29],focus:[1,6,7,20,27,28],fold:[6,9,21],folder:[0,1,4,6,13,21,26,29],follow:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],font:[0,7,23,26],fontdict:23,fontsiz:[1,6,8,9,10,23],fontweight:1,footprint:3,foral:[8,27],forc:[0,5,6,10,11,26,27,28],forcast:4,forecast:[4,12],forelesningsvideo:[],forest:[0,1,9,19,26],forget:11,form:[0,3,4,5,6,7,8,9,11,12,13,17,19,20,21,23,26,27,28,29],formal:[3,4,14,23],format:[0,1,2,3,4,6,7,8,9,10,11,19,23,25,27,28,29],format_data:4,formatstrformatt:[6,13,21],formul:[4,6,11,14],formula:[3,13,23],forth:[4,12],fortran2003:[19,26],fortran2008:21,fortran90:23,fortran:[0,15,19,20,26],fortun:[0,11,27],forward:[0,3,6,19,20,26,29],forward_backward:[],forward_compatibility_horizon:[3,4,14],found:[1,2,4,5,6,12,13,21,26,27,28,29],foundat:[19,26],four:[4,5,6,8,12,20,21,22,24,26,28],fourier:[0,26],fourierdef1:3,fourierdef2:3,fourierseriessign:3,fourth:[12,26,27],fp:7,fr:[],frac:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,17,20,21,23,26,27,28,29],fractal:9,fraction:9,frame:[7,26],framework:[1,8,10,23],frank:[5,11],frankefunct:[5,6,11,21,27],fredli:[24,26],free:[0,6,11,13,15,16,19,20,21,23,24,25,26],freecodecamp:19,freedom:[5,28],freeli:[0,21,26],frequenc:[3,6,7,23,29],frequent:[0,8,9,13,26],frequentist:19,fresh:10,frida:[],fridai:[16,24,26],friedman:[6,18,21,22,25,26,27],friendli:4,frodo:26,frog:3,from:[0,1,2,3,4,6,7,8,9,11,13,14,16,17,18,19,20,21,23,24,25,26],from_cod:9,from_logit:[3,4],from_tensor_slic:4,fromnumer:2,front:[0,4,5,15,16,26,27,28],fstream:[],fulfil:[2,5,12,27],full:[0,1,3,5,7,9,10,13,23,26,27],full_matric:[5,27],fulli:[3,6,12,22,23,29],fun:[2,13,19,26],fun_nam:2,func:[0,2,26,27],functionali:11,functool:[3,4,14],fundament:[0,6,19,26,29],funtion:2,further:[2,7,9,26],furthermor:[0,3,5,6,7,11,12,13,17,19,21,26,27,28,29],futur:[0,4,8,9,26,27],futurewarn:[0,26,27],fx:[],fy:[15,21,22,24,25,26],fys4155:21,fys5419:[25,26],fys5429:[25,26],g0:2,g42mrgv128v34gnnhxwk9nrc0000gp:[],g:[0,1,2,3,4,6,8,9,10,11,13,14,23,26,28,29],g_0:2,g_1:[2,10],g_2:[2,10],g_:[2,9,10],g_analyt:2,g_dnn_ag:2,g_euler:2,g_i:2,g_m:[3,10],g_n:3,g_re:2,g_t:2,g_t_d2t:2,g_t_d2x:2,g_t_dt:2,g_t_hessian:2,g_t_hessian_func:2,g_t_jacobian:2,g_t_jacobian_func:2,g_trial:2,g_trial_deep:2,g_vec:2,gain:[1,5,7,9,10,13,27,28,29],galleri:[0,26],game:4,gamge:26,gamma1:8,gamma2:8,gamma:[0,2,8,9,10,11,13,26],gamma_0:10,gamma_1:10,gamma_1x:10,gamma_:[0,26],gamma_i:[0,8,23,26],gamma_j:13,gamma_k:13,gamma_m:10,gamma_x:[0,26],gap:[6,8],gate:[4,12],gather:[0,1,12,27],gaug:12,gaussbacksub:20,gaussian:[4,5,6,8,14,23,28,29],gaussian_point:14,gaussian_rbf:8,gave:13,gbc:[22,26],gca:[2,6,8,13,21],gd:1,gd_clf:10,gdclassiffiercgain:10,gdclassiffierconfus:10,gdclassiffierroc:10,gdm:13,gdregress:10,ge:[1,5,7,23,27],gemv:[],gen:[],gen_loss:4,gen_tap:4,gender:[0,26],genener:4,gener:[0,1,2,3,5,6,8,10,11,12,13,14,15,16,17,20,21,23,25,27,28,29],generallay:12,generate_and_save_imag:4,generate_imag:4,generate_latent_point:4,generate_simple_clustering_dataset:14,generated_imag:4,generator_loss:4,generator_loss_list:4,generator_model:4,generator_optim:4,genexpr:[],genom:19,geodes:11,geometr:[0,13,26],geometri:[5,28,29],georg:25,geotif:[6,21],geq:[2,5,8,9,13,27,28],geron:[0,15,22,25,26],get:[0,1,2,3,4,5,6,7,9,10,11,13,15,19,20,21,23,24,26,27,28,29],get_dummi:9,get_loc:[],get_next_color:[],get_paramet:2,get_split:9,get_yaxi:8,get_yticklabel:6,getattr:2,getsolutionslic:[],getval:2,gg:[],gibb:[19,26],gif:4,gini:10,gini_index:9,ginvers:13,git:[0,15,19,26],giter:13,github:[0,3,4,6,14,15,17,19,21,22,24,25,26,27],gitlab:[0,15,19,21,26],give:[0,1,2,3,5,6,7,8,9,10,12,13,14,17,19,21,23,26,27,28,29],given:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],global:[6,7,13,21],glorot:1,gmail:[],gnew:13,go:[0,1,3,5,6,8,9,11,12,13,21,26,27,28,29],goal:[0,7,9,26],goe:[0,1,2,5,6,13,14,18,20,26,27,28,29],golden:13,gone:[5,27,28],gong:1,good:[1,3,4,5,6,9,10,11,13,17,19,23,25,27,28,29],goodfellow:[4,22,25,26,27,28],googl:[1,3,4,14,19,26],got:[1,6,21],gov:[6,21],gp:25,gpu:[1,13,19,26],grad:[2,13],grad_analyt:13,grade:22,gradient:[0,3,4,7,8,9,12,19,26,27],gradient_desc:[],gradientboostingclassifi:10,gradientboostingregressor:10,gradients_of_discrimin:4,gradients_of_gener:4,gradienttap:4,gradual:[1,14],grai:[4,6,21],graph:[1,9,11,12,13],graph_debug_info_pb2:[],graph_from_dot_data:9,graph_funct:[],graphic:[0,1,9,26],grasp:[0,26],gray_r:[1,3],grayscal:3,great:[5,13],greater:[1,7,23],greatli:13,greedi:9,green:[0,3,9,23],grei:4,grid:[1,3,6,7,8,12,23,27,28,29],grossli:13,ground:[0,26],group:[0,6,7,9,14,19,21,22,24,26,29],groupbi:[0,26],grow:[1,3,9,10],growth:[0,26],gru:4,guarante:[0,4,13,23,26,27],guess:[1,4,10,13,14],guestrin:10,guid:1,guilherm:[],h1:2,h21:[],h:[0,1,5,6,8,13,16,18,21,23,24,25,26,27],h_1:[2,13],h_2:[2,13],h_:[0,13,26],h_m:10,ha:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,20,21,23,26,27,28,29],haa:[24,26],habit:[0,27],had:[0,1,6,7,13,26,29],hadamard:[1,12,13],half:[1,8,9],halv:10,hand:[0,1,2,3,5,11,12,13,15,19,20,21,22,23,24,25,26,27,28,29],handi:[3,21],handl:[0,1,2,5,9,11,19,28],handle_unknown:9,handsid:12,handwrit:12,handwritten:[1,5,26],handwrittennot:26,happen:[1,2,3,4,5,6,10,13,23,27],hard:[1,7,8,10,13],hardcopi:[19,26],harder:[0,1,27],harmon:3,hasn:1,hassl:[0,15,19,26],hast:[19,26],hasti:[0,6,15,16,17,18,21,22,25,26,27,28,29],hat:[0,1,5,6,7,9,10,11,12,13,17,18,20,21,27,28,29],have:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,27,28,29],haven:1,hdtta7prrfi:[],he:7,head:[0,4,10,23,27],header:[0,26,27],heads_proba:10,health:[0,27],hear:[0,13,26],heart:[0,7,26],heatmap:[0,1,3,7,26,27],heavili:[0,26],heavisid:1,height:[1,3,6,27,28],held:13,help:[0,1,4,12,13,16,21,26],helper:[4,14],henc:[0,5,6,8,9,10,12,13,18,21,26,27,28,29],henrik:[24,26],her:7,here:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,26,27,28,29],hereaft:[0,8,12,26],hermitian:20,hessenberg:20,hessian:[0,2,5,13,16,28],heterogen:[9,10],hfft:[3,4,14],hi:7,hidden:[1,3,4,12],hidden_bia:1,hidden_bias_gradi:1,hidden_layer_s:[0,1,26],hidden_neuron:4,hidden_weight:1,hidden_weights_gradi:1,hierarch:[5,27,28],high:[0,1,2,3,4,5,6,9,10,11,13,14,19,20,21,26,27,29],higher:[0,1,3,5,6,8,13,21,26,27,28,29],highest:[1,2],highli:[0,3,4,10,18,19,20,25,26,27,28,29],highwai:[0,27],hing:8,hint:[13,16,27],hip:19,hire:[0,26],hist:[4,6,7,23,29],histogram:[0,6,7,23,27],histor:[7,11],histori:[3,4,12],histplot:27,hit:[],hitherto:[5,28],hjorth:[24,26,27,28,29],hm6ay5ds6o0:[],hobbi:23,hoc:[5,27],hoff:25,hold:[1,3,6,13,14,29],holder:[0,26],holomorphic_grad:[2,13],home:[0,27],homepag:[21,26],homework:[6,13],homogen:[1,3,9,10,13],honchar:2,hopefulli:[0,11,23,26],horizont:11,hors:[3,7],hot:[1,9],hour:[1,19,22,23,24,26,29],house_pric:27,how:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,26,27,28,29],howev:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,20,21,23,26,27,28,29],hs:[],hspace:[0,4,8,10,23,26],hstack:[1,27],htf:[22,26],html:[0,7,11,17,19,21,22,24,25,26,27],http:[0,3,4,6,7,11,13,14,17,19,20,21,22,24,25,26,27,28],huang:[0,26],huber:[0,26],huge:[1,3,4,19],human:[0,1,3,6,9,12,26,27,28],humid:9,hundr:1,hungri:1,hybrid:22,hydrogen:[0,26],hyperbol:[1,4,12],hyperparam:8,hyperparamet:[3,4,5,6,9,13,17,26,27,28],hyperplan:11,i0:[0,26],i1:[0,6,8,12,26,27,28],i2:[0,8,12,26],i3:[0,12,26],i5:[0,26],i:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,24,26,27,28,29],i_1:[5,6,28,29],i_2:[5,6,28,29],i_:13,ian:25,ic:1,id:[7,13],ida:[24,26],idea:[0,1,2,3,4,6,9,10,12,13,20,21,26,27,28,29],ideal:[0,2,6,8,13,23,26,27,29],idem:[6,29],ident:[5,6,12,13,17,20,21,27],identical:[28,29],identifi:[0,1,7,9,11,12,13,14,26,27],idum:[],ieor:23,ifft2:[3,4,14],ifft:[3,4,14],ifftn:[3,4,14],ifftshift:[3,4,14],ifi:25,ifs:[19,26],ignor:[0,1,3,9,27],ihfft:[3,4,14],ii:[20,23],iii:[20,26],ij:[0,1,3,6,8,12,14,16,18,20,21,23,26,27,28],ik:[0,20,26,27],illustr:[5,7,10,12,13,14,19,26,29],im:6,imag:[1,3,4,6,9,11,12,14,25,26],image_at_epoch_:4,image_batch:4,image_height:3,image_path:[0,6,7,9,26,27,29],image_width:3,imageio:[6,21],imagenet:26,images_from_seed_imag:4,imagin:1,immedi:[0,3,4,6,15,19,26],implement:[0,2,3,4,5,6,8,9,10,11,12,13,14,17,21,23,26,27,28],impli:[3,5,6,7,13,20,27,28,29],implicit:3,implicitli:[11,23],importantli:3,importerror:[3,4,14],impos:[0,6,11,12,26],imposs:[0,5,26,27],impress:[0,12,26],improv:[0,4,5,9,10,11,13,21,27],impur:9,imread:[6,21],imshow:[1,3,4,6,21],in1:[],in2:[],in3050:[25,26],in3310:26,in4080:[25,26],in4300:[25,26],in4310:25,in5400:3,in5550:25,in_out_neuron:4,inaccur:13,inacio:[],inact:12,inadequ:[0,26],inch:[6,27,28],includ:[0,1,2,3,4,5,6,7,11,12,15,16,17,19,21,23,24,25,26,27,28,29],include_bia:[6,9,29],inclus:17,incom:12,incorrect:1,incoveni:8,increas:[0,1,3,4,5,6,7,8,9,11,12,13,21,23,26,27,28,29],increasingli:23,ind:6,inde:[0,2,4,5,6,13,26,27],indefinit:4,indent:[],indentationerror:[],independ:[0,5,6,7,8,12,13,23,26,27],index:[0,1,3,4,10,14,19,20,23,25,26,27],index_col:[0,26],index_of:[],indic:[0,1,3,4,5,6,9,10,11,13,16,21,26,27],indispens:[6,29],individu:[1,6,7,10,12,23,26,27,29],indu:[0,27],indx1:2,indx2:2,indx3:2,indx:20,ineffici:[3,13],inequ:[8,13],inequaltii:[],inertia:13,inf1000:[19,26],inf1100:[19,26],inf1100l:[19,26],inf1110:[19,26],inf3000:26,inf4490:[],inf5860:[],infeas:9,infer:[0,1,4,6,25,26,29],infer_nrow:[0,26],inferenc:1,infil:[0,6,7,9,26,29],infin:[5,6,7,11,18,27,28,29],infinit:3,infinitesim:23,influenc:[6,10,29],influenti:1,info:26,inform:[0,1,3,4,6,9,11,12,13,14,20,21,25,26,29],infti:[3,6,13,23,29],ingeni:13,ingrad:2,ingredi:[0,9,26],ingular:[],inher:[6,29],inherit:[20,26],initi:[0,1,2,3,4,6,10,13,14,20,23,26,29],initial_epoch:[],initialis:[],initialize_root:[],inject:14,inlin:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],inner:[0,13,27],innov:[],inp:4,inplac:13,input:[0,1,3,4,5,6,7,8,9,10,12,13,14,15,21,23,26,27,28,29],input_dim:1,input_shap:[3,4],inputs:1,inputs_shuffl:[0,1,27],insert:[3,5,6,8,10,23,27,28,29],insid:[0,4,7,27],insight:[0,1,5,19,26,27,28,29],insist:[6,13,27,28],inspir:[0,1,12,26],instabl:2,instal:[0,1,5,6,9,15],instanc:[0,1,2,4,6,9,11,13,26,27,29],instanti:10,instead:[0,1,2,3,4,5,6,8,9,11,13,14,20,21,23,26,27,29],institut:1,instruct:[0,1,15,16,26],int32:10,int64:27,int64index:26,int_0:23,int_:[3,6,23,29],int_a:23,intak:[0,27],integ:[1,2,13,14,20,23,26],integer_vector:1,integr:[3,6,23,26,29],intellectu:26,intellig:[0,14,25,26],intend:[10,26],intens:1,intention:14,interact:[0,6,9,12,19,21,26],intercept:[0,6,8,11,13,16,21,26,28,29],intercept_:[0,6,8,9,13,26,27,28],interchang:[5,12,20,28,29],interconnect:1,interest:[0,1,2,3,4,5,6,7,8,9,12,19,21,23,26,27,28,29],interfac:[0,1,20,27],interior:[0,9,26],intermedi:[20,27],intern:[1,10,12],interpol:[1,3,4,6,12,21],interpr:[5,27,28],interpret:[0,1,3,4,6,9,10,12,13,14,17,20,21,23],interv:[0,3,5,6,7,13,18,21,23,26,27,28],intial:13,intract:[0,4,27],intrins:[3,11,20,23,26],intro:[19,25,26],introduc:[0,1,5,6,8,10,12,17,20,21,23,26,29],introduct:[1,2,4,13,17,25,27],introductori:[0,4,20,25,26,27],intuit:[0,5,6,8,12,13,21,26,28,29],inv:[0,5,13,26,27,28],invalid:[1,8],invalu:[0,13,15,19,26],invari:1,invd:[5,28],inver:8,invers:[0,3,6,13,15,16,17,21,26,27],invers_period:[],inverse_transform:8,invert:[0,5,7,10,13,26,28],invh:13,invok:[0,8,27],involv:[0,2,6,7,11,12,27,29],io:[0,17,19,21,22,24,25,26,27],iomanip:[],iostream:[],ip:[0,8,23,26],ipca:11,ipykernel_16145:[],ipykernel_16213:[],ipykernel_16261:[],ipykernel_16293:[],ipykernel_19350:[],ipykernel_20528:[],ipykernel_20614:[],ipykernel_20669:[],ipykernel_20702:[],ipykernel_41811:[],ipykernel_42331:[],ipykernel_42376:[],ipykernel_42449:[],ipykernel_42456:[],ipykernel_42530:[],ipykernel_42541:[],ipykernel_42553:[],ipykernel_42573:[],ipykernel_42580:[],ipykernel_42586:[],ipykernel_47411:[],ipykernel_47448:[],ipykernel_47647:[],ipykernel_47724:[],ipykernel_47735:[],ipykernel_65914:[],ipykernel_7604:1,ipykernel_7650:6,ipykernel_7714:13,ipykernel_7740:21,ipykernel_7752:26,ipykernel_7768:29,ipykernel_9414:[],ipykernel_94478:[],ipykernel_94529:[],ipykernel_94582:[],ipykernel_96694:[],ipynb:[19,26],ipython:[0,5,7,9,11,14,15,19,21,26,27],iq:[6,29],irfft2:[3,4,14],irfft:[3,4,14],irfftn:[3,4,14],iri:[8,9],irreduc:[6,29],irrelev:[5,27],irrespect:[0,26],is_integ:[],isbox:[2,13],iscomplexobj:2,isinst:[],isn:[5,28,29],isnul:[0,27],isomap:11,isotop:[26,29],issu:[1,3,4,9,14,20,27,28],it_arrai:13,item:[0,13,26],items:[20,26],iter:[1,2,4,6,7,8,11,13,14,23,29],itr:[],its:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,18,19,20,21,23,26,28,29],itself:[5,6,12,21,23,27,28,29],ix:[],j1:20,j:[0,1,2,3,4,5,6,8,9,11,12,13,14,16,17,18,20,21,23,25,26,27,28,29],j_:6,j_lasso_sk:6,j_ridge_sk:6,j_sk:6,jackknavg:[],jackknif:[6,19,26,29],jackknstd:[],jackknvar:[],jackknvec:[],jacobian:[2,13],jacobian_shap:2,jakobsen:[24,26],jargon:[],jason:4,jax:[3,4,14,19,26],jaxlib:[3,4,14],jensen:[24,26,27,28,29],jerom:[18,21,25],jhfgq77fpq:[],ji:[12,20],jit:13,jj:[0,5,6,26,28,29],jk:[0,1,6,12,20,26],jl:[0,26],jm:20,jnp:[3,4,13,14],joao:[],joaogca:[],job:[2,8,10],join:[0,4,6,7,9,26,27,29],joint:[4,5,28,29],journal:[],judg:13,judgement:[6,21],julia:[19,20,21],jump:[21,23],junk:4,jupit:26,jupyt:[0,15,19,21,25,26,29],just:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,19,21,23,26,27,28,29],justif:[0,26],justifi:[3,10],jy:[],jydg2xzka5:[],k0:7,k1:7,k:[0,1,3,5,6,7,8,9,10,11,12,13,14,19,20,21,23,24,26,27,28],kaggl:[6,21],kappa_d:23,karim:[],karl:[24,26],karlsen:[],karush:8,kate:26,katrin:[24,26],keep:[0,1,4,5,6,11,13,14,20,21,26,27,28,29],keepdim:[1,6,10,20,29],kei:[0,1,3,6,12,27],kept:[4,6,14,29],kera:[0,4,19,21,26],kernel:[0,1,3,19,26,27],kernel_regular:[1,3],kernel_s:4,kernelpca:11,kev:[0,26],kevin:[25,26],keyboardinterrupt:2,keyerror:[],keyword:[6,13,20,21,26],kfold:[6,29],kg:1,ki:20,kick:[1,13],kiener:2,kilomet:[6,27,28],kind:[0,2,3,4,8,12,13,14,26,27],kj:[6,12,20,27,28],kjm:[19,26],kkt:8,kktsolver:[],kl0m3:[],kl:23,km:[12,26],kmean:14,kmeanspoint:14,kn_k:14,know:[0,1,2,5,6,8,13,19,26,27,28],knowledg:[0,19,26],known:[1,3,4,5,6,7,8,9,12,20,21,23,25,27,28,29],kondev:[0,26],kp:23,kpca:11,kroneck:14,kuhn:8,kutz:[],kvalsund:[24,26],kwarg:[0,2,13,26],kwd:[0,26],kwown:[0,26],l0:7,l1:[0,1,3,7,26],l1_l2:[1,3],l1regl:5,l1regls_mosek2:[],l1regls_mosek:[],l2:[1,3],l:[0,1,2,3,5,6,7,8,10,11,12,13,20,21,23,26,27],l_1:7,l_2:[7,13],l_:20,l_j:12,la:13,la_i:12,la_k:12,lab:[19,21,26],label:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,19,20,21,23,26,27,28,29],label_prob:[],labelencod:[7,10],labels:[6,8,9],labels_shuffl:[0,1,27],laboratori:[22,27,28],lack:[0,26],lagari:2,lagrang:[8,11],lambda:[0,1,2,3,5,6,7,8,10,12,13,17,18,21,23,26,27,28,29],lambda_0:11,lambda_1:[5,8,11,27],lambda_2:[8,11],lambda_:11,lambda_i:[8,11],lambda_iy_i:8,lambda_jy_iy_j:8,lambda_k:8,lambda_n:[5,8,27],lamda:1,lamdbda:[],land:[0,8,27],landmark:8,landscap:13,langl:[0,6,11,23,26,27],languag:[0,1,4,8,15,19,20,21,25,26],lapack:[20,26],laplac:[5,28,29],laptop:19,larg:[0,1,2,4,5,6,8,9,10,11,13,15,19,20,21,23,25,26,27,29],larger:[0,3,5,6,8,10,11,13,17,23,26,27,28,29],largest:[4,8,11],lasso:[0,7,19,26],lasso_sk:6,last:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17,18,20,21,23,24,26,28],latent:4,latent_dim:4,latent_point:4,latent_space_value_rang:4,later:[0,1,4,6,7,8,12,13,14,15,19,21,26],latest:[4,19],latest_checkpoint:4,latex:26,latter:[0,3,6,7,8,11,13,16,17,20,21,23,26,27,28,29],lattic:12,law:[0,26],lax:[3,4,14],layer:[0,4,13,14,26],lbfg:[7,9,10,11],lcc:[5,6,28,29],lda:11,ldot:[0,6,11,21,26,29],le:[5,7,10,13,17,23,27,28],lead:[0,1,3,5,6,7,8,9,10,11,12,13,16,17,20,23,26,27,28,29],leaf:9,leaki:1,leakyrelu:4,lear:13,learn:[3,4,5,6,7,8,9,10,12,18,20,22,24,25],learnabl:3,learner:10,learnig:26,learning_r:[8,10],learning_rate_init:[0,1,26],learning_schedul:13,learnt:21,least:[0,2,7,8,10,11,15,16,17,19,20,23,29],leat:13,leav:[0,1,3,5,6,9,11,26,28,29],lectur:[0,1,5,10,11,12,13,15,17,19,20,21,22,24,25],lecturenot:[0,17,19,21,25,26],lecturenovember11:[],lecturenovember12:[],lecturenovember19:[],lecturenovember25:[],lecturenovember26:[],lecturenovember4:[],lecturenovember5:[],lectureoctober14:[],lectureoctober15:[],lectureoctober1:[],lectureoctober21:[],lectureoctober22:[],lectureoctober28:[],lectureoctober29:[],lectureoctober7:[],lectureoctober8:[],lectureseptember10:[],lectureseptember16firstpart:[],lectureseptember16secondpart:[],lectureseptember17:[],lectureseptember23:[],lectureseptember24:[],lectureseptember2:[],lectureseptember30:[],lectureseptember3:[],lectureseptember9:[],lecturethursdayaugust26:[],lecturethursdayaugust27:[],left:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,16,17,20,21,23,26,27,28,29],leftarrow:[8,12],legend:[0,2,3,4,5,6,7,8,9,10,13,26,27,28,29],len:[0,1,2,3,4,5,6,8,9,10,11,12,20,26,27,28,29],len_index:[0,26],length:[0,1,2,3,4,8,9,13,16,19,26,27],length_of_sequ:4,leq:[0,5,7,8,13,14,17,23,26,27,28],less:[0,1,3,4,5,6,8,9,13,19,23,26,27,28,29],lessen:1,let:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,23,26,27,28,29],letter:[0,16,20,23,26,27],level:[0,1,5,6,9,19,20,21,22,24,26,27,29],li:[8,11,26],lib:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],liblinear:[8,10],librari:[0,1,2,3,4,5,6,9,10,11,15,20,21,23,25,27,28],licens:[0,1,3,4,14,15,19,21,26,29],lie:[0,6,11,23,27,29],life:[0,1,8,12,26],lifetim:13,like:[0,1,2,3,4,5,6,7,9,10,11,12,13,15,16,19,20,21,23,26,27,28,29],likelihood:[0,1,5,9,26,27],lim_:23,lima:[24,26],limit:[0,5,6,7,8,11,12,20,21,26,27],lin_clf:8,lin_model:[0,27],lin_reg:9,linalg:[0,2,3,4,5,6,8,11,13,14,20,23,26,27,28],line1:8,line2:8,line2d:13,line3:8,line:[0,2,3,4,6,8,9,10,11,13,14,15,21,26,27,28,29],linear:[1,3,5,6,7,9,10,11,12,17,18,19,21,23,29],linear_model:[0,5,6,7,8,9,10,11,13,26,27,28,29],linear_regress:[6,29],linear_util:[3,4,14],linearli:[5,27],linearloc:[6,13,21],linearregress:[0,6,7,9,26,27,28,29],linearsvc:8,lineat:28,liner:[1,3],linerar:10,linewidth:[0,2,4,6,8,9,10,21,29],link:[0,4,9,12,19,21,24,26],linlag:[5,28],linpack:[20,26],linreg:[0,26],linspac:[0,2,3,4,6,8,9,10,13,15,16,20,23,26,27,28,29],linu:4,linuek:[],linux:[0,1,15,19,21,26],liquid:[0,26],list:[0,1,2,3,4,9,19,21,26,27],listcomp:2,listedcolormap:[9,10],lite:[3,4,14],lite_const:[3,4,14],literatur:[1,7,14,25,29],littl:[1,3,9,12],live:8,ll:[0,23,26,27],llc:[3,4,14],lle:[0,27],lloyd:[4,14],lmb:[0,2,5,6,28,29],lmbd:[0,1,3,26],lmbd_val:[0,1,3,26],lmbda:13,ln:[1,13],lo:[],load:[0,1,4,6,7,9,10,21,27],load_boston:[0,27],load_breast_canc:[1,7,9,10,11],load_data:[3,4],load_digit:[1,3],load_iri:[8,9],loc:[0,3,6,7,8,9,10,26,29],local:[0,1,2,3,7,12,13,26,27],locat:[2,3,8],lock:[],log10:[0,5,6,28,29],log1p:[],log:[0,1,2,3,4,5,6,7,9,10,11,13,14,20,21,26,27,28,29],log_:[0,26],log_clf:10,logarithm:[0,5,7,20,26,28,29],logbook:21,logic:[0,1,9,26],logist:[0,1,2,8,9,10,11,12,13,19,27],logistic_predict:[],logisticregress:[7,9,10,11],logit:7,logreg:[7,9,10,11],logspac:[0,1,3,5,6,26,28,29],longer:[2,3,8,10,14,20,23,26],loocv:[6,29],look:[0,1,2,3,4,5,6,7,8,9,10,11,13,18,20,21,23,26,27,28,29],lookup:[3,4,14],loop:[1,4,6,10,12,14,19,20,26,29],lose:1,loss:[0,1,3,4,5,6,7,8,10,11,13,20,21,26,28,29],loss_fil:4,lossfil:4,lost:4,lot:[0,1,4,6,27,29],low:[0,6,9,10,11,21,26,27,29],lower:[0,1,3,6,9,10,16,20,27],lowercas:[20,26],lowest:[9,13,23],lr:[1,3,4,10],lstat:[0,27],lstm:4,lstm_2layer:4,lstsq:[0,26,27],lt:[6,29],lu:[0,3,4,5,14,26,27],lubksb:20,luckili:2,ludcmp:20,lux:20,lvert:1,lw:[0,26],m:[0,1,2,3,5,6,8,9,10,11,12,13,20,22,23,24,25,26,27,28,29],m_1:14,m_:[9,12],m_h:[0,26],m_k:14,m_l:12,m_n:[0,26],m_p:[0,26],m_t:13,ma:11,mac:[3,4,14],machin:[1,3,4,5,6,7,9,10,11,12,14,15,20,22,25,27,28,29],machinelearn:[0,6,17,19,21,22,24,25,26,27],machinelearningmurphi:[],mackai:25,made:[0,1,3,4,5,6,7,9,11,12,18,21,26,27],mae:[0,26],magic:4,magnitud:[1,6,7,13,27,28],mai:[0,1,2,3,5,6,7,8,9,11,12,13,18,19,20,21,23,27,28,29],mail:[22,24],main:[0,1,3,4,5,6,7,9,20,21,25,26,27],mainli:[0,5,6,7,9,26,27,28,29],maintain:[6,27,29],major:[1,6,9,10,13,20,26,29],make:[1,2,3,4,5,6,7,8,11,12,13,19,20,21,23,25,28,29],make_axes_locat:6,make_moon:[8,9,10],make_pipelin:[0,6,10,27,29],make_vjp:[2,13],makedir:[0,6,7,9,26,27,29],makeplot:[0,26],malcondit:20,malign:[1,7,9],mammographi:[5,28,29],manag:[0,2,3,15,19,21,26],mandatori:[24,26],mani:[0,1,3,4,5,6,7,8,9,11,13,14,19,20,21,23,25,26,27,28,29],manifold:11,manner:3,manual:[6,27,28],map:[0,1,2,3,4,6,7,8,11,12,14,21,23,26],margin:[0,5,8,26],mari:[],marit:[0,26],mark:26,marker:[0,7,20,26,27],markov:[19,26],marsaglia:23,mask:[],mass:[0,1,5,13,27],massag:[0,26],masses2016:[0,26],masses2016ol:[0,26],masses2016tre:0,masseval2016:[0,26],master:[6,21,22,24,26],mat1100:[19,26],mat1110:[19,26],mat1120:[19,26],mat3155:[],mat4155:[],mat:[19,26],match:[0,1,4,5,13,14,26,27],materi:[4,5,7,13,17,20,24],math:[3,7,12,13,20,23,25,26],mathbb:[0,4,5,6,7,8,11,12,13,14,17,18,20,21,23,26,27,28,29],mathbf:[0,5,6,7,8,13,18,20,21,26,27,28,29],mathcal:[1,5,6,7,13,21,28,29],matheemat:3,mathemat:[0,6,11,12,13,17,19,20,23,25,26,28],mathemati:26,mathrm:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,17,18,21,23,26,27,28,29],matmul:[1,2,5,28],matmul_adjoint_1:2,matmul_vjp_0:[],matmul_vjp_1:2,matnat:25,matplotlib:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,26,27,28,29],matplotlibdeprecationwarn:[6,13,21],matric:[0,1,3,4,6,7,8,11,13,16,17,19,27],matrix:[0,2,3,4,6,7,8,10,13,15,16,17,18,21,23,29],matshow:1,matter:[2,3,13,27],max:[0,1,2,3,4,9,10,12,13,24,26],max_depth:[0,9,10],max_diff1:2,max_diff2:2,max_diff:2,max_it:[0,1,7,8,11,13,26],max_iter:14,max_leaf_nod:10,max_queue_s:[],max_sampl:10,maxdegre:[0,6,10,27,29],maxdepth:10,maxim:[1,4,5,7,8,11,28,29],maximum:[0,1,2,3,5,7,8,9,10,13,14,26,27],maxpolydegre:[5,6,28,29],maxpooling2d:3,mbox:[5,6,18,21,27,28,29],mc:[],mccorduck:26,mcculloch:12,mcint:[],mcintsqr2:[],md:[11,21],mdoel:4,mdyt6uwokt0:[],mean:[1,2,3,4,5,6,7,9,10,11,12,13,14,15,16,17,18,19,20,21,23,29],mean_absolute_error:[0,26],mean_divisor:14,mean_i:23,mean_matrix:14,mean_squared_error:[0,4,6,7,10,26,27,29],mean_squared_log_error:[0,26],mean_vector:14,mean_x:23,meaning:[0,4,7,26],meansquarederror:[0,26],meant:[2,3,7,10,13],meantempvec:[],meanvec:[],measur:[0,1,2,5,6,9,11,12,14,21,23,26,27,28,29],mechan:[0,4,23,26],median:[0,26,27],medicin:12,medium:[4,8,13],medv:[0,27],meet:[0,24],mehta:[0,26,27,28],memori:[3,4,11,12,13,20,26],mention:[0,12,13,21,23,26],mere:[0,15,26],mersienn:[],meshgrid:[2,5,6,8,9,10,11,21,27],messag:[5,13],messi:2,met:[0,3,8,26,27],metadata:2,meteorolog:9,meter:[6,27,28],method:[0,1,2,3,4,5,7,8,11,12,14,16,17,18,19,20,23,25,26,27,28],metion:[6,21],metric:[0,1,3,4,6,7,9,10,14,15,16,26,27,29],metropoli:[19,26],mev:[0,23,26],mgd:13,mglearn:[19,26],mgrid:13,mhjensen:[1,2,3,4,6,7,8,11,14,27],mi:10,mia:[24,26],michigan:[26,27,28,29],microsoft:25,mid:1,midel:4,midnight:[16,17,18],midpoint:9,might:[0,1,2,4,6,9,13,27,28],mild:9,miller:[],millimet:[6,27,28],million:[0,26,27],mimic:12,min:[0,2,5,8,9,26],min_:[0,2,5,14,17,26,27,28],min_samples_leaf:9,mind:[0,6,13,21,26,27,28,29],mindboard:4,mine:[19,26],mini:[1,11,12,13],minibatch:[1,11,13],minibathc:13,miniforge3:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],minim:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,21,27,28,29],minima:[0,1,7,13,26],minimum:[0,1,2,6,8,9,11,13,27,29],minmaxscal:[0,27],minor:[6,13,21,23],minst:1,minu:7,mirror:9,misc:[6,21],misclassif:[8,9,10],misclassifi:[8,10],miser:0,mismatch:1,miss:[0,7,10,27],mistak:4,mit:25,mix:[1,2,26],mixtur:13,mk48pfcxgyk:[],mk:[9,20],mkdir:[0,6,7,9,26,27,29],ml:[0,1,10,13,20,21,27],mlab:23,mle:[5,7],mline:[],mlir:[],mlir_graph_optimization_pass:[],mlp:1,mlpclassifi:1,mlpregressor:[0,26],mm:20,mn:[12,23],mnist:[1,11],mod:23,mode:[22,24,26],model:[2,3,5,7,8,9,10,11,13,14,15,16,18,19,21,23,25,27,28,29],model_select:[0,1,3,5,6,7,9,10,11,26,27,28,29],modelanalyz:[3,4,14],moder:10,modern:[0,6,7,19,26,29],modif:[2,12,13],modifi:[0,1,3,5,7,8,10,12,13,26,27,28],modul:[0,3,4,7,11,14,20,26,28],modular:23,modulenotfounderror:[],modulo:23,moe:[11,27],moment:[5,6,13,23,28,29],monitor:13,monoton:[5,12,23,28,29],mont:[0,6,19,23,25,26,29],montecarlocycl:[],moor:[5,6,28],more:[0,1,2,4,5,7,8,9,10,11,12,13,14,18,19,23],moreov:[0,3,26],morten:[24,26,27,28,29],mortenimac:[],mosek:[],most:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,26,27,28,29],mostli:[1,11],motion:[0,13,26],motiv:[1,4],move:[0,4,5,6,7,9,12,13,14,21,23,26,27,28,29],mp4:[],mpl:[0,7,26],mpl_toolkit:[2,6,13,21],mplot3d:[2,6,13,21],mplregressor:1,mse:[0,4,5,6,9,10,15,16,17,21,26,27,28,29],mse_simpletre:10,mselassopredict:[5,28],mselassotrain:[5,28],mseownridgepredict:[6,28],msepredict:[5,28],mseridgepredict:[0,5,6,28],msetrain:[5,28],msg:[0,26,27],msle:[0,26],mt19937_64:[],mt:[7,12],mu0:23,mu1:23,mu2:23,mu:[0,6,11,13,23,26,29],mu_1:27,mu_:[6,23,27,28,29],mu_i:[6,27,28],mu_n:11,mu_x:23,much:[0,1,2,3,4,5,6,8,9,10,11,12,13,20,21,23,26,27,28,29],mul:[],multi:[0,1,3,7,19,26,27],multiclass:[1,7],multidimension:[11,12,26],multilay:1,multinomi:7,multipl:[2,4,5,6,7,12,13,23,27,29],multipli:[3,5,6,11,13,20,23,27,28],multiplum:8,multivari:[0,2,10,11,19,23,26],multivariate_norm:[11,14],murphi:[11,25,26,28],must:[0,1,2,5,6,8,10,12,13,14,23,26,27,29],mutat:7,mutual:[1,3,6,13,29],mx_:23,myenv:[0,1,2,3,4,6,7,8,11,13,14,26,27,28],myriad:[0,15,19,26],mz1:23,mz2:23,n1:20,n2:20,n:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],n_0:[12,23],n_:[1,2,3,8,12,23],n_b:[],n_boostrap:[6,10,29],n_bootstrap:[6,29],n_categori:[1,3],n_cluster:14,n_compon:11,n_epoch:13,n_estim:10,n_examples_to_gener:4,n_featur:1,n_filter:3,n_hidden:2,n_hidden_neuron:[0,1,26],n_i:23,n_input:[0,1,3,27],n_instanc:9,n_iter_i:[7,11],n_job:10,n_k:14,n_l:[12,23],n_layer:1,n_m:9,n_neuron:1,n_neurons_connect:3,n_neurons_layer1:1,n_neurons_layer2:1,n_point:14,n_sampl:[6,8,9,10,14,29],n_split:[6,29],n_step:4,n_t:2,n_x:2,nabla:[1,13],nabla_:[2,13],nabla_w:13,nag:13,naimi:[0,26],naiv:7,naive_kmean:14,najafi:[24,26],nall:[0,26],name:[0,1,3,4,5,6,7,8,9,10,12,13,14,15,19,20,21,23,24,26,27,28,29],nameerror:[6,10,15,21],namespac:[3,4,14],narrow:13,nary_f:[2,13],nary_op_arg:[2,13],nary_op_kwarg:[2,13],nary_oper:[2,13],nation:[1,5,27,28,29],nativ:[19,26],natur:[0,1,4,8,9,12,13,21,23,25,26],navier:12,nb:23,nb_:20,nbconvert:26,nboot:[],nd:14,ndarrai:[2,6],ndef:13,ndim:2,ne:[9,10,20,23,27],ne_xcl2ctm0:[],nearest:[1,3,6,11],nearli:13,neat:26,neccesari:[6,29],necess:2,necessari:[0,1,3,4,8,14,26],necessarili:[0,4,11,23,26],necesserali:[5,28,29],neck:7,need:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,23,27,28,29],neg:[0,1,3,5,6,7,10,13,20,23,26,28,29],neg_mean_squared_error:[6,29],neglect:23,neglig:23,neighbor:[3,6,11],neither:[4,13],neq:[13,14,23],nervou:12,nest:[2,9,12],nesterov:13,net:[2,4,12],netlib:[20,26],network:[0,9,13,19,25,27],neural:[0,13,19,25,27],neural_network:[0,1,2,26],neuralnetwork:1,neuron:[1,2,3,4,12],neutral:[0,26],neutron:[0,26],never:[1,4,6,9,23,29],new_box:[2,13],new_chang:13,new_hobbit:26,new_root:[2,13],new_trac:[2,13],new_tracing_count:[],newaxi:[0,3,6,9,29],newli:[0,26],newton:[1,7,8,13,23],next:[0,1,2,3,4,5,6,8,9,13,14,26,27,28],next_guess:13,next_input:4,nf8_grad:13,nfrom:13,ng:1,ngini:9,ni:14,nian:[],nice:[0,1,5,11,26,27,28,29],nichola:[],nicholaskarlsen1102:[],nimport:13,niter:13,nitric:[0,27],nlambda:[0,5,6,28,29],nlevel:[],nlp:25,nm:23,nm_n:[0,26],nmse:[6,29],nn:[2,5,6,12,20,28,29],nn_model:1,nnmin:2,node:[1,2,3,9,10,12],node_constructor:2,nois:[0,4,5,6,8,9,10,13,15,16,21,26,27,28,29],noise_dimens:4,noisi:[1,6,21,29],non:[0,1,3,5,6,7,9,10,11,12,13,14,20,21,23,26,27,28,29],noncommerci:29,none:[0,1,2,3,4,5,9,10,13,14,23,26,27,28,29],nonlinear:[3,6,8,9,11,12,29],nonneg:[6,9,13,29],nonparametr:6,nonsens:23,nonsingular:20,nonumb:[3,7,8,13,20],noqa:[3,4,14],nor:[1,4,13],norm:[0,1,5,6,8,11,13,17,26,27,28,29],normal:[3,4,5,6,7,8,9,10,11,12,13,15,16,18,19,20,21,23,26,27,28],normali:[20,26],normalize_kwarg:[],norvig:26,norwai:[6,21,26],notat:[0,2,5,6,13,14,23,26,27,28,29],note:[0,1,2,3,4,5,6,7,8,11,12,13,14,15,16,19,20,21,23,25,26,29],notebook:[0,1,3,9,15,19,21,26,29],notesexercise5week452022:26,noth:[1,2,5,8,12,14,23,27,28],notic:[4,5,12,13,20,23,26,28],notimplementederror:2,notion:3,notrace_primit:2,novel:[3,6,10],novemb:[1,22,24,26],now:[0,2,4,5,6,7,8,10,11,12,14,15,16,17,19,20,21,23,26,27],nowadai:[0,1,3,9,19,26],nox:[0,27],np:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,20,21,23,26,27,28,29],npr:2,nprint:13,nsampl:[6,9,29],nspin:[],nt:2,nthi:[0,26],ntrained_model:6,nu:23,nuclear:[5,27],nuclei:[0,23,26],nucleon:[0,26],nucleu:[0,26],num:4,num_allow_arg:[0,26],num_coordin:2,num_hidden_neuron:2,num_it:2,num_neuron:2,num_neurons_hidden:2,num_output:[],num_point:2,num_tre:10,num_valu:2,number:[1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,20,21,22,24,26,28,29],numberid:7,numberparamet:3,numer:[0,5,6,9,10,11,12,13,17,19,20,25,26,27,28,29],numpi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,21,23,27,28,29],numpy_box:[],numpy_rank_promot:[3,4,14],numpy_vjp:2,numpy_wrapp:2,nunmpi:[5,27],nvalu:9,nx:[2,13],nx_test:6,nx_train:6,nx_train_mean:6,ny:23,ny_pr:6,ny_train:6,ny_train_mean:6,o:[0,6,7,8,9,11,15,20,24,25,26,27],obei:[6,11,13,27,28],object:[0,1,2,4,6,8,10,13,20,26,27],objsens:[],obliqu:[5,27],observ:[0,1,3,5,6,7,8,9,10,11,12,13,14,23,28,29],obtain:[0,1,5,6,7,8,9,10,12,13,14,17,20,21,23,26,27,28,29],obviou:[5,6,11,23,27],obviouli:26,obvious:[0,4,5,6,20,26,28,29],oc:27,occupi:[0,27],occur:[0,6,8,9,20,23,26],octob:[22,24,26],od:0,odd:[0,3,7,26,27],odenum:2,odesi:2,oen:0,off:[1,3,4,5,9,13,23,28,29],offer:[6,11,19,20,22,24,26,29],offic:[24,26],offici:[22,26],ofil:[],ofstream:[],often:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,16,19,20,23,26,27,29],ofter:[20,26],oh6i_oscwpc:[],ol:[0,13,15,16,17,18,26,27],old:[1,5,10,13,28,29],ols_sk:6,ols_svd:6,olsbeta:[0,5,28],olxksel3p4a:[],omega:[2,3,6],omega_0:3,omit:[0,5,26,27,28,29],on_train_batch_begin:[],onc:[1,6,9,11,13,29],one:[0,1,3,4,5,6,7,8,9,10,11,13,14,19,20,21,23,24,26,27,28,29],oneapi:[],onednn:[],onehot:1,onehot_vector:1,onehotencod:9,ones:[0,2,5,6,8,9,10,11,13,20,21,26,27,28,29],ones_lik:4,onl:3,onli:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,20,21,23,26,27,28,29],onlin:[11,22],onto:[5,11,27],op_nam:[],opbdi:[],open:[0,1,4,6,7,9,15,19,21,22,24,26,29],oper:[0,1,3,4,5,6,10,11,12,13,14,15,19,23,26,27,28,29],operation:23,ophint:[3,4,14],oplu:23,opmiz:13,opportun:[0,26],oppos:[6,13],opposit:[1,5,8,27],opresolvertyp:[3,4,14],opt:[1,5,26,28],optim:[0,2,3,4,5,6,7,9,10,11,14,15,16,17,18,21,28,29],optimis:[1,3],optimizer_v2:[],option:[0,1,3,5,6,7,8,11,20,21,27,28,29],optionalxlacontext:[],optmiz:[1,8,13,27],oral:26,orang:0,order:[0,1,2,3,5,6,7,8,9,10,11,12,15,16,20,21,23,26,27,28,29],ordinari:[0,2,3,7,11,13,15,16,17,19,29],oreilli:25,org:[0,3,4,7,11,19,20,25,26],organ:[6,7,10,20,29],orient:[1,5,23,27,28],origin:[0,3,5,6,8,11,12,13,20,26,27,29],orthogn:[5,27],orthogon:[0,5,6,8,11,13,17,20,26,27,28],orthonorm:[5,27,28],os:[0,1,4,5,6,7,8,9,24,26,27,28,29],oscar:1,oscil:[3,13],oslo:[0,15,19,21,22,24,26,27,28,29],osx:[0,15,19,21,26],other:[0,1,2,3,5,6,7,8,10,13,14,15,19,21,22,23,24,25,28,29],otherwis:[0,1,4,7,13,20,26,27],ouput:[5,7,12,28,29],our:[1,2,3,6,7,8,9,10,12,14,15,16,17,18,19,20,21,23,28,29],ourmodel:0,ourselv:[0,5,6,8,11,13,26,27,28,29],out1:[],out2:[],out:[0,1,2,4,5,6,7,8,9,10,11,12,13,16,19,20,21,23,26,27,28,29],out_fil:9,outcom:[0,7,9,10,12,23,27],outdoor:9,outer:[6,12,13],outfil:4,outfilenam:[],outgrad:2,outlier:[0,8,26,27],outlin:[6,10,11,29],outlook:9,outperform:10,output:[0,1,3,4,5,6,7,8,9,10,12,13,16,20,21,23,26,27,28,29],output_bia:1,output_bias_gradi:1,output_shap:4,output_weight:1,output_weights_gradi:1,outputlayer1:12,outputlayer2:12,outsid:4,over1:13,over:[0,1,3,4,5,6,9,10,12,13,20,21,26,27,28,29],overal:[1,10],overcast:9,overcom:[12,13],overdetermin:[0,26],overfit:[0,1,3,6,9,10,13,29],overflow:[1,5,28,29],overhead:12,overlap:[3,7,8,9],overlin:[0,5,6,9,10,11,14,20,26,27,28],overst:[0,26],overtrain:4,overview:3,own:[4,5,6,8,12,13,19,20,21,28],owner:[0,27],ownmsepredict:0,ownmsetrain:0,ownridgebeta:[0,6,28],ownypredictridg:0,ownytilderidg:0,oxid:[0,27],oyvinssc:[],p0:2,p1:2,p:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,17,20,23,26,27,28,29],p_:[2,4,8,9],p_hidden:2,p_i:[5,23,28,29],p_j:23,p_n:23,p_output:2,p_x:23,pack:[0,26],packag:[0,1,2,3,4,5,6,7,8,11,13,14,15,19,21,23,27,28],pad:[3,4],page:[0,19,26],pai:[0,1,9,13],pair:[0,2,3,9,19,23,26,27],pamilla:26,panda:[0,4,5,6,7,9,11,15,19,21,28,29],panel:26,paper:1,paradigm:[0,26],parallel:[3,4,10,13,14,19,20,26],param:2,paramat:2,paramet:[0,1,2,3,4,5,6,7,8,9,10,12,13,16,17,18,21,23,28,29],parameter:[0,6,10,21,26,27],parametr:[0,6,15,16,26,27,29],paramt:[3,5,28,29],parent:2,parent_argnum:2,park:[],parser:[0,26],part:[0,1,3,5,6,10,17,18,20,22,23,24,26,28,29],partial:[0,1,3,4,5,6,7,8,10,11,12,13,14,16,23,26,27,28],partialmethod:[3,4,14],particip:[19,22,24,26],particl:[0,4,13,23,26],particular:[0,1,2,3,5,6,9,10,11,12,13,16,21,23,25,26,27,28,29],particularli:[5,6,8,11,13,23,27,29],partit:[1,4,9],partli:[6,26],pass:[2,3,12,14],password:21,past:[10,23],patch:[6,23,29],path:[0,4,6,7,9,15,19,26,27,29],patient:7,patter:4,pattern:[0,3,4,12,22,25,26],pauli:[0,26],pc:[11,19],pca:[0,7,19,26,27],pcolor:6,pcolormesh:6,pd:[0,4,5,6,7,9,11,26,27,28,29],pde:2,pdf:[0,3,4,5,6,9,21,25,26,28,29],pedagog:[0,26,27],penal:[6,27,28],penalti:[6,13,21,27,28],penros:[5,6,28],pentagon:13,peopl:[0,1,9,13,19,27],per:[0,1,6,22,24,26,27,29],percentag:[0,10,11,24,27],perceptron:[0,1,7,26],peregrin:26,perfect:[0,1,13,26],perfectli:[4,6,29],perform:[0,2,3,4,5,6,8,10,11,12,13,14,15,16,17,19,20,21,23,26,27,28,29],performac:4,perhap:[0,5,13,26,27,28],perimet:[1,9],period:[1,4,23],permut:11,persist:13,person:[5,6,7,22,24,26,27,28,29],perspect:25,pertin:12,petal:[8,9],peter:25,petersen:[],phantom:23,phase:[6,12],phawkin:[3,4,14],phenomena:23,phi:8,phi_k:8,philip:[],philosophi:13,phone:[24,26],photo:4,phrase:[0,26],physic:[0,1,4,7,12,13,23,24,25,26,27,28,29],pi:[2,3,5,6,7,9,12,13,23,28,29],pick:[1,9,10,11,13,14],pickl:1,pictur:[0,26],pie:[19,26],piec:[11,14],pillow:[0,15,19,21,26],pinv:[5,6,13,27,28],pip3:[0,1,15,21,26],pip:[0,1,15,19,21,26],pipelin:[0,6,8,10,27,29],pippin:26,pit:4,pitfal:[6,27,28],pitt:12,pixel:[1,3,4],pixel_height:[1,3],pixel_width:[1,3],place:[0,4,6,8,13,20,21,26,29],plai:[0,3,4,5,6,8,11,19,26,27,28,29],plain:[8,10,12,13,14],plan:[6,9,24,25,26],plane:[8,9],plateau:[5,28],platform:[19,26],plausibl:12,pleas:[3,4,6,7,11,13,14,21,24,26,27],plenti:1,plethora:[3,12],plot:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,23,26,27,28],plot_confusion_matrix:[7,10],plot_count:6,plot_cumulative_gain:[7,10],plot_data:1,plot_dataset:8,plot_decision_boundari:[9,10],plot_import:10,plot_max:4,plot_min:4,plot_model:4,plot_numb:4,plot_predict:8,plot_regression_predict:9,plot_result:4,plot_roc:[7,10],plot_surfac:[2,6,13,21],plot_train:9,plot_tre:[9,10],plt:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],plu:[0,3,5,7,26,27,28],pm:[8,29],pmatrix:2,pml:25,pn:3,png:[0,4,6,7,9,26,27,29],pocketfft:[3,4,14],point:[0,1,2,3,5,6,7,8,9,10,11,13,14,15,16,17,20,21,23,24,26,27,29],point_1:4,point_2:4,poisson:[19,23,26],poli:[6,8,29],pollut:[3,4,14],poly100_kernel_svm_clf:8,poly3:0,poly3_plot:0,poly3dcollect:13,poly_featur:[8,9],poly_features10:9,poly_fit10:9,poly_fit:9,poly_kernel_svm_clf:8,polydegre:[0,5,6,10,27,28,29],polygon:13,polym:12,polymi:21,polynomi:[0,5,6,7,8,9,10,11,15,16,17,21,26,27,29],polynomial_featur:[6,29],polynomial_svm_clf:8,polynomialfeatur:[0,6,8,9,27,29],polytrop:[0,6,26,29],pool:3,pool_siz:3,poor:[1,13],poorli:[0,27],pop:2,popul:[0,5,26,27,28,29],popular:[0,1,3,6,7,8,9,11,12,15,19,20,21,23,26,27],popularli:[0,26],portabl:10,portion:[11,13],pose:[0,4,5,6,11,23,26,29],posit:[0,1,2,3,5,7,8,10,11,13,14,17,20,23,26,27,28,29],possibl:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,15,19,20,21,23,24,26,27,28,29],posterior:[5,28,29],postpon:[0,26,27],postscript:21,postul:[5,28,29],potenti:[0,3,5,6,12,13,26,27,28,29],potr:[],potrf:[],pott:12,power:[0,1,5,6,8,9,12,13,26,27,29],pp:[5,6,18,28,29],practic:[0,5,6,7,8,21,23,27,28,29],practition:[0,1,3,26],pre:26,preced:[1,11,12,23],preceed:4,preceq:8,precis:[0,2,5,11,13,20,21,23,26,27,28,29],pred:[6,29],predicit:0,predict:[0,1,5,6,7,8,9,10,15,16,19,21,25,26,27,28,29],predict_prob:1,predict_proba:[7,10],predictor:[0,5,6,7,9,10,11,26,27,28],prefer:[0,1,6,8,9,11,13,15,19,21,26],prepar:[0,6,20,21,26,27],preprocess:[0,4,6,7,8,9,10,11,28,29],prerequisit:0,prescript:21,presenc:13,present:[0,5,6,7,9,12,13,20,21,23,26,27,28],preserv:[3,11,20],press:[13,25],pretrain:[1,4],pretti:[0,4,8,9,15,19,21,26],prev_centroid:14,prevent:[13,23],previou:[0,1,2,3,4,5,6,8,10,11,12,13,20,21,23,27],previous:[2,3,9,10,23],price:[0,4,9,13,27],primal:8,primari:[0,7,26],prime:23,primit:2,primitive_batch:[3,4,14],princip:[0,5,7,19,26,27],principl:[0,6,7,8,14,26,29],print:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,20,23,26,27,28,29],print_funct:[8,9],printout:[0,26],prior:[0,5,6,26,27,28,29],privat:[0,26],prob:[1,23],probabilist:[0,25,26,27],probabl:[0,1,3,4,6,7,10,13,19,26,27],problem:[0,3,4,5,6,7,8,9,10,11,12,14,16,17,19,20,21,23,29],probml:25,proce:[0,5,6,7,8,9,10,11,13,20,26,27,28,29],procedur:[2,4,5,6,8,10,11,13,21,27,28,29],proceed:20,process:[0,2,4,6,9,10,12,13,15,19,20,21,23,25,26,29],prod:25,prod_:[1,5,7,28,29],produc:[0,3,4,5,6,9,10,11,12,13,19,20,21,23,26,27,28,29],product:[0,1,3,5,6,7,8,12,13,16,19,20,26,27,28,29],profess:[0,26],program:[0,1,4,5,6,8,12,14,15,16,19,20,22,23,24,26,27],programm:20,progress:[1,4,14],prohibit:[6,29],project1:[6,21],project:[0,1,2,3,5,11,13,17,18,19,22,24,27,28,29],project_root_dir:[0,6,7,9,26,27,29],promin:12,promis:8,promot:[24,26],prone:9,pronounc:[13,19,26],proof:[0,11,12,13,26,29],propag:[2,3,13],proper:[0,2,6,7,21,26,29],properli:[1,6,8,10,13,21],properti:[0,1,3,12,13,16,17,20,26,29],proport:[0,1,5,9,11,13,16,23,26,27],propos:[1,4,6,10,21,26],propto:[5,13,28,29],protect:[],protobuf:[],proton:[0,26],prove:[3,13],provid:[0,1,3,4,5,6,8,9,10,12,13,15,19,20,21,23,26,27,28,29],proxi:[1,13],prune:9,pseudo:[20,23],pseudocod:21,pseudoinv:[5,28],pseudoinvers:[5,6,28],pseudorandom:[6,23,29],psycholog:[0,26],pt:13,ptratio:27,punish:[0,1,26],pure:[3,9,23],purest:9,puriti:9,purpos:[0,3,10,12,14,26,27],put:[1,26],putarow:[],putboundslic:[],putclist:[],putobjsens:[],putqobj:[],py:[0,1,2,3,4,5,6,7,8,11,13,14,21,26,27,28,29],pycod:26,pydata:19,pydot:9,pyhton2:26,pylab:[0,7,26],pylint:[3,4,14],pypi:19,pyplot:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28,29],pythagora:[5,28,29],python2:[0,15,21,26],python3:[0,1,2,3,4,6,7,8,11,13,14,15,19,21,26,27,28],python:[1,2,3,4,5,6,8,11,12,13,14,16,21,23,27,28],pytorch:[0,19,21,26],pywrap_tf:[],q:[5,6,8,11,23,27,29],qn_bavhmd8u:[],qp:8,qquad:[2,11,13,20],qr:[5,6,20,27],quad:[1,13,20],quadrat:[0,8,9,13,15,16,26],qualit:[4,9,21,23],qualiti:[0,9,15,16,19,26,27],quantifi:1,quantil:10,quantit:[0,6,9,21,26,29],quantiti:[0,2,5,6,7,9,10,11,12,14,16,20,23,26,27,28,29],quantum:[4,12,25,26],quartil:[0,27],quench:5,queri:9,question:[0,5,6,9,11,12,13,21,24,26,27,28,29],qugan:4,quick:[4,23],quick_execut:[],quickli:[1,3,9,11,13],quit:[1,5,6,9,10,12,27,29],quot:[4,26],r2:[0,5,6,26,27,28],r2_score:[0,26,27],r2score:[0,26],r:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,27,28,29],r_1:9,r_2:9,r_j:9,r_m:9,rad:[0,27],radial:[0,8,12,27],radioact:23,radiu:[0,1,9,27],rag:2,rain:9,rais:[0,2,13,26,28],ramp:1,ran0:23,ran1:23,ran2:23,ran3:23,rand:[0,4,5,6,9,10,13,15,16,20,26,27,28,29],rand_max:[],randint:[6,9,13,29],randn:[0,1,2,5,6,9,11,13,15,16,26,27,28,29],random:[0,1,2,3,4,5,6,8,9,13,14,15,16,19,20,21,26,27,28,29],random_devic:[],random_forest_model:10,random_index:13,random_indic:[1,3],random_st:[0,7,8,9,10,11,27],randomforestclassifi:10,randomli:[1,6,9,13,14,29],randomnumbergener:[],rang:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,20,23,26,27,28,29],rangl:[0,6,11,23,26,27],rangle_x:23,rank:[5,27],rankdir:4,raphson:[1,8,13],rapidli:[0,26],rare:[1,13,26,29],rate:[0,1,2,3,4,8,9,10,12,13,27],rather:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,20,23,26,27,28,29],ratio:[4,7,9,10,11],rational:[0,26],ravel:[5,6,7,8,9,10,11,13,20,27,29],raw:3,raw_df:27,rbahrf:[],rbf:[8,11,12],rbf_kernel_svm_clf:8,rbf_pca:11,rc:[0,23,27],rcond:[0,26,27],rcparam:[0,1,3,7,8,9,10,23,26],rd:[],rdbj50lv3go:[],re:[2,4,13],reach:[1,4,5,6,7,9,10,11,12,13,14,28,29],read:[0,2,3,4,5,6,7,8,11,12,16,20,21,22,23,25,28],read_csv:[0,6,7,9,26,27,29],read_fwf:[0,26],reader:[0,6,20,23,26,27,28],readi:[0,1,5,6,8,10,11,12,20,21,26,28,29],readili:1,readthedoc:19,real:[0,1,2,4,7,10,11,12,13,20,27,29],real_loss:4,real_output:4,realist:8,realiti:23,realiz:[1,12],realli:[0,1,26],rearrang:13,reason:[0,1,3,4,10,13,25,26],reassign:1,rebuild:[],recal:[5,6,9,10,11,12,20,23,26,27,28,29],recalcul:[],recast:3,receiv:[1,3,10,12,23],recent:[0,2,3,4,6,9,10,13,14,15,21,25,26,28,29],recept:[3,12],receptive_field:3,recip:[0,6,7,20,21,26,27],reciproc:[5,28],recogn:[0,4,5,10,26,28,29],recognit:[0,1,3,12,22,25,26],recommend:[0,2,3,4,5,6,8,13,15,18,19,20,21,25,28,29],reconsid:9,reconstruct:11,record:[10,21,22,24,26],recreat:[],rectangl:[9,13],rectangular:[5,27],rectifi:[1,3,12],recur:[0,19,26],recurr:[0,1,19,26],recurs:[9,19,20,26],recycl:[],red:[0,3,4,6,8,9,29],redefin:[0,10,26,27],redefinit:28,reduc:[1,3,5,6,9,10,11,13,26,28,29],reduct:[0,10,11,19,23,26,27],refer:[0,1,2,3,5,6,7,11,12,13,14,20,21,25,26,27,29],referenc:2,refin:12,refit:[6,29],reflect:[0,1,4,5,21,23,26],refresh:[19,26],refreshprogrammingskil:26,reg:[10,11],regard:[1,9,13],regardless:12,region:[3,4,6,9,12,21],regist:[6,21,23],reglasso:[5,28],regr_1:[0,9],regr_2:[0,9],regr_3:[0,9],regress:[1,8,11,12,15,16,19,20],regressor:[0,7,10,26],regridg:[0,5,6,28],regular:[0,3,4,5,6,7,9,13,17,24,26,27,28,29],regularis:6,reilli:[0,15,25,26],reinforc:[0,8,19,26],reiter:1,reject:7,rel:[0,4,6,7,9,12,13,23,26,27,29],relat:[0,1,3,4,5,11,13,14,20,23,26,28,29],relationship:[0,4,9,26],relativeerror:[0,26,27],releas:[1,6,13,19,21,26,29],relev:[0,1,5,7,11,15,19,21,23,26],reli:[0,3,4,6,8,14,26],reliabilti:21,reliabl:[7,23],relu:[3,4],remain:[1,2,4,6,12,20,23,27,28,29],remaind:23,reman:2,remark:1,rememb:[0,8,13,20,21,26],remind:[0,5,11,13,20,23,27,28,29],remov:[0,4,5,6,26,27,28],render:[0,26,27],reorder:[5,7,27,28],reorgan:[0,26],repeat:[0,1,3,4,5,6,9,10,11,13,14,20,21,23,26,28,29],repeated:26,repeatedli:[0,6,10,13,29],repet:3,repetit:[6,26,27,29],rephras:13,replac:[0,1,3,4,5,6,10,12,14,15,16,19,21,26,27,28,29],replica:[6,29],repo:21,report:26,reportexampl:21,repositori:[0,4,21,26,27],repres:[0,1,2,3,4,5,6,7,8,9,10,12,13,23,26,27,28,29],represent:[0,1,3,6,23,26,29],representd:3,reproduc:[0,5,6,9,12,15,16,19,21,23,26,27],repuls:[0,26],request:[0,13,26],requir:[0,1,3,4,5,6,8,9,11,12,13,17,20,26,27,28,29],res1:2,res2:2,res3:2,res_analyt:2,res_analytical1:2,res_analytical2:2,res_analytical3:2,resaml:[6,21],resampl:[0,7,10,19,26,27],rescal:[0,11,12,27],rescu:[5,28,29],reseach:[6,21],research:[0,4,13,19,25,26],resembl:[6,23,29],reserv:[1,5,6,23,28,29],reset:[],reshap:[0,1,2,3,4,6,8,9,10,15,16,20,26,27,29],residenti:[0,27],residu:[0,5,13,26],resiz:[5,27],resourc:26,respect:[0,1,2,3,5,6,7,8,10,11,12,13,14,16,17,21,23,26,27,28,29],respond:12,respons:[0,7,9,12,26,27],rest:[0,5,27],restat:[0,12,26],restor:4,restored_discrimin:4,restored_gener:4,restrict:[0,3,9,12,26],result:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,23,26,29],result_typ:2,retail:[0,27],retain:[5,6,27,29],rethink:29,return_data:14,return_kwarg:[],return_sequ:4,return_x_i:9,reus:[1,3,6,18,21],reveal:[0,12,26],revers:[1,20],review:[19,20],revisit:14,revolut:26,reward:[0,4,26],rewrit:[0,3,5,6,7,8,10,11,12,13,20,21,23,28],rewritten:[2,6,8,10,23,29],rewrot:13,rf:10,rfft2:[3,4,14],rfft:[3,4,14],rfftfreq:[3,4,14],rfftn:[3,4,14],rgb:3,rgoj5yh7evk:19,rh:[6,29],rho:[0,10,13,26],rho_1:10,rho_2:10,rho_m:10,rich:[0,26],ride:9,rideclass:9,ridedata:9,ridg:[7,11,13,19,26],ridge_sk:6,ridgebeta:[5,28],right:[0,1,2,3,5,6,7,8,9,10,12,13,14,16,17,20,21,23,26,27,28,29],right_sid:2,rightarrow:[0,1,5,6,8,11,12,13,23,26,27,29],rigor:[0,26],ring:6,rise:[0,26],risk:[0,13,15,16,26],rival:4,river:[0,27],rm:[0,23,27],rmse:[0,27],rmsporp:13,rmsprop:[1,3,4,13],rnd_clf:10,rng:23,rnn1:4,rnn2:4,rnn:[4,12],rnn_2layer:4,rnn_input:4,rnn_output:4,rnn_train:4,rntrick1:23,rntrick2:23,rntrick3:23,rntrick4:23,ro:[0,13,26],robert:[18,21,25],robust:[0,26],robustscal:[0,27],roc:[7,10],role:[0,2,5,6,8,19,26,27,28,29],roll:6,room:[0,24,26,27],root:[0,5,9,13,23,26,27,28],rot:26,rotat:[1,8,9,10],rotation_matrix:9,roughli:[1,3],round:[0,7,9,13,27],routin:[13,20,26],row:[0,1,2,5,6,7,9,11,20,26,27,28,29],rr:[5,27],rrr:[5,27],rthe:[],rug:13,rule:[0,1,5,6,13,21,26,27,28,29],run:[0,1,2,4,5,6,8,9,11,13,15,19,21,26,27,28,29],runtim:[1,6,14],runtimewarn:[1,6,29],russel:26,rust:[0,15,19,20,26],rustad:[],rvert:1,rvert_2:1,s:[0,1,2,3,4,5,6,7,9,11,12,13,16,17,18,19,20,21,23,26,27,28],s_1:6,s_:[3,6],s_i:[6,7],s_j:6,s_k:6,saddl:13,safe:[],safe_zip:[3,4,14],sai:[0,1,2,3,4,5,6,7,8,9,10,11,12,20,21,23,26,27,28,29],said:[6,9,13],sake:[0,5,7,11,26,27,28],sale:[0,26],sam:26,same:[0,1,2,3,4,5,6,8,9,11,12,14,17,20,21,23,26,27,28],samm:10,sampl:[0,1,2,3,4,5,6,7,8,9,10,13,14,15,16,19,20,21,23,26,27,28,29],sample_vari:14,sample_weight:[],sampleexptvari:23,samwis:26,sastri:11,satisfactori:[0,26],satisfi:[1,2,3,6,8,13,17,20,23,29],satur:[1,6,29],save:[0,4,6,7,9,13,26,27,29],save_fig:[0,6,7,9,10,26,27,29],savefig:[0,4,6,7,9,23,26,27,29],savetxt:4,saw:[5,27],scalabl:10,scalar:[2,5,6,10,13,27,28,29],scale:[0,1,3,5,6,7,8,9,10,11,12,13,15,16,17,19,20,21,24,26,28,29],scale_mean:4,scale_std:4,scalei:[],scaler:[0,7,8,9,10,11,27],scalex:[],scan:[5,7,28,29],scari:[5,28,29],scatter:[0,1,6,7,8,9,14,26,27,28,29],scenario:[6,13],schedul:[13,24],scheme:[1,13],schigsog2l:[],schrage:23,scienc:[0,1,10,12,13,19,22,23,24,25,27],scientif:[0,15,19,21,26],scientist:[0,26],scikit:[3,5,6,7,8,9,10,13,19,20,21,22,25],scikit_learn:[0,16],scikitlearn:26,scikitplot:[7,10],scipi:[0,3,5,6,13,15,19,20,21,26,27,28,29],scl:6,score:[0,1,3,6,7,9,10,11,15,16,21,24,26,27,29],scores_kfold:[6,29],scratch:[1,13],sdg:13,sdt_bfla8ua:[],seaborn:[0,1,3,6,7,26,27],seamless:[0,15,19,21,26],search:[0,1,3,5,9,13,26,28],sec:6,second:[0,2,3,4,5,6,7,8,9,11,12,14,15,16,19,20,21,23,24,26,27,29],secondeigvector:11,secondli:12,section:[4,11,17,20,23,27,28],sector:[0,26],see:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,18,19,20,21,23,26,27,28,29],seed:[0,1,2,3,4,5,6,8,9,11,13,14,15,16,21,23,26,27,28,29],seed_imag:4,seek:[1,2,8],seem:[1,3,4],seemingli:[0,26],seen:[0,1,3,5,10,12,17,23,26],segment:13,seismic:6,seldomli:[0,26],select:[1,5,6,8,9,10,11,17,21,22,23,24,25,26,27,28,29],self:[1,2,5,27,28],sell:4,semest:[7,22],semi:[8,13],semilogx:6,send:[5,12,13,24,26],senior:[22,24],sens:[0,4,6,8,21,26,29],sensibl:3,sensit:[0,5,6,9,13,26,27,28,29],sent:[2,29],sentenc:[4,12],sep:[27,29],separ:[0,1,2,3,4,6,8,9,12,14,15,19,21,23,26,29],septemb:[16,17,18,21,26,27],sequenc:[2,3,4,7,9,10,12,13,19,20,23,26],sequenti:[1,3,4,10,12,23],seri:[0,1,2,3,4,5,6,10,11,12,13,20,26,27,28,29],serif:[0,7,23,26],serv:[0,1,2,3,5,7,13,21,25,26,27,28,29],servic:21,session:[1,21,22,24,26,29],set:[1,4,5,6,7,8,10,11,13,14,16,17,19,20,21,23,24,28,29],set_major_formatt:[6,21],set_major_loc:[6,21],set_stream:[],set_tick:[1,8],set_ticklabel:1,set_titl:[0,1,2,3,7,12,14,26],set_xlabel:[0,1,2,3,7,12,26],set_xlim:[7,12],set_xticklabel:1,set_ylabel:[0,1,2,3,7,26],set_ylim:[7,12],set_ytick:7,set_yticklabel:[1,6],set_zlim:[6,21],seth:4,setiosflag:[],setminu:6,setosa:[8,9],setosa_or_versicolor:8,setp:[6,29],setprecis:[],setse:[],setup:[1,4,6,8,19,26,27],setw:[],sever:[0,3,5,6,7,8,9,11,12,13,19,20,23,26,27,28,29],sgd:[1,3],sgd_clf:8,sgdclassifi:8,sgdreg:13,sgdregressor:13,sgn:[5,27,28],sh:[],shallow:13,shape:[0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,20,26,27,28,29],shape_bas:2,share:[1,3,26],she:7,shift:[1,6,12,23,28],ship:3,shire:26,shortcom:13,shorten:4,shorter:23,shorthand:[26,29],shortli:[20,26],should:[0,2,3,5,6,8,9,11,12,13,15,16,20,21,23,26,27,28,29],should_sync:[],show:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,18,20,21,23,26,27,28,29],show_shap:4,shown:[0,4,5,7,8,11,12,13,20,27],showpoint:[],shrink:[3,5,6,8,11,21,27,28],shrinkag:[5,6,27,28],shrunk:11,shuffl:[0,1,4,6,13,27,29],side:[0,2,5,8,12,13,20,26,28,29],sigh:[19,26],sigma0:23,sigma1:23,sigma2:23,sigma:[0,1,5,6,7,10,11,12,13,17,18,20,21,23,26,27,28,29],sigma_0:[5,27,28],sigma_1:[5,27,28],sigma_2:[5,27,28],sigma_:[5,20,26,27,28,29],sigma_fn:[7,12],sigma_i:[0,5,26,27,28],sigma_j:[5,17,27,28],sigma_m:[6,23,29],sigma_n:[11,23],sigma_t:13,sigma_x:23,sigmoid:[1,2,4,7,8,10,12],sigmundson:[6,27,28],sign:[1,2,7,8,10,23,24],signal:[1,3,10,12],signatur:[],signifi:4,signific:1,significantli:[1,13,23],sigurd:[],sim:[4,5,6,13,18,21,23,28,29],similar:[0,1,2,3,4,5,6,7,8,9,10,11,14,17,19,20,21,26,27,28,29],similarli:[0,1,3,5,8,10,13,23,26,27,28],simpl:[1,2,3,5,6,7,8,10,11,12,14,15,16,17,19,20,21,23,29],simple_rnn:[],simplepredict:10,simpler:[0,1,5,6,7,13,16,19,21,26,28],simplernn:4,simplest:[0,1,3,4,9,10,12,14,26],simpletre:10,simpli:[0,1,2,4,5,6,8,9,10,11,12,15,19,20,21,23,26,27,28,29],simplic:[2,5,6,7,8,9,10,11,12,14,27,28],simplicti:[5,27],simplifi:[0,6,9,15,19,21,26,27,28,29],simplist:[3,6,23,29],simul:[6,29],simultan:[6,29],sin:[0,1,2,3,4,9,12,13,20,26],sinc:[0,1,2,3,5,6,7,8,9,10,11,13,20,23,25,26,27,28,29],sine:[3,12],singl:[0,1,2,3,5,6,7,8,9,12,13,20,23,26,27,28,29],singular:[0,6,13,20,21,26,28,29],sinusoid:3,site:[0,1,2,3,4,6,7,8,11,13,14,21,22,26,27,28],situat:[0,4,5,7,13,23,26,27],six:[3,4,14,23],size:[0,1,2,3,4,5,6,8,9,10,11,13,20,21,23,26,28,29],sketch:10,ski:9,skill:[0,26],skip:11,skiprow:27,skl:[0,6,26,27,28],sklearn:[0,1,3,5,6,7,8,9,10,11,13,14,26,27,28,29],skplt:[7,10],sl:[6,28],slack:8,slice:[2,20,26],slide:[0,3,15,16,21,23,26,27],slight:[6,13,29],slightli:[1,2,3,5,6,7,10,23,27,28,29],slope:[8,11,12],slow:[0,2,8,13,27],slower:[5,20,26,27,28],slowest:20,slowli:12,slp:1,small:[0,1,2,3,5,6,8,9,10,11,12,13,19,20,21,23,26,27,28,29],smaller:[0,1,2,5,6,8,9,11,13,23,26,27,28,29],smallest:[0,4,14,15,16,26],smallest_row_index:14,smooth:[0,3,6,9,13,21,26],sn:[0,1,3,6,7,26,27],sne:11,sneak:26,so:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,18,19,20,21,23,24,26,27,28,29],soar:6,social:[0,26],societ:26,soft:[1,7,10,12],soft_pmap:[3,4,14],soften:8,softmax:[3,7],softwar:[0,8,15,19,20],sol:8,sole:[0,6,26],solid:[0,7,26],solitem:[],soltyp:[],solut:[0,1,2,3,5,6,8,10,11,13,17,18,20,21,23,26,27,28,29],solutionsummari:[],soluton:2,solv:[0,1,3,5,6,8,10,11,12,13,20,21,26,27],solve_expdec:2,solve_ode_deep_neural_network:2,solve_ode_neural_network:2,solve_pde_deep_neural_network:2,solveod:2,solveode_popul:2,solver:[2,7,8,9,10,11,20],some:[0,1,2,3,4,5,6,7,8,9,10,11,12,14,23,28,29],some_model:[6,27,28],somehow:4,someth:[0,1,3,4,7,9,11,21,23,26,27],sometim:[0,1,11,12,13,14,26,27],soon:[20,24],sophist:[0,26],sopt:13,sort:[5,6,9,11,23,27,29],sound:[3,5,28,29],sourc:[0,1,3,6,15,19,20,21,23,26,27,29],space:[0,1,4,5,8,9,11,12,13,14,23,27,28,29],span:[0,3,5,9,11,20,26,27],spare:1,spars:[3,6,20,26],sparse_mtx:[20,26],sparsecategoricalcrossentropi:3,sparsiti:10,spatial:[1,2,3,12],spdiag:[],speak:23,special:[6,7,10,12,13,20,23,26,27,28],specif:[0,1,2,3,4,5,6,7,8,9,11,12,16,19,20,21,23,25,26,27,28,29],specifi:[0,3,5,6,7,9,11,13,14,21,23,26,28,29],specifici:[0,10,26],spectacular:3,spectral:1,speech:[0,1,3,4,12,26],speed:[1,2,4,13],spend:23,spent:21,sphere:[0,27],spin:6,spite:[0,26],spline:8,split:[1,3,4,5,6,8,9,10,11,14,17,21,23,28,29],splite:0,splitter:[1,10],spmatrix:[],spontan:23,spot:3,spread:[0,11,23,26,27],springer:[18,21,25,26,28],spuriou:13,sqquar:28,sqrsignal:3,sqrt:[0,3,4,5,6,8,10,11,13,17,23,27,28,29],squar:[1,2,3,4,7,8,9,11,13,14,15,16,17,19,20,23,29],squarederror:10,squaredeuclidean:14,squash:12,squeez:2,srand:[],srtm:[6,21],srtm_data_norway_1:[6,21],sse4:[],stabil:[5,21],stabl:[0,4,5,6,7,9,11,19,26,27,28],stack:[2,3,4],stacklevel:[0,26],stage:[5,13,21],stai:[0,2,4,5,11,17,26,27],stand:[0,5,9,12,26,27,28],standadscal:27,standard:[0,1,4,5,6,7,8,10,12,15,16,17,20,21,23,26,28],standard_basi:2,standardscal:[0,6,7,8,9,10,11,27,28],stanford:13,start:[0,1,2,3,4,5,6,8,9,10,11,12,13,14,20,21,23,24,26,27,29],start_box:[2,13],start_nod:[2,13],start_tim:14,startpoint:[],stat:[6,27,29],state:[1,2,4,5,6,7,8,10,11,12,13,19,23,27,28,29],statement:[0,7,20],statist:[0,1,3,4,7,9,10,11,12,13,14,18,20,21,22,25,27],statu:[0,7,11,26,27],stavang:[6,21],std:[0,4,6,26,27,29],stdev:[],stdout:[],steep:13,step:[0,1,2,4,6,7,9,10,11,12,13,14,20,21,26],step_fn:[7,12],step_length:13,steps_list:9,steps_per_epoch:[],stereo:3,stian:[],still:[0,2,3,5,6,11,13,17,23,29],stimuli:12,stk2100:[25,26],stk3155:[15,21,22,24],stk4021:[25,26],stk4051:[25,26],stk4155:[22,24],stk5000:25,stk:[25,26],stochast:[0,1,5,6,8,11,12,15,16,21,26,28,29],stock:4,stoke:12,stone:[0,7,21,26],stop:[1,4,7,9,11,13,14],storag:[5,27],store:[0,1,2,3,6,11,13,21,23,26],storehaug:[24,26],str:[1,3,4],straight:[0,6,8,13,26,27,29],straightforward:[0,2,3,5,6,8,9,10,13,20,26,27,28,29],strategi:[0,1,9,26],stratifi:[6,29],streamtyp:[],strength:[0,5,14,27],stretch:11,strict:[8,13],strictli:[8,13],stride:[4,20],strike:6,string:1,stroke:7,strong:[3,6,9,10,12,20,23,29],strongli:[0,8,19,20,27],stronli:[0,27],structur:[0,1,2,3,6,9,10,12,19,26,29],stuck:[1,13],student:[0,21,22,24,25,26],studi:[0,3,4,5,6,7,8,11,12,13,17,19,21,25,26,27,28],studier:25,style:[0,7,9,20,26],sub:[9,12],subarg:[2,13],subdivid:[0,20,26],subfield:[0,26],subject:[6,8,23],submit:26,submodul:[3,4,14],subplot:[0,1,3,4,6,7,8,9,10,13,14,21,26,27,29],subplots_adjust:[8,23],subprogram:[20,26],subract:[0,27],subroutin:[0,26],subscript:1,subsequ:[1,4,5,6,12,20,23,27,29],subset:[1,6,9,12,13,19,26,29],subspac:[0,8,11,27],substanti:[9,10],substep:11,substitut:[3,6,12,20,29],subsubset:9,subtask:6,subtl:1,subtract:[0,4,5,6,11,13,17,20,21,23,28,29],subtre:9,subval:[2,13],succeed:[0,4,26],success:[3,7,9,13,23],successfulli:[4,9],sucess:[],sudo:[0,15,19,21,26],suffer:[0,1,2,5,10,26,27,28],suffici:[1,6,8,11,13,29],suggest:[1,13,25],suit:[8,12],suitabl:[0,23,27],sum:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,20,21,23,26,27,28],sum_:[0,1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],sum_i:[0,2,5,6,8,13,17,21,27,28,29],sum_j:6,sum_ja_:0,sum_k:[6,8,12,20],sum_logist:13,sum_m:3,sum_n:3,sum_nx_:3,summar:[5,6,9,28,29],summari:[1,3,4,10,22,28],summat:[0,3,16,27],sundai:[17,18],sunni:9,superconduct:[27,28],superfici:3,superscript:[1,12],supervis:[0,5,6,7,9,12,19,26,27,29],supplement:7,support:[0,1,9,10,11,13,19,26,27],suppos:[0,5,6,7,8,10,11,12,13,20,26,27,28,29],suppress:[5,13,28],sure:[0,1,4,6,21],surf:[6,21],surfac:[0,6,21,26],surpass:6,surpris:[0,26],surround:[3,19],survei:[0,5,6,26,27,28,29],suyrzm0:[],svc:[8,9,10],svd:[0,6,11,17,26,29],svdinv:[5,28],svm:[8,9,10,11],svm_clf:[8,10],swap:2,swapax:2,swath:[5,27],sy:[3,4,13,14],symbol:[1,5,11,13,19,23,26,27,28],symmeteri:1,symmetr:[0,5,8,11,12,13,20,26,27],symmetri:[6,9],sympi:[0,15,19,21,26],synonim:23,syntax:[1,13],syntaxerror:[1,8],syrk:[],system:[0,1,3,4,6,7,9,10,12,13,15,19,20,21,26],systemat:[4,6,29],t0:[3,6,13],t1:[2,13],t2:2,t3:2,t:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,26,29],t_0:[2,9,13],t_1:13,t_:2,t_b:10,t_i:[1,2,5,12,27],t_j:12,t_k:9,tabl:[9,21,23,24,26],tabul:[0,26],tabular:26,tackl:4,tag:[2,3,4,5,6,7,12,13,14,20,23,27],taht:[0,26],tail:23,tailor:[2,8,11,26],taiwan:[0,26],take:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,19,20,21,23,26,27,28,29],taken:[0,1,3,6,10,13,20,26,29],tan:3,tangent:[1,4,12,13],tanh:[1,4,7,8,12],tape:[],target:[0,1,3,4,5,6,7,8,9,10,11,12,26,27,28,29],target_meta:[],target_nam:9,task:[0,1,3,6,9,11,12,14,21,26,29],tau:[3,5,23,28,29],taught:26,tax:[0,27],taylor:[2,13],taylornr:13,tc:8,td:[1,6,13,21,26,29],teach:26,team:1,teaser:0,technic:[0,5,6,13,15,26,27],techniqu:[0,1,8,10,13,19,23,25,26,27,29],technolog:[0,1,26],tek5040:[],tell:[0,4,6,10,11,13,23,29],temp1:1,temp2:1,temp:1,temperatur:[0,9,26],temporarili:1,ten:[3,17,26],tend:[3,5,6,8,9,10,12,13,14,27,28,29],tendenc:[0,26],tension:[6,29],tensor:3,tensorflow:[0,2,4,8,14,15,19,20,21,22,25,26,27],term1:[5,6,11,21,27],term2:[5,6,11,21,27],term3:[5,6,11,21,27],term4:[5,6,11,21,27],term:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,21,23,26,27,28],termin:[0,4,5,9,10,13,15,26,27],terrain1:[6,21],terrain:[6,21],test:[3,4,5,6,7,8,9,10,13,14,17,20,21,23,29],test_acc:3,test_accuraci:[1,3],test_error:6,test_imag:[3,4],test_ind:[6,29],test_input:4,test_label:[3,4],test_loss:3,test_pr:1,test_predict:1,test_rnn:4,test_scor:[7,10],test_siz:[0,1,3,5,6,10,26,27,28,29],test_split:9,tester:28,testerror:[0,6,27,29],testi:4,testpredict:4,testx:4,text:[0,1,2,4,5,8,9,11,13,15,20,23,25,26,27,29],textbook:[17,21,27,29],textual:9,textur:[1,9],tf:[1,3,4,13,14],tfe_py_execut:[],tflite:[3,4,14],th:[0,1,2,5,6,7,9,12,13,14,15,16,20,21,23,26,27,28,29],than:[0,1,2,3,4,5,6,7,9,10,11,12,13,17,19,23,26,27,28,29],thank:[4,6,27,28],theano:[1,19,26],thei:[0,1,2,3,4,5,6,7,8,9,11,12,13,14,20,21,23,27,28,29],them:[0,1,3,4,6,8,9,10,11,12,13,20,21,26,27,28],theme:[0,26],themselv:[0,23,26],thenc:[6,29],theorem:[2,6,7,27],theoret:[0,4,10,26],theori:[0,1,3,8,9,12,13,18,19,21,25,26,28],thereaft:[0,5,6,11,12,15,16,17,20,21,26,29],therebi:[0,5,7,11,26,27,28],therefor:[0,1,2,3,4,6,7,8,11,13,23,26,27,29],therein:11,thereof:[0,6,13,26,29],theta:[1,4,13,23],theta_:[1,13],theta_i:1,theta_k:[],theta_linreg:13,theta_t:13,thi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,22,23,25,27,28,29],thing:[0,1,2,4,5,7,9,23,26,28,29],think:[0,1,3,4,6,9,12,13,14,23,26,29],third:[0,3,6,13,24,26],thirti:7,thorughout:26,those:[0,3,5,6,8,9,10,11,20,21,22,26,27,29],though:[1,2,3,4,13,20,23],thought:[6,14,21,23,29],thousand:[0,1,21,26,27],thread:[],three:[0,1,3,5,6,8,9,12,15,20,21,22,23,24,26,27,28,29],threshold:[1,3,9,10,11,12,13],through:[0,1,2,3,4,5,6,8,11,12,13,14,19,20,23,26,27,28,29],throughout:[0,4,5,14,19,20,23,26],thu:[0,1,2,5,6,7,8,10,11,12,13,21,24,26,27,28,29],thumb:[0,6,21,26,27],thursdai:[24,26],tibshirani:[6,18,21,22,25,26,27,29],tick_param:6,ticker:[6,13,21,23],tif:[6,21],tight_layout:[1,7],tightli:11,tild:[0,5,6,7,11,15,16,17,18,21,23,26,27,28,29],till:[0,4,7,8,9,10,12,20,26,27],time:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,19,20,21,23,26,27,28,29],timefunct:[],timeit:4,timer:4,tini:1,tip:3,titl:[0,1,2,3,4,6,7,8,9,10,13,21,23,26,27,29],tmp:13,tmp_log:[],tn:[2,3,7],to_categor:[1,3,4],to_categorical_numpi:1,to_numer:[0,6,26,29],to_str:[],todai:3,todo:[3,4,14],togeth:[0,3,6,8,11,13,19,26,27],toi:14,told:13,toler:[2,6,14],tolist:4,tomographi:12,too:[0,2,4,5,6,9,11,13,15,23,25,26,27,29],took:[8,26],tool:[0,1,3,6,13,19,27,29],toolbox:8,top:[0,3,4,5,6,9,10,14,18,19,26,28,29],top_node_typ:[],top_trac:[],topic:[0,5,6,7,8,19,21,27,28],topograph:21,topolog:[1,3,12],toposort:2,torkjellsdatt:[24,26],toss:[10,23],total:[0,1,2,3,4,6,7,8,10,11,12,13,14,15,16,20,23,24,26,27,29],total_loss:4,totalclustervari:14,totalscatt:14,totalvari:[],toward:[1,2,7,12,13,21],town:[0,27],tp:[4,7],tpng:9,tpu:[13,19,26],tqdm:6,tr:27,trace:[2,13],trace_stack:[2,13],traceback:[0,2,3,4,6,9,10,13,14,15,21,26,28],traceback_util:[],tracer:[2,13],track:[3,13,14,20],tract:[0,27],tractabl:[0,26,27],trade:[5,9,28,29],tradeoff:[0,5,21,26,27,28],tradit:[0,1,4,6,26,29],train:[2,3,5,6,8,9,10,11,12,13,17,21,28,29],train_accuraci:[0,1,3,26],train_dataset:4,train_end:[0,1,27],train_error:6,train_funct:[],train_imag:[3,4],train_ind:[6,29],train_label:[3,4],train_pr:1,train_siz:[0,1,3,27],train_step:4,train_test_split:[0,1,3,5,6,7,9,10,11,26,27,28,29],train_test_split_numpi:[0,1,27],trainabl:[],trainable_vari:4,trained_model:[6,27,28],trainerror:[0,27],traini:4,training_checkpoint:4,training_dataset:4,training_gradi:13,training_gradient_fun:[],training_loss:[],trainingerror:[6,29],trainpredict:4,trainscor:4,trainx:4,trait:[0,26],trajectori:4,tran:[],transfer:9,transform:[0,5,6,7,8,9,10,11,12,13,19,20,26,27,28,29],transit:[6,12],translat:[1,4,6,10,27,28],translate_vjp:[],transpos:[1,5,11,20,27,28],travers:[0,5],treat:[0,1,3,6,12,13,23,26,27,28,29],tree:[0,1,6,19,21,26],tree_clf:[9,10],tree_clf_:9,tree_clf_sr:9,tree_reg1:9,tree_reg2:9,tree_reg:9,trend:23,treue:7,trevor:[18,21,25],tri:[2,3,4,9,13],triain:0,trial:[0,2,4,6,13,23,26,29],triangl:13,triangular:20,trick:[3,4,8,11,13,23],trickier:23,tridiagon:20,trillion:19,trivial:[0,1,5,11,23,26,28],troubl:[0,8,12,27],truck:3,true_beta:[6,27,28],true_divid:1,true_fun:[6,29],truli:26,tucker:8,tuesdai:[24,26,29],tumor:[7,9],tumour:7,tunabl:1,tune:[4,9,13,20,26],tup:[],tupl:[2,13],turn:[0,1,5,6,7,8,9,10,11,12,13,20,21,23,26,27,29],tutori:[1,4],tv:2,tveito:2,tweak:[1,4,10,23],twice:13,twist:11,twister:[],two:[0,1,2,4,5,6,7,9,10,11,12,13,14,15,16,20,22,23,25,26,28,29],tx:13,tx_1:13,txt:4,ty:13,type:[0,1,3,6,8,10,13,16,20,21,23,27,28,29],typeerror:[2,13],typic:[0,1,2,3,4,5,7,9,10,12,13,23,26,27,28,29],typo:21,u:[0,2,5,6,8,10,11,12,17,20,26,27,28],u_:20,u_i:12,u_m:10,ua:[0,26],ubuntu:[0,15,19,21,26],uci:[0,27],ufunc:[],uio:[21,24,25],un:14,unari:[20,26],unary_f:[2,13],unary_oper:[2,13],unary_to_nari:[2,13],unbalanc:[6,9,29],unbias:[0,5,6,26,28,29],unbroadcast:[],unbroadcast_f:[],uncent:[6,28],uncertainti:[0,5,26,28,29],uncertitud:23,unchang:[1,3],uncorrel:[10,23],undefin:[5,27],under:[0,1,3,4,5,6,10,13,14,15,16,19,21,26,27,28,29],underdetermin:[0,26],underfit:[1,6,29],underflowproblem:[5,28,29],undergo:[5,28,29],undergradu:[22,24],underli:[0,1,9,13,23,26],underset:[4,14],understand:[0,1,3,5,6,10,13,14,19,26,27,28],understood:[8,13],undesir:8,undetermin:[5,8,28,29],undo:4,unexpect:[6,29],unexpected:23,unfair:[6,27,28],unfortun:[1,8,9,10],unicode_liter:[8,9],uniform:[0,1,5,6,11,13,15,16,21,23,26,27],uniform_real_distribut:[],uniformli:[13,23],unifrompdf:23,unimport:13,union:[5,6,28,29],uniqu:[0,2,6,13,14,20,26,29],unique_cluster_label:14,unit:[0,1,3,4,5,10,12,23,26,27,28],unitari:[5,6,20,27],unitarili:[20,26],uniti:23,univari:23,univers:[0,1,2,13,15,19,21,22,24,26,27,28,29],unix:1,unknow:[0,20,26],unknown:[0,1,3,4,5,6,8,10,13,16,20,26,27,28,29],unknowwn:12,unlabel:1,unless:[0,3,6,11,13,21,26,27,29],unlik:[1,3,8,13],unnecessarili:9,unord:3,unravel:1,unrol:[3,11],unseen:[0,7,9],unstabl:1,unsupervis:[0,1,4,12,19,26],unsymmetr:[20,26],until:[1,2,4,9,12,13,14],untouch:0,unus:[3,4,14],unusu:12,up:[1,3,4,5,6,8,10,11,13,14,16,18,19,20,21,23,24,28],updat:[1,2,3,4,10,12,13,14],uploa:26,upload:[19,21,25],upon:[0,1,6,7,11,20,21],upper:[0,8,9,16,20,27],uppercas:[20,26],upsampl:4,upscal:4,url:[],us:[4,5,6,8,9,10,11,12,14,15,16,17,18,20,21,22,23,25,28,29],usa:26,usag:[0,8,19,26,27],usd10000:[0,27],usd:[0,27],use_bia:4,use_multiprocess:[],usecol:[0,26],useless:1,user:[0,1,2,3,4,6,7,8,11,14,15,19,20,21,26,27,28],usernam:21,userwarn:[3,4,6,14],usetex:23,usg:[6,21],usr:23,usual:[0,3,4,7,12,13,14,26],ut:[5,28],util:[0,1,3,4,6,7,10,14,26,27,29],ux:20,v0:23,v1:[3,4,14,23],v2:[3,4,14,23],v8ejbfezkui:[],v:[2,4,5,6,11,13,17,19,27,28],v_0:11,va:1,val:13,val_accuraci:3,val_loss:4,vale:2,valid:[0,1,4,7,9,10,13,19,23,26,27],validation_batch_s:[],validation_data:3,validation_freq:[],validation_split:4,validation_step:[],valu:[0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,19,20,21,26],valuat:9,valueerror:[0,2,26],valy:4,van:[0,18,21,26,27,28,29],vandenbergh:[8,13],vandermond:[0,26],vanilla:[0,6,11,14,27,28],vanish:[1,4,13,23],var_x:23,varabl:8,varepsilon:[5,6,18,21,28,29],varepsilon_:[5,6,28,29],varepsilon_i:[5,6,28,29],vari:[0,1,3,5,6,10,15,16,26,29],variabl:[0,1,2,5,6,7,8,10,11,12,13,14,20,26,27,28,29],varianc:[0,1,5,7,9,10,11,13,14,18,19,20,23,26,27],variance_i:[5,11,27],variance_x:[5,11,27],variant:[0,1,6,8,12,13,26,27],variat:[3,4,11,26],varieti:[0,3,12,15,19,21,26],variou:[1,3,5,6,7,8,9,11,12,13,16,18,19,20,21,23,27,28],vartempvec:[],varvec:[],varydimens:4,vastli:3,vaue:1,vault:0,vdot:[2,13],ve:21,vec:[6,29],vector:[0,1,2,3,4,5,6,7,9,10,11,13,14,16,17,19,28,29],vector_mean:14,ventur:[0,8,15,19,26],verbos:[1,3,4],veri:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,21,23,25,26,27,28,29],verifi:[3,11,20,26],versatil:[8,26],versicolor:[8,9],version:[0,3,4,10,13,14,15,19,20,21,23,26,27],versu:1,vert:[0,1,5,6,7,8,9,11,13,17,26,27,28,29],vert_1:[5,6,27,28,29],vert_2:[5,6,11,17,27,28,29],via:[0,5,6,7,8,9,10,11,12,15,19,20,21,22,23,24,26,27,28,29],vidal:11,video:[0,1,12,19,22,24,26,27,28,29],view:[1,3,5,6,12,13,23,25,26,28,29],violat:8,virginica:9,viridi:[0,1,2,3,26],virtual:1,viscos:13,viscou:13,visibledeprecationwarn:2,vision:[0,3,26],visual:[0,3,11,12,15,19,26,27],visualis:1,viz:[6,8,23],vjiseqm2lci:[],vjp:[2,13],vjp_0:2,vjp_0_fun:2,vjp_1:2,vjp_1_fun:2,vjp_argnum:2,vjpfun:2,vjpmaker:2,vjpnode:[2,13],vjps_dict:[],vmap:13,vmax:[1,6],vmc:[],vmin:[1,6],voic:3,volum:[0,3,26],vote:10,voting_clf:10,votingclassifi:10,votingsimpl:10,vrtx:[],vs:[0,4,6,27,29],vspace:[2,13],vstack:[5,11,20,23,26,27],vt:[5,27,28],w1:8,w2:[8,11],w3:8,w:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,20,23,26,27,29],w_1:[8,20],w_1x_1:8,w_1x_:8,w_2:[8,20],w_2x_2:8,w_2x_:8,w_3:20,w_4:20,w_:[1,12],w_hidden:2,w_i:[1,2,10],w_ix_i:12,w_j:20,w_m:20,w_output:2,w_px_:8,w_px_p:8,wa:[0,1,3,4,5,6,7,10,11,12,13,14,17,20,21,26,27,28,29],wai:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,17,20,23,26,27,28],walk:9,walker:23,wang:[0,26],want:[0,1,2,3,4,5,6,8,9,10,11,12,13,14,15,16,17,19,21,23,26,27,28,29],warn:[0,1,3,4,8,14,26,27],warrant:[6,29],wast:3,watch:19,wave:3,wavelet:8,we:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,29],weak:[9,10,14],weather:[1,12],web:[19,22,24,26,27],webpag:26,websit:[6,20,21,22,26],wedg:[8,23],wednesdai:[24,26,29],wee:11,week:[0,5,6,7,21,22,24],weekli:[19,21,24,25,26],weight:[0,1,2,3,6,7,9,10,12,13,21,23,27],weigth:2,welcom:[8,19],well:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,16,19,20,21,23,25,26,27,28,29],went:8,were:[0,1,3,4,5,6,7,8,10,11,12,14,23,26,27,29],wessel:[0,18,21,26,27,28,29],westbi:[],westby:[],what:[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,23],whatev:3,when:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,18,20,21,23,26,27,28,29],whenev:[13,23],where:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,26,27,28,29],wherea:[6,23,29],wherein:[1,12],whether:[0,3,5,7,9,21,23,26,28],which:[0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,28,29],whichev:[1,3],while_p:[3,4,14],white:9,whiteboard:[27,28],who:[0,22,26],whole:[1,3,4,5,9,11,13,28,29],whose:[0,6,10,23,27,29],whow:[11,27],why:[0,1,3,6,13,21,28],wide:[0,1,3,6,7,12,15,19,20,21,26,29],widehat:[6,29],width:[0,3,8,9,26],wieringen:[0,18,21,26,27,28,29],win:10,wind:9,wing:[24,26],winther:2,wiothout:[6,27,28],wiscons:7,wisconsin:10,wisdom:[6,28],wise:[0,1,5,12,13,27],wish:[0,2,5,7,8,11,13,14,20,21,26,27],with_std:[0,27],wither:6,within:[0,2,3,4,7,9,12,13,14,23,25,26],withinclust:14,without:[0,1,5,6,8,9,11,12,13,26,27,28,29],won:[0,26],wonder:8,woodi:26,word:[0,1,3,4,5,6,7,14,23,26,27,28,29],work:[0,1,4,6,7,8,9,13,15,19,21,22,23,24,26,27,29],worker:[],workshop:26,world:[0,8,26,27],worldwid:[0,26],wors:[0,1,3,4,6,26,29],worst:9,worth:9,worthi:21,would:[0,1,3,5,6,7,8,9,10,11,12,13,20,21,23,26,27,28,29],wrap:[2,3,4,6,14,20,26],wrap_toco:[3,4,14],wrap_util:[2,13],wrapper:[0,26],write:[0,1,2,3,5,6,7,8,12,13,15,16,17,20,21,26,27,29],written:[0,2,3,5,11,12,13,16,19,20,21,23,26,27,28,29],wrong:[1,8],wrongli:10,wrote:[5,11,27],wrt:[2,10,13],wth:[10,13],www:[19,20,21,25,26],wx_1:8,x0:8,x1:[4,8,9,10,13],x1_exampl:8,x1d:8,x2:[8,9,10,13],x2d:[8,11],x2d_train:11,x2dsl:11,x3:8,x:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,29],x_0:[0,5,11,20,26,27,28,29],x_1:[0,2,5,6,7,8,9,10,11,13,20,23,26,27,28,29],x_2:[0,2,5,6,7,8,9,10,11,13,20,23,26,27,29],x_3:[8,20,23],x_4:20,x_:[0,2,3,5,6,8,10,11,13,14,18,20,21,23,26,27,28,29],x_center:11,x_data:1,x_data_ful:1,x_hidden:2,x_i:[0,1,2,5,6,7,8,9,10,11,12,13,14,17,20,23,26,27,28,29],x_input:2,x_ix_:[0,26],x_iy_i:8,x_j:[0,2,8,9,12,16,23,27],x_jy_j:8,x_k:[12,14,20,23,27],x_l:23,x_m:[6,12,20,23,29],x_n:[0,2,3,6,8,11,12,13,20,23,26,29],x_new:[9,10],x_offset:[6,27,28],x_output:2,x_p:[3,7,9],x_poli:9,x_poly10:9,x_pred:4,x_prev:2,x_reduc:11,x_scale:8,x_small:13,x_test:[0,1,3,5,6,7,9,10,11,26,27,28,29],x_test_own:6,x_test_scal:[0,6,7,9,10,11,27,28],x_tot:4,x_train:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],x_train_mean:[6,27,28],x_train_own:6,x_train_scal:[0,6,7,9,10,11,27,28],x_val:1,xarrai:[19,26],xavier:1,xbnew:13,xcode:[0,15,19,21,26],xdclassiffierconfus:10,xdclassiffierroc:10,xeinsum:[3,4,14],xg_clf:10,xgb:10,xgbclassifi:10,xgboost:9,xgboot:10,xgbregressor:10,xgparam:10,xgtree:10,xi:[8,13],xi_1:8,xi_:8,xi_i:8,xinv:28,xk:8,xla:[3,4,13,14,19,26],xla_client:[3,4,14],xla_comput:[3,4,14],xlabel:[0,1,2,3,4,5,6,7,8,9,10,13,21,23,26,27,28,29],xlim:[6,10,29],xm:9,xmesh:13,xnew:[0,13,26],xp:23,xpanda:[0,27],xpd:[5,11,27],xplot:0,xs:9,xscale:[0,27],xsr:9,xt_x:13,xtest:[6,29],xtick:[3,6,8,9,29],xtrain:[6,29],xu:[0,26],xx:[0,20,26],xy:[0,6,8,20,21,26],xytext:8,xz:[20,26],y1:4,y2:4,y3:4,y:[0,1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,23,26,27,28,29],y_0:[0,5,11,20,26,27,28,29],y_1:[0,5,8,9,11,13,20,26,27,28,29],y_1y_1:8,y_1y_1k:8,y_1y_2:8,y_1y_2k:8,y_1y_n:8,y_1y_nk:8,y_2:[0,5,8,9,11,20,26,27],y_2y_1:8,y_2y_1k:8,y_2y_2:8,y_2y_2k:8,y_3:[0,9,20],y_4:20,y_:[0,1,5,6,10,11,20,26,27,28,29],y_data:[0,1,5,6,26,27,28],y_data_ful:1,y_decis:8,y_fit:[0,27],y_i:[0,1,5,6,7,8,9,10,11,12,13,15,16,17,18,20,21,26,27,28,29],y_if_:10,y_ix_:[0,26],y_ix_i:[7,8,13,27],y_iy_jk:8,y_j:[6,8,12,21,29],y_k:12,y_m:20,y_model:[0,4,5,6,26,27,28],y_n:[8,13],y_ny_1:8,y_ny_1k:8,y_ny_2:8,y_ny_2k:8,y_ny_n:8,y_ny_nk:8,y_offset:[6,27,28],y_plot:9,y_pred1:9,y_pred2:9,y_pred:[0,1,4,6,7,8,9,10,27,28,29],y_pred_rf:10,y_pred_tre:10,y_proba:[7,10],y_scaler:[6,28],y_test:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],y_test_onehot:1,y_test_predict:[0,27],y_tot:4,y_train:[0,1,3,4,5,6,7,9,10,11,26,27,28,29],y_train_mean:[6,27,28],y_train_onehot:1,y_train_predict:[0,27],y_train_scal:[6,28],y_val:1,ye:[3,6,7,29],year:[0,19,26],yet:[0,1,6,8,11,13,26],yhd5w:[],yi:13,yield:[0,2,5,6,8,10,12,13,14,20,23,26,28,29],yk:8,ylabel:[0,1,2,3,4,5,6,7,8,9,10,13,21,23,26,27,28,29],ylim:[3,6,29],ym:9,ymesh:13,yn:0,yo:[8,9,10],yoshua:[1,25],you:[0,1,2,3,4,5,6,8,9,10,11,13,15,16,17,19,20,21,23,24,25,27,28,29],young:[0,26],your:[1,2,4,5,6,8,11,13,17,19,20,21,28,29],yourself:[11,13,26],youtu:[],youtub:19,ypred:[6,29],ypredict2:13,ypredict:[0,13,26,27],ypredictlasso:[5,28],ypredictol:[0,5,28],ypredictown:[6,27,28],ypredictownridg:[6,28],ypredictridg:[0,5,6,28],ypredictskl:[6,27,28],yridg:26,ys:9,ytest:[6,29],ytick:[3,6,8,9,29],ytild:[0,6,26,27,29],ytildelasso:[5,28],ytildenp:[0,26,27],ytildeol:[0,5,28],ytildeownridg:[6,28],ytilderidg:[5,6,28],ytrain:[6,29],yvqgvcsovpw:[],yx:[20,26],yy:[20,26],yz:[20,26],yzbxji6lgl0:[],z:[0,1,2,3,4,5,6,7,8,9,11,12,13,20,21,23,26,27,29],z_0:[20,26],z_1:[20,26],z_2:[20,26],z_:[1,2,12,20,26],z_c:1,z_h:1,z_hidden:2,z_i:[1,12],z_j:[1,12],z_k:[12,27],z_m:1,z_mod:9,z_o:1,z_output:2,zaman:23,zaxi:[6,21],zero:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,17,18,20,21,23,26,27,28,29],zeros_lik:4,zeroth:27,zfill:4,zip:[2,4,6,13],zl:[],zm_h:[0,26],zn:[0,27],zone:[0,27],zoom:26,zx:[20,26],zy:[20,26],zz:[20,26]},titles:["3. Linear Regression","14. Building a Feed Forward Neural Network","15. Solving Differential Equations with Deep Learning","16. Convolutional Neural Networks","17. Recurrent neural networks: Overarching view","4. Ridge and Lasso Regression","5. Resampling Methods","6. Logistic Regression","8. Support Vector Machines, overarching aims","9. Decision trees, overarching aims","10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods","11. Basic ideas of the Principal Component Analysis (PCA)","13. Neural networks","7. Optimization, the central part of any Machine Learning algortithm","12. Clustering and Unsupervised Learning","Exercises week 34","Exercises week 35","Exercises week 36","Exercises week 37","Applied Data Analysis and Machine Learning","2. Linear Algebra, Handling of Arrays and more Python Features","Project 1 on Machine Learning, deadline October 9 (midnight), 2023","Teaching schedule with links to material","1. Elements of Probability Theory and Statistical Data Analysis","Teachers and Grading","Textbooks","Week 34: Introduction to the course, Logistics and Practicalities","Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression","Week 36: Statistical interpretation of Linear Regression and Resampling techniques","Week 37: Statitsitcal interpretations and Resampling Methods"],titleterms:{"1":[0,15,16,17,18,21,26,27],"10":[],"11":[],"12":[],"13":[],"14":29,"15":[],"16":[],"17":[],"18":[],"19":[],"2":[0,15,16,17,18,26,27],"20":[],"2021":[],"2022":[],"2023":[21,24],"21":[],"22":[],"23":[],"24":[],"25":[],"26":[],"27":[],"28":[],"29":[],"3":[0,15,16,26,27],"30":[],"31":27,"34":[15,26],"35":[16,27],"36":[17,28],"37":[18,29],"38":[],"39":[],"4":[0,27],"40":[],"41":[],"4155":[],"42":[],"43":[],"44":[],"45":[],"46":[],"47":[],"5":0,"6":[],"7":28,"8":[],"9":21,"case":[8,10,23,27,28],"do":[1,26,28,29],"final":[12,27,28],"function":[0,1,6,7,8,10,11,12,13,21,23,26,27,28,29],"import":[5,20,26,27,28],"new":[4,28,29],A:[0,1,4,8,9,26,28,29],And:[26,27,28],In:24,Ising:6,The:[0,1,2,3,5,6,7,8,9,11,12,17,19,26,27,28,29],To:[26,27],With:[4,28],about:[26,27],abov:28,activ:[1,12,28],actual:[],ad:[0,6,17,21,26,27],adaboost:10,adagrad:13,adam:13,adapt:10,adjust:1,adversari:4,again:[3,9],ai:26,aim:[8,9,17,18,26],aka:[26,27],algebra:[20,26],algorithm:[9,10,11,12,27],algortithm:13,all:8,an:[0,4,10,26],analys:[5,27],analysi:[0,5,6,11,19,21,23,26,27,28,29],analyt:[0,16,17],ani:13,anoth:[9,28,29],appli:19,approach:[0,8,14,26,29],approxim:12,architectur:1,arrai:[20,26],assist:24,assumpt:[28,29],august:27,autocorrel:23,autograd:[2,13],automat:13,b:[17,21],back:[1,11,12],background:[19,21,29],bag:10,base:[13,29],basic:[0,5,7,9,10,11,20,27,28,29],batch:1,bay:[5,28,29],befor:11,beta:[28,29],better:8,bia:[6,21,29],binari:1,bind:26,binomi:[],bird:10,block:[],boldsymbol:[27,28,29],boost:10,bootstrap:[6,10,29],boston:[0,27],breast:1,brief:[26,29],bring:12,build:[1,3,9],c:[21,26],calcul:27,can:[26,29],cancer:[1,7,9,11],cart:9,center:27,central:[13,19,23,29],chain:12,chang:10,channel:26,chi:[0,26],choos:1,cifar01:3,classic:11,classif:[1,9,10],classifi:8,clip:1,cluster:14,cnn:3,code:[0,1,2,5,9,11,12,13,14,26,27,28,29],collect:[1,3],commun:26,compar:[2,10],comparison:28,complet:27,complex:[0,6,21,27],complic:6,compon:11,comput:9,computation:29,computerlab:26,con:9,concept:23,condit:[28,29],confid:29,conjug:13,continu:[],contn:26,convex:[8,13],convolut:[3,12],correctli:[28,29],correl:[11,27],cost:[1,10,27,28,29],cours:[19,25,26],covari:[5,11,23,27],cover:26,cross:[6,21,29],cumul:[],cython:26,d:21,data:[0,1,3,6,7,9,11,15,16,19,21,23,26,27,28],dataset:[1,3],deadlin:[21,26],decai:2,decis:[9,10],decomposit:[5,11,17,20,27],deep:[1,2],defin:[1,26],definit:[],degre:[0,27],deliveri:21,delta:29,demonstr:[],dens:[0,26],deriv:[5,12,27,28,29],descent:[2,10,13],descript:21,design:27,detail:[3,26],develop:1,deviat:[],diagon:11,dice:[],differ:8,differenti:[2,13],diffus:2,dimension:[2,3,8,21,27],disadvantag:9,discret:23,disguis:[],distribut:[5,23,28,29],distrubut:[28,29],doe:[27,28],domain:23,down:1,dropout:1,e:21,economi:27,electron:21,element:[0,23,26],elimin:20,energi:26,ensembl:10,entropi:9,environ:[0,15,26],equat:[0,2,12,26,27,28],error:[0,10,26,27,29],essenti:26,estim:[28,29],etc:26,euler:2,evalu:1,event:[],exampl:[0,1,2,3,4,6,7,8,9,10,26,27,28,29],exercis:[0,6,15,16,17,18,26,27,29],expect:[18,23,28,29],expens:29,experi:23,explor:[0,15,16,26],exponenti:2,express:[17,18,26,27],extrapol:4,extrem:[10,26],ey:10,f:21,fall:24,famili:[1,26,27],famou:20,fantast:27,featur:[9,20,27],feed:[1,12],find:29,fine:1,first:[4,12,26,27,28],fit:[0,10,26,28],fix:27,fold:29,forc:3,forest:10,format:[21,26],forward:[1,2,12],fourier:3,frank:[6,21,27],freedom:[0,27],frequent:27,frequentist:[0,26],from:[5,10,12,27,28,29],full:2,further:[3,5,27],fy:[],g:21,gan:4,gaussian:20,gd:13,gener:[4,9,26],geometr:11,gini:9,good:[0,26],grade:[24,26],gradient:[1,2,10,13],growth:2,ha:19,handl:[20,26,27],happen:[28,29],have:26,heard:26,hessian:27,hidden:2,histogram:29,hous:[0,27],how:[],hyperparamet:1,hyperplan:8,i:1,id3:9,idea:11,ident:[28,29],identifi:29,ii:26,iid:[28,29],illustr:28,implement:1,implic:[5,27],improv:1,includ:13,increment:11,independ:[28,29],index:9,inform:24,input:2,instal:[19,21,26],instructor:24,intercept:27,interpret:[5,11,26,27,28,29],interv:29,introduc:[11,13,27],introduct:[0,6,19,20,21,26],invers:[5,20,28],invert:27,invok:[],iter:10,its:27,jackknif:[],jacobian:27,jax:13,julia:26,jungl:10,k:29,kera:[1,3],kernel:[8,11],lagrangian:8,lasso:[5,6,21,27,28,29],last:[27,29],later:[5,27],layer:[1,2,3,12],learn:[0,1,2,11,13,14,15,16,19,21,26,27,28,29],least:[5,6,18,21,26,27,28],lectur:[26,27,28,29],level:10,librari:[19,26],likelihood:[7,28,29],limit:[1,13,23,29],linear:[0,8,13,20,26,27,28],link:[5,11,22,25,27,28,29],literatur:21,logist:[7,26],loss:27,lu:20,machin:[0,8,13,19,21,26],made:[28,29],mai:26,main:23,make:[0,9,10,15,16,26,27],mani:[10,12],manipul:27,margin:[28,29],mass:26,materi:[22,26,27,28,29],math:[5,27],mathemat:[3,5,8,27],matric:[5,20,26,28],matrix:[1,5,11,12,20,26,27,28],matter:[0,26],max:27,maximum:[28,29],mean:[0,26,27,28],meet:[5,10,23,26,27],mercer:8,mersenn:[],method:[6,9,10,13,21,29],midnight:21,min:27,minim:26,ml:26,mle:[28,29],mlp:12,mnist:[3,4],model:[0,1,4,6,12,26],moment:[],momentum:13,moon:[8,9],more:[3,6,20,21,26,27,28,29],multilay:12,multipl:[1,3],multipli:8,name:[],need:[21,26],network:[1,2,3,4,7,12,26],neural:[1,2,3,4,7,12,26],non:8,normal:[0,1,29],norwai:[],notat:12,note:[27,28],novemb:[],now:[1,9,13,28,29],nuclear:[0,26],nueral:[],numba:26,number:[0,2,23,27],numer:[2,21,23],numpi:[20,26],object:3,observ:[],obtain:11,octob:21,od:2,off:[6,21],ol:[5,6,21,28,29],one:[2,12],oper:20,optim:[1,8,13,19,26,27],order:13,ordinari:[5,6,18,21,26,27,28],organ:[0,26],oslo:25,other:[4,9,11,12,20,26,27],our:[0,4,5,11,13,26,27],outcom:[19,26],output:2,overarch:[0,4,8,9,17,18,26,27],overview:[10,26],own:[0,10,11,15,16,26,27],packag:[20,26],panda:[26,27],paper:21,paramet:[26,27],part:[13,19,21,27],partial:2,pass:1,pca:11,pdf:23,pencil:21,perceptron:12,perform:[1,9],period:3,perspect:1,plan:[27,28,29],plot:29,point:4,poisson:2,polynomi:[3,28],popul:2,practic:[13,24,26],pre:[1,3],predict:4,preprocess:27,prerequisit:[3,19,26],princip:11,principl:3,pro:9,probabl:[5,23,28,29],problem:[1,2,13,26,27,28],procedur:[9,26],process:[1,3],program:[2,13,21],project:[6,21,26],prop:13,propag:[1,12],properti:[5,23,27],pseudo:[],python:[0,9,15,19,20,26],quick:8,r:26,ran0:[],random:[10,11,23],read:[9,26,27,29],real:[6,21,26],recip:[],recommend:[26,27],rectangular:28,recurr:[4,12],reduc:[0,27],reduct:3,reformul:2,regress:[0,5,6,7,9,10,13,17,18,21,26,27,28,29],regular:1,relat:27,relev:[25,27,29],relu:1,remark:3,remind:[6,8,26],repeat:27,replac:13,report:21,repositori:29,requir:[2,19],resampl:[6,21,28,29],rescal:[6,28],residu:27,resourc:2,result:[27,28],revisit:13,rewrit:[26,27,29],ridg:[0,5,6,17,18,21,27,28,29],rm:13,rng:[],rule:12,s:[8,10,29],same:[13,29],sampl:11,scale:27,schedul:[22,26],schemat:9,scheme:2,scienc:26,scikit:[0,1,11,15,16,26,27,28,29],second:13,select:[],semest:24,septemb:[28,29],session:28,set:[0,2,3,9,12,15,26,27],sgd:13,should:1,similar:13,simpl:[0,4,9,13,26,27,28],singl:10,singular:[5,11,17,27],situat:[],size:27,soft:8,softmax:1,softwar:[21,26],solv:[2,28],solver:13,some:[13,20,26,27],specifi:2,split:[0,15,16,26,27],squar:[0,5,6,10,18,21,26,27,28],standard:[13,27,29],state:[0,26],statement:26,statist:[5,6,19,23,26,28,29],statitsitc:29,steepest:[10,13],step:29,still:27,stk3155:[],stochast:[13,23],strongli:26,subtract:27,suggest:26,sum:29,summari:[24,26],superposit:3,supervis:1,support:8,svd:[5,27,28],systemat:3,t:[27,28],teach:[22,24],teacher:[24,26],technic:28,techniqu:[6,11,21,28],technolog:19,tensorflow:[1,3],tent:26,term:29,test:[0,1,15,16,26,27,28],textbook:[25,26],thei:26,theorem:[5,8,11,12,23,28,29],theori:23,thi:[17,18,26],think:27,three:[],thursdai:[27,28,29],tip:13,togeth:12,tool:26,top:1,topic:26,toss:[],toward:11,trade:[6,21],tradeoff:[6,29],train:[0,1,4,15,16,26,27],transform:3,tree:[9,10],tuesdai:28,tune:1,two:[3,8,19,21,27],type:[2,4,12,26],uio:26,uncorrel:[],understand:29,uniform:[],univers:[12,25],unsupervis:14,up:[0,2,9,12,15,26,27,29],us:[0,1,2,3,7,13,19,26,27],usag:[28,29],valid:[6,21,29],valu:[5,11,17,18,23,27,28,29],variabl:23,varianc:[6,21,28,29],variou:[0,15,26,29],vector:[8,12,20,26,27],view:[0,4,10,27],visual:[1,9],vs:3,wai:[9,29],wave:2,we:26,wednesdai:28,week:[15,16,17,18,26,27,28,29],weekli:[22,29],what:[0,26,27,28,29],which:1,why:[26,27,29],wisconsin:7,wrap:[27,29],write:[4,11,28],x:[27,28],xgboost:10,yet:28,you:26,your:[0,10,15,16,26,27]}}) \ No newline at end of file diff --git a/doc/LectureNotes/_build/html/week35.html b/doc/LectureNotes/_build/html/week35.html index b002b3156..11c1e9409 100644 --- a/doc/LectureNotes/_build/html/week35.html +++ b/doc/LectureNotes/_build/html/week35.html @@ -1646,7 +1646,7 @@ Since we are not using Scikit-Learn here we can define our own
-
0.996535469511469
+
0.9952714635635723
@@ -1663,7 +1663,7 @@ Since we are not using Scikit-Learn here we can define our own
-
0.0077750600806588575
+
0.009039197557276654
@@ -1678,23 +1678,31 @@ Since we are not using Scikit-Learn here we can define our own
-
[0.05751737 0.02644748 0.02533184 0.02193035 0.02521356 0.00712899
- 0.00857688 0.0101339  0.01097227 0.01132947 0.01644452 0.01655455
- 0.00771046 0.05177021 0.01374437 0.04208274 0.01667873 0.00228986
- 0.00749579 0.00852965 0.01628823 0.0341899  0.01178253 0.00626339
- 0.00514913 0.00791019 0.00363986 0.00471966 0.00390805 0.00910241
- 0.00232255 0.02670606 0.03224523 0.01637147 0.00918914 0.02705154
- 0.00321694 0.01904583 0.0177181  0.00538287 0.02291121 0.01028255
- 0.04566103 0.00954257 0.00011551 0.0283589  0.00953728 0.03399521
- 0.01205286 0.02251524 0.00353655 0.02344605 0.05288681 0.01950574
- 0.00330907 0.01492855 0.01472766 0.03952093 0.01491299 0.00187154
- 0.02897168 0.00072037 0.00895315 0.02775293 0.01547992 0.04117184
- 0.0116726  0.03330335 0.01515966 0.0070381  0.01317074 0.00654702
- 0.00329264 0.02502101 0.00226569 0.05029894 0.01929643 0.03028379
- 0.053994   0.03413302 0.01854824 0.00393744 0.0600658  0.01855624
- 0.0590702  0.01826743 0.01039549 0.02151219 0.00928016 0.03536062
- 0.00503218 0.08526352 0.00506765 0.02609885 0.04771105 0.0010059
- 0.00659545 0.00143188 0.01489692 0.08237141]
+
[6.11430166e-02 7.59428035e-03 7.64295978e-03 5.34442654e-02
+ 2.53861506e-02 7.46543658e-02 2.79339933e-02 1.32946226e-02
+ 3.80320028e-03 2.40474117e-02 4.36028477e-02 3.24382528e-02
+ 1.26022420e-02 5.06848116e-03 1.96793569e-02 1.04826413e-04
+ 9.25580178e-03 1.80413812e-02 1.22519792e-02 1.01505087e-02
+ 2.67180442e-02 6.71316703e-02 7.68415876e-03 3.33174939e-02
+ 1.43570786e-02 8.43459933e-03 4.11046407e-03 9.10453697e-04
+ 4.33142087e-03 7.64507671e-02 2.19696497e-02 1.18460525e-02
+ 4.35665906e-02 3.62818572e-02 1.61018347e-03 3.78738205e-02
+ 4.57031517e-03 3.11279984e-02 1.07745809e-02 7.09848429e-03
+ 2.30006341e-03 4.62535124e-02 6.70493970e-02 6.26624267e-03
+ 4.23399553e-02 1.88690267e-02 7.46184816e-03 1.75934097e-02
+ 3.60820168e-02 1.09322328e-02 3.16749353e-02 2.30626515e-02
+ 5.76196895e-04 1.51000420e-02 1.95456434e-03 1.22138768e-02
+ 2.52945186e-03 2.12804129e-03 3.32695433e-03 1.90886410e-02
+ 1.84291049e-02 1.78104004e-02 6.34498455e-02 1.50032818e-02
+ 3.43525003e-02 1.52033394e-03 3.95921014e-02 7.67148008e-02
+ 2.40893740e-02 6.75638425e-02 1.22105472e-02 7.72706640e-03
+ 3.00721135e-02 8.77081499e-02 1.40684484e-02 4.57081718e-02
+ 3.51190686e-02 7.42092776e-02 4.03105744e-02 1.08994018e-02
+ 1.37071826e-02 3.69106251e-02 1.94966662e-03 1.71112140e-02
+ 3.12669602e-03 3.01610677e-02 1.72139847e-02 2.75703556e-02
+ 2.23333773e-02 2.76892410e-03 8.05886306e-02 1.15545946e-01
+ 4.88029792e-02 4.59648068e-02 1.98750794e-02 2.69178261e-03
+ 1.07925220e-02 1.13151043e-03 5.00620642e-03 3.69871367e-03]
@@ -1763,15 +1771,15 @@ but now splitting the data into a training set and a test set.

-
[ 1.95976303  0.48974963  2.82202765  3.79224555 -2.14248744]
+
[ 2.04038959 -0.81343612  7.94304103 -3.83770135  1.70073443]
 Training R2
-0.9947613223847728
+0.9954441212986554
 Training MSE
-0.010596520060222926
+0.008653750612762521
 Test R2
-0.989645675273948
+0.99255249100346
 Test MSE
-0.01889099091713886
+0.016970019460234076
diff --git a/doc/LectureNotes/_build/html/week36.html b/doc/LectureNotes/_build/html/week36.html index 3bad39527..d9983eaef 100644 --- a/doc/LectureNotes/_build/html/week36.html +++ b/doc/LectureNotes/_build/html/week36.html @@ -519,6 +519,16 @@ const thebe_selector_output = ".output, .cell_output" Material for lecture Thursday September 7 +
  • + + Important technicalities: More on Rescaling data + +
    • +
  • + + Test Function for what happens with OLS, Ridge and Lasso + +
  • Linking the regression analysis with a statistical interpretation @@ -602,16 +612,6 @@ const thebe_selector_output = ".output, .cell_output" Bayes’ Theorem and Ridge and Lasso Regression
    • -
  • - - Test Function for what happens with OLS, Ridge and Lasso - -
    • -
  • - - Invoking Bayes’ theorem - -
  • Ridge and Bayes @@ -790,6 +790,16 @@ const thebe_selector_output = ".output, .cell_output" Material for lecture Thursday September 7
    • +
  • + + Important technicalities: More on Rescaling data + +
    • +
  • + + Test Function for what happens with OLS, Ridge and Lasso + +
  • Linking the regression analysis with a statistical interpretation @@ -873,16 +883,6 @@ const thebe_selector_output = ".output, .cell_output" Bayes’ Theorem and Ridge and Lasso Regression
    • -
  • - - Test Function for what happens with OLS, Ridge and Lasso - -
    • -
  • - - Invoking Bayes’ theorem - -
  • Ridge and Bayes @@ -920,16 +920,19 @@ doconce format html week36.do.txt --no_mako -->

  • Material for the lecture on Thursday September 7

    • Material for the active learning sessions Tuesday and Wednesday

    -

    The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples

    +

    The material here contains a summary from last week and discussion of SVD, Ridge and Lasso regression with examples

    Linear Regression and the SVD

    @@ -1700,6 +1703,531 @@ C(\boldsymbol{\beta})=(4-2\beta_0)^2+(2-\beta_1)^2+\lambda(\vert\beta_0\vert+\ve

    Material for lecture Thursday September 7

    +
    +

    Important technicalities: More on Rescaling data

    +

    When you are comparing your own code with for example Scikit-Learn’s +library, there are some technicalities to keep in mind. The examples +here demonstrate some of these aspects with potential pitfalls.

    +

    The discussion here focuses on the role of the intercept, how we can +set up the design matrix, what scaling we should use and other topics +which tend confuse us.

    +

    The intercept can be interpreted as the expected value of our +target/output variables when all other predictors are set to zero. +Thus, if we cannot assume that the expected outputs/targets are zero +when all predictors are zero (the columns in the design matrix), it +may be a bad idea to implement a model which penalizes the intercept. +Furthermore, in for example Ridge and Lasso regression, the default solutions +from the library Scikit-Learn (when not shrinking \(\beta_0\)) for the unknown parameters +\(\boldsymbol{\beta}\), are derived under the assumption that both \(\boldsymbol{y}\) and +\(\boldsymbol{X}\) are zero centered, that is we subtract the mean values.

    +

    If our predictors represent different scales, then it is important to +standardize the design matrix \(\boldsymbol{X}\) by subtracting the mean of each +column from the corresponding column and dividing the column with its +standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, +the results may differ.

    +

    The +Standardscaler +function in Scikit-Learn does this for us. For the data sets we +have been studying in our various examples, the data are in many cases +already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a +survey of your data, with a critical assessment of them in case you need to scale the data.

    +

    If you need to scale the data, not doing so will give an unfair +penalization of the parameters since their magnitude depends on the +scale of their corresponding predictor.

    +

    Suppose as an example that you +you have an input variable given by the heights of different persons. +Human height might be measured in inches or meters or +kilometers. If measured in kilometers, a standard linear regression +model with this predictor would probably give a much bigger +coefficient term, than if measured in millimeters. +This can clearly lead to problems in evaluating the cost/loss functions.

    +

    Keep in mind that when you transform your data set before training a model, the same transformation needs to be done +on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as

    +
    +
    +
    """
+#Model training, we compute the mean value of y and X
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+# The we fit our model with the training data
+trained_model = some_model.fit(X_train,y_train)
+
+
+#Model prediction, we need also to transform our data set used for the prediction.
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+"""
+
    +
    +
    +
    +

    Let us try to understand what this may imply mathematically when we +subtract the mean values, also known as zero centering. For +simplicity, we will focus on ordinary regression, as done in the above example.

    +

    The cost/loss function for regression is

    +
    +\[ +C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. +\]
    +

    Recall also that we use the squared value. This expression can lead to an +increased penalty for higher differences between predicted and +output/target values.

    +

    What we have done is to single out the \(\beta_0\) term in the +definition of the mean squared error (MSE). The design matrix \(X\) +does in this case not contain any intercept column. When we take the +derivative with respect to \(\beta_0\), we want the derivative to obey

    +
    +\[ +\frac{\partial C}{\partial \beta_j} = 0, +\]
    +

    for all \(j\). For \(\beta_0\) we have

    +
    +\[ +\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). +\]
    +

    Multiplying away the constant \(2/n\), we obtain

    +
    +\[ +\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. +\]
    +

    Let us specialize first to the case where we have only two parameters \(\beta_0\) and \(\beta_1\). +Our result for \(\beta_0\) simplifies then to

    +
    +\[ +n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. +\]
    +

    We obtain then

    +
    +\[ +\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. +\]
    +

    If we define

    +
    +\[ +\mu_{\boldsymbol{x}_1}=\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}, +\]
    +

    and the mean value of the outputs as

    +
    +\[ +\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, +\]
    +

    we have

    +
    +\[ +\beta_0 = \mu_y - \beta_1\mu_{\boldsymbol{x}_1}. +\]
    +

    In the general case with more parameters than \(\beta_0\) and \(\beta_1\), we have

    +
    +\[ +\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. +\]
    +

    We can rewrite the latter equation as

    +
    +\[ +\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \sum_{j=1}^{p-1} \mu_{\boldsymbol{x}_j}\beta_j, +\]
    +

    where we have defined

    +
    +\[ +\mu_{\boldsymbol{x}_j}=\frac{1}{n}\sum_{i=0}^{n-1} X_{ij}, +\]
    +

    the mean value for all elements of the column vector \(\boldsymbol{x}_j\).

    +

    Replacing \(y_i\) with \(y_i - y_i - \overline{\boldsymbol{y}}\) and centering also our design matrix results in a cost function (in vector-matrix disguise)

    +
    +\[ +C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). +\]
    +

    If we minimize with respect to \(\boldsymbol{\beta}\) we have then

    +
    +\[ +\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, +\]
    +

    where \(\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}}\) +and \(\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}\).

    +

    For Ridge regression we need to add \(\lambda \boldsymbol{\beta}^T\boldsymbol{\beta}\) to the cost function and get then

    +
    +\[ +\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. +\]
    +

    What does this mean? And why do we insist on all this? Let us look at some examples.

    +

    This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only. +Note also that we do not split the data into training and test.

    +
    +
    +
    import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+def fit_beta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_beta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+beta = fit_beta(X, y)
+
+# Intercept is included in the design matrix
+skl = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True beta: {true_beta}")
+print(f"Fitted beta: {beta}")
+print(f"Sklearn fitted beta: {skl.coef_}")
+ypredictOwn = X @ beta
+ypredictSKL = skl.predict(X)
+print(f"MSE with intercept column")
+print(MSE(y,ypredictOwn))
+print(f"MSE with intercept column from SKL")
+print(MSE(y,ypredictSKL))
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ beta, label="Fit")
+plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+skl = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+beta = fit_beta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ beta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted beta (wiothout intercept): {beta}")
+print(f"Sklearn intercept: {skl.intercept_}")
+print(f"Sklearn fitted beta (without intercept): {skl.coef_}")
+ypredictOwn = X @ beta
+ypredictSKL = skl.predict(X)
+print(f"MSE with Manual intercept")
+print(MSE(y,ypredictOwn+intercept))
+print(f"MSE with Sklearn intercept")
+print(MSE(y,ypredictSKL))
+
+plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
+plt.show()
+
    +
    +
    +
    +

    The intercept is the value of our output/target variable +when all our features are zero and our function crosses the \(y\)-axis (for a one-dimensional case).

    +

    Printing the MSE, we see first that both methods give the same MSE, as +they should. However, when we move to for example Ridge regression, +the way we treat the intercept may give a larger or smaller MSE, +meaning that the MSE can be penalized by the value of the +intercept. Not including the intercept in the fit, means that the +regularization term does not include \(\beta_0\). For different values +of \(\lambda\), this may lead to different MSE values.

    +

    To remind the reader, the regularization term, with the intercept in Ridge regression, is given by

    +
    +\[ +\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, +\]
    +

    but when we take out the intercept, this equation becomes

    +
    +\[ +\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. +\]
    +

    For Lasso regression we have

    +
    +\[ +\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. +\]
    +

    It means that, when scaling the design matrix and the outputs/targets, +by subtracting the mean values, we have an optimization problem which +is not penalized by the intercept. The MSE value can then be smaller +since it focuses only on the remaining quantities. If we however bring +back the intercept, we will get a MSE which then contains the +intercept.

    +

    Armed with this wisdom, we attempt first to simply set the intercept equal to False in our implementation of Ridge regression for our well-known vanilla data set.

    +
    +
    +
    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 20
+X = np.zeros((n,Maxpolydegree))
+#We include explicitely the intercept column
+for degree in range(Maxpolydegree):
+    X[:,degree] = x**degree
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+p = Maxpolydegree
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 6
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 2, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
+    # Note: we include the intercept column and no scaling
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    # and then make the prediction
+    ytildeOwnRidge = X_train @ OwnRidgeBeta
+    ypredictOwnRidge = X_test @ OwnRidgeBeta
+    ytildeRidge = RegRidge.predict(X_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta)
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print("MSE values for own Ridge implementation")
+    print(MSEOwnRidgePredict[i])
+    print("MSE values for Scikit-Learn Ridge implementation")
+    print(MSERidgePredict[i])
+
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +
    +
    +
    +

    The results here agree when we force Scikit-Learn’s Ridge function to include the first column in our design matrix. +We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix. +What happens if we do not include the intercept in our fit? +Let us see how we can change this code by zero centering.

    +
    +
    +
    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 20
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+#Center by removing mean from each feature
+X_train_scaled = X_train - X_train_mean 
+X_test_scaled = X_test - X_train_mean
+#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)
+#Remove the intercept from the training data.
+y_scaler = np.mean(y_train)           
+y_train_scaled = y_train - y_scaler   
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 6
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 2, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data
+    #Add intercept to prediction
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler 
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta) #Intercept is given by mean of target variable
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+    print("MSE values for own Ridge implementation")
+    print(MSEOwnRidgePredict[i])
+    print("MSE values for Scikit-Learn Ridge implementation")
+    print(MSERidgePredict[i])
+
+
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +
    +
    +
    +

    We see here, when compared to the code which includes explicitely the +intercept column, that our MSE value is actually smaller. This is +because the regularization term does not include the intercept value +\(\beta_0\) in the fitting. This applies to Lasso regularization as +well. It means that our optimization is now done only with the +centered matrix and/or vector that enter the fitting procedure.

    +
    +
    +

    Test Function for what happens with OLS, Ridge and Lasso

    +

    Hitherto we have discussed Ridge and Lasso regression in terms of a +linear analysis. This may to many of you feel rather technical and +perhaps not that intuitive. The question is whether we can develop a +more intuitive way of understanding what Ridge and Lasso express.

    +

    Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit.

    +

    We will play around with a study of the values for the optimal +parameters \(\boldsymbol{\beta}\) using OLS, Ridge and Lasso regression. For +OLS, you will notice as function of the noise and polynomial degree, +that the parameters \(\beta\) will fluctuate from order to order in the +polynomial fit and that for larger and larger polynomial degrees of freedom, the parameters will tend to increase in value for OLS.

    +

    For Ridge and Lasso regression, the higher order parameters will typically be reduced, providing thereby less fluctuations from one order to another one.

    +
    +
    +
    import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+# Make data set.
+n = 10000
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.randn(n)
+
+Maxpolydegree = 5
+X = np.zeros((len(x),Maxpolydegree))
+X[:,0] = 1.0
+
+
+for polydegree in range(1,Maxpolydegree):
+    X[:,polydegree] = x**(polydegree)
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.pinv(X_train.T @ X_train) @ X_train.T @ y_train
+print(OLSbeta)
+ypredictOLS = X_test @ OLSbeta
+print("Test MSE OLS")
+print(MSE(y_test,ypredictOLS))
+# Repeat now for Lasso and Ridge regression and various values of the regularization parameter using Scikit-Learn
+# Decide which values of lambda to use
+nlambdas = 4
+MSERidgePredict = np.zeros(nlambdas)
+MSELassoPredict = np.zeros(nlambdas)
+lambdas = np.logspace(-3, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    # Make the fit using Ridge and Lasso
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    RegLasso = linear_model.Lasso(lmb,fit_intercept=False)
+    RegLasso.fit(X_train,y_train)
+    # and then make the prediction
+    ypredictRidge = RegRidge.predict(X_test)
+    ypredictLasso = RegLasso.predict(X_test)
+    # Compute the MSE and print it
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    MSELassoPredict[i] = MSE(y_test,ypredictLasso)
+    print(lmb,RegRidge.coef_)
+    print(lmb,RegLasso.coef_)
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSERidgePredict, 'b', label = 'MSE Ridge Test')
+plt.plot(np.log10(lambdas), MSELassoPredict, 'r', label = 'MSE Lasso Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +
    +
    +
    +

    How can we understand this?

    +

    Linking the regression analysis with a statistical interpretation

    We will now couple the discussions of ordinary least squares, Ridge @@ -2036,93 +2564,6 @@ p(Y=1\vert X=1)=\frac{p(X=1\vert Y=1)p(Y=1)}{p(X=1\vert Y=1)p(Y=1)+p(X=1\vert Y=

    Bayes’ Theorem and Ridge and Lasso Regression

    -

    Hitherto we have discussed Ridge and Lasso regression in terms of a -linear analysis. This may to many of you feel rather technical and -perhaps not that intuitive. The question is whether we can develop a -more intuitive way of understanding what Ridge and Lasso express.

    -

    Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit.

    -
    -
    -

    Test Function for what happens with OLS, Ridge and Lasso

    -

    We will play around with a study of the values for the optimal -parameters \(\boldsymbol{\beta}\) using OLS, Ridge and Lasso regression. For -OLS, you will notice as function of the noise and polynomial degree, -that the parameters \(\beta\) will fluctuate from order to order in the -polynomial fit and that for larger and larger polynomial degrees of freedom, the parameters will tend to increase in value for OLS.

    -

    For Ridge and Lasso regression, the higher order parameters will typically be reduced, providing thereby less fluctuations from one order to another one.

    -
    -
    -
    import numpy as np
-import matplotlib.pyplot as plt
-from sklearn.model_selection import train_test_split
-from sklearn import linear_model
-
-def R2(y_data, y_model):
-    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
-def MSE(y_data,y_model):
-    n = np.size(y_model)
-    return np.sum((y_data-y_model)**2)/n
-
-# Make data set.
-n = 10000
-x = np.random.rand(n)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.randn(n)
-
-Maxpolydegree = 5
-X = np.zeros((len(x),Maxpolydegree))
-X[:,0] = 1.0
-
-for polydegree in range(1, Maxpolydegree):
-    for degree in range(polydegree):
-        X[:,degree] = x**(degree)
-
-
-# We split the data in test and training data
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
-
-# matrix inversion to find beta
-OLSbeta = np.linalg.pinv(X_train.T @ X_train) @ X_train.T @ y_train
-print(OLSbeta)
-ypredictOLS = X_test @ OLSbeta
-print("Test MSE OLS")
-print(MSE(y_test,ypredictOLS))
-# Repeat now for Lasso and Ridge regression and various values of the regularization parameter using Scikit-Learn
-# Decide which values of lambda to use
-nlambdas = 4
-MSERidgePredict = np.zeros(nlambdas)
-MSELassoPredict = np.zeros(nlambdas)
-lambdas = np.logspace(-3, 1, nlambdas)
-for i in range(nlambdas):
-    lmb = lambdas[i]
-    # Make the fit using Ridge and Lasso
-    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
-    RegRidge.fit(X_train,y_train)
-    RegLasso = linear_model.Lasso(lmb,fit_intercept=False)
-    RegLasso.fit(X_train,y_train)
-    # and then make the prediction
-    ypredictRidge = RegRidge.predict(X_test)
-    ypredictLasso = RegLasso.predict(X_test)
-    # Compute the MSE and print it
-    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
-    MSELassoPredict[i] = MSE(y_test,ypredictLasso)
-    print(lmb,RegRidge.coef_)
-    print(lmb,RegLasso.coef_)
-# Now plot the results
-plt.figure()
-plt.plot(np.log10(lambdas), MSERidgePredict, 'b', label = 'MSE Ridge Test')
-plt.plot(np.log10(lambdas), MSELassoPredict, 'r', label = 'MSE Lasso Test')
-plt.xlabel('log10(lambda)')
-plt.ylabel('MSE')
-plt.legend()
-plt.show()
-
    -
    -
    -
    -

    How can we understand this?

    -
    -
    -

    Invoking Bayes’ theorem

    Using Bayes’ theorem we can gain a better intuition about Ridge and Lasso regression.

    For ordinary least squares we postulated that the maximum likelihood for the doamin of events \(\boldsymbol{D}\) (one-dimensional case)

    diff --git a/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb b/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb index cc9e06cfc..2efe96e13 100644 --- a/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb +++ b/doc/LectureNotes/_build/jupyter_execute/exercisesweek37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "cfebae03", + "id": "1ff4a4e9", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "353ebb07", + "id": "0569f7a9", "metadata": { "editable": true }, @@ -27,25 +27,34 @@ }, { "cell_type": "markdown", - "id": "c02815dd", + "id": "6f34e507", "metadata": { "editable": true }, "source": [ "## Overarching aims of the exercises this week\n", "\n", - "This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercise is also a part of project 1 and can be reused in the theory part of the project.\n", + "This exercise deals with various mean values and variances in linear\n", + "regression method (here it may be useful to look up chapter 3,\n", + "equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome\n", + "H. Friedman, The Elements of Statistical Learning,\n", + "Springer](https://www.springer.com/gp/book/9780387848570)). The\n", + "exercise is also a part of project 1 and can be reused in the theory\n", + "part of the project.\n", "\n", - "For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "For more discussions on Ridge regression and calculation of\n", + "expectation values, [Wessel van\n", + "Wieringen's](https://arxiv.org/abs/1509.09169) article is highly\n", + "recommended.\n", "\n", - "The assumption we have made is \n", - "that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$\n", - "which describes our data" + "The assumption we have made is that there exists a continuous function\n", + "$f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0,\n", + "\\sigma^2)$ which describes our data" ] }, { "cell_type": "markdown", - "id": "fb152043", + "id": "3ff574a4", "metadata": { "editable": true }, @@ -57,7 +66,7 @@ }, { "cell_type": "markdown", - "id": "9aba7a46", + "id": "096a3165", "metadata": { "editable": true }, @@ -68,7 +77,7 @@ }, { "cell_type": "markdown", - "id": "be52c097", + "id": "cc9fb258", "metadata": { "editable": true }, @@ -80,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "c54979e5", + "id": "daf4eb5d", "metadata": { "editable": true }, @@ -90,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "4b1eb811", + "id": "3b491679", "metadata": { "editable": true }, @@ -102,7 +111,7 @@ }, { "cell_type": "markdown", - "id": "036a5440", + "id": "93a49739", "metadata": { "editable": true }, @@ -114,7 +123,7 @@ }, { "cell_type": "markdown", - "id": "f7fb8094", + "id": "a3fa80db", "metadata": { "editable": true }, @@ -125,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "d7d92995", + "id": "e6e2defd", "metadata": { "editable": true }, @@ -137,7 +146,7 @@ }, { "cell_type": "markdown", - "id": "29eb9701", + "id": "9aa19afa", "metadata": { "editable": true }, @@ -150,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "aa106511", + "id": "5fa82a47", "metadata": { "editable": true }, @@ -162,7 +171,7 @@ }, { "cell_type": "markdown", - "id": "58be1091", + "id": "33099abb", "metadata": { "editable": true }, @@ -172,7 +181,7 @@ }, { "cell_type": "markdown", - "id": "4cb58e41", + "id": "5d03f001", "metadata": { "editable": true }, @@ -184,7 +193,7 @@ }, { "cell_type": "markdown", - "id": "a04df28d", + "id": "94bd151d", "metadata": { "editable": true }, @@ -195,7 +204,7 @@ }, { "cell_type": "markdown", - "id": "c095c96a", + "id": "f0b84941", "metadata": { "editable": true }, @@ -207,20 +216,20 @@ }, { "cell_type": "markdown", - "id": "5d425a79", + "id": "60370a63", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", + "\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", "\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "fb89cfab", + "id": "af251217", "metadata": { "editable": true }, @@ -233,19 +242,19 @@ }, { "cell_type": "markdown", - "id": "aea8801c", + "id": "02e978da", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\mbox{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e885693", + "id": "3656a896", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/jupyter_execute/week35.ipynb b/doc/LectureNotes/_build/jupyter_execute/week35.ipynb index fcd45c57e..71e93c686 100644 --- a/doc/LectureNotes/_build/jupyter_execute/week35.ipynb +++ b/doc/LectureNotes/_build/jupyter_execute/week35.ipynb @@ -1519,7 +1519,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "0.996535469511469\n" + "0.9952714635635723\n" ] } ], @@ -1550,7 +1550,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "0.0077750600806588575\n" + "0.009039197557276654\n" ] } ], @@ -1585,23 +1585,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "[0.05751737 0.02644748 0.02533184 0.02193035 0.02521356 0.00712899\n", - " 0.00857688 0.0101339 0.01097227 0.01132947 0.01644452 0.01655455\n", - " 0.00771046 0.05177021 0.01374437 0.04208274 0.01667873 0.00228986\n", - " 0.00749579 0.00852965 0.01628823 0.0341899 0.01178253 0.00626339\n", - " 0.00514913 0.00791019 0.00363986 0.00471966 0.00390805 0.00910241\n", - " 0.00232255 0.02670606 0.03224523 0.01637147 0.00918914 0.02705154\n", - " 0.00321694 0.01904583 0.0177181 0.00538287 0.02291121 0.01028255\n", - " 0.04566103 0.00954257 0.00011551 0.0283589 0.00953728 0.03399521\n", - " 0.01205286 0.02251524 0.00353655 0.02344605 0.05288681 0.01950574\n", - " 0.00330907 0.01492855 0.01472766 0.03952093 0.01491299 0.00187154\n", - " 0.02897168 0.00072037 0.00895315 0.02775293 0.01547992 0.04117184\n", - " 0.0116726 0.03330335 0.01515966 0.0070381 0.01317074 0.00654702\n", - " 0.00329264 0.02502101 0.00226569 0.05029894 0.01929643 0.03028379\n", - " 0.053994 0.03413302 0.01854824 0.00393744 0.0600658 0.01855624\n", - " 0.0590702 0.01826743 0.01039549 0.02151219 0.00928016 0.03536062\n", - " 0.00503218 0.08526352 0.00506765 0.02609885 0.04771105 0.0010059\n", - " 0.00659545 0.00143188 0.01489692 0.08237141]\n" + "[6.11430166e-02 7.59428035e-03 7.64295978e-03 5.34442654e-02\n", + " 2.53861506e-02 7.46543658e-02 2.79339933e-02 1.32946226e-02\n", + " 3.80320028e-03 2.40474117e-02 4.36028477e-02 3.24382528e-02\n", + " 1.26022420e-02 5.06848116e-03 1.96793569e-02 1.04826413e-04\n", + " 9.25580178e-03 1.80413812e-02 1.22519792e-02 1.01505087e-02\n", + " 2.67180442e-02 6.71316703e-02 7.68415876e-03 3.33174939e-02\n", + " 1.43570786e-02 8.43459933e-03 4.11046407e-03 9.10453697e-04\n", + " 4.33142087e-03 7.64507671e-02 2.19696497e-02 1.18460525e-02\n", + " 4.35665906e-02 3.62818572e-02 1.61018347e-03 3.78738205e-02\n", + " 4.57031517e-03 3.11279984e-02 1.07745809e-02 7.09848429e-03\n", + " 2.30006341e-03 4.62535124e-02 6.70493970e-02 6.26624267e-03\n", + " 4.23399553e-02 1.88690267e-02 7.46184816e-03 1.75934097e-02\n", + " 3.60820168e-02 1.09322328e-02 3.16749353e-02 2.30626515e-02\n", + " 5.76196895e-04 1.51000420e-02 1.95456434e-03 1.22138768e-02\n", + " 2.52945186e-03 2.12804129e-03 3.32695433e-03 1.90886410e-02\n", + " 1.84291049e-02 1.78104004e-02 6.34498455e-02 1.50032818e-02\n", + " 3.43525003e-02 1.52033394e-03 3.95921014e-02 7.67148008e-02\n", + " 2.40893740e-02 6.75638425e-02 1.22105472e-02 7.72706640e-03\n", + " 3.00721135e-02 8.77081499e-02 1.40684484e-02 4.57081718e-02\n", + " 3.51190686e-02 7.42092776e-02 4.03105744e-02 1.08994018e-02\n", + " 1.37071826e-02 3.69106251e-02 1.94966662e-03 1.71112140e-02\n", + " 3.12669602e-03 3.01610677e-02 1.72139847e-02 2.75703556e-02\n", + " 2.23333773e-02 2.76892410e-03 8.05886306e-02 1.15545946e-01\n", + " 4.88029792e-02 4.59648068e-02 1.98750794e-02 2.69178261e-03\n", + " 1.07925220e-02 1.13151043e-03 5.00620642e-03 3.69871367e-03]\n" ] } ], @@ -1655,15 +1663,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "[ 1.95976303 0.48974963 2.82202765 3.79224555 -2.14248744]\n", + "[ 2.04038959 -0.81343612 7.94304103 -3.83770135 1.70073443]\n", "Training R2\n", - "0.9947613223847728\n", + "0.9954441212986554\n", "Training MSE\n", - "0.010596520060222926\n", + "0.008653750612762521\n", "Test R2\n", - "0.989645675273948\n", + "0.99255249100346\n", "Test MSE\n", - "0.01889099091713886\n" + "0.016970019460234076\n" ] } ], diff --git a/doc/LectureNotes/_build/jupyter_execute/week36.ipynb b/doc/LectureNotes/_build/jupyter_execute/week36.ipynb index c68516565..d15ac0234 100644 --- a/doc/LectureNotes/_build/jupyter_execute/week36.ipynb +++ b/doc/LectureNotes/_build/jupyter_execute/week36.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "7411173e", + "id": "2b900fea", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "f50db913", + "id": "2286e211", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "4a90e380", + "id": "edbcd7f9", "metadata": { "editable": true }, @@ -44,28 +44,34 @@ "\n", "* Material for the lecture on Thursday September 7\n", "\n", - " * Linear Regression and links with Statistics, Resampling methods\n", + " * Technicalities related to scaling and other issues with data handling\n", "\n", - " * Recommended Reading: Goodfellow et al chapter 3 on probability theory, see URL:\"\"\n", + " * Linear Regression and links with Statistics\n", "\n", - " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)" + " * [Recommended Reading: Goodfellow et al chapter 3 on probability theory](https://www.deeplearningbook.org/)\n", + "\n", + " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)\n", + "\n", + " * [Video of lecture](https://youtu.be/Kc20CFK0z7Y) \n", + "\n", + " * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep7.pdf)" ] }, { "cell_type": "markdown", - "id": "351d36fd", + "id": "5dbf3ff5", "metadata": { "editable": true }, "source": [ "## Material for the active learning sessions Tuesday and Wednesday\n", "\n", - "The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples" + "The material here contains a summary from last week and discussion of SVD, Ridge and Lasso regression with examples" ] }, { "cell_type": "markdown", - "id": "0f9f4d2f", + "id": "2dae8175", "metadata": { "editable": true }, @@ -77,7 +83,7 @@ }, { "cell_type": "markdown", - "id": "9ec36bdd", + "id": "fb3d486d", "metadata": { "editable": true }, @@ -89,7 +95,7 @@ }, { "cell_type": "markdown", - "id": "614493c6", + "id": "5f8492a1", "metadata": { "editable": true }, @@ -99,7 +105,7 @@ }, { "cell_type": "markdown", - "id": "dc4be423", + "id": "a35422b4", "metadata": { "editable": true }, @@ -111,7 +117,7 @@ }, { "cell_type": "markdown", - "id": "b262332c", + "id": "aef923ea", "metadata": { "editable": true }, @@ -121,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "ab1829a4", + "id": "5da8277b", "metadata": { "editable": true }, @@ -138,7 +144,7 @@ }, { "cell_type": "markdown", - "id": "7c82dd9e", + "id": "eb34e530", "metadata": { "editable": true }, @@ -148,7 +154,7 @@ }, { "cell_type": "markdown", - "id": "5bab8721", + "id": "a2a33728", "metadata": { "editable": true }, @@ -160,7 +166,7 @@ }, { "cell_type": "markdown", - "id": "72273a4f", + "id": "9bea1837", "metadata": { "editable": true }, @@ -170,7 +176,7 @@ }, { "cell_type": "markdown", - "id": "d4af03ea", + "id": "af711505", "metadata": { "editable": true }, @@ -182,7 +188,7 @@ }, { "cell_type": "markdown", - "id": "c0d35853", + "id": "f02b3738", "metadata": { "editable": true }, @@ -196,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "db5ea827", + "id": "d3fe4855", "metadata": { "editable": true }, @@ -208,7 +214,7 @@ }, { "cell_type": "markdown", - "id": "a6b341ce", + "id": "7b5fa376", "metadata": { "editable": true }, @@ -230,7 +236,7 @@ }, { "cell_type": "markdown", - "id": "db723709", + "id": "d212be4e", "metadata": { "editable": true }, @@ -242,7 +248,7 @@ }, { "cell_type": "markdown", - "id": "1e600268", + "id": "eca13576", "metadata": { "editable": true }, @@ -257,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "b9452938", + "id": "fbf0e2be", "metadata": { "editable": true }, @@ -269,7 +275,7 @@ }, { "cell_type": "markdown", - "id": "53fea1e6", + "id": "acbd86ec", "metadata": { "editable": true }, @@ -281,7 +287,7 @@ }, { "cell_type": "markdown", - "id": "ed10baad", + "id": "8887e338", "metadata": { "editable": true }, @@ -291,7 +297,7 @@ }, { "cell_type": "markdown", - "id": "a4d0db96", + "id": "e95bcbda", "metadata": { "editable": true }, @@ -303,7 +309,7 @@ }, { "cell_type": "markdown", - "id": "fae3cadc", + "id": "e66200c2", "metadata": { "editable": true }, @@ -313,7 +319,7 @@ }, { "cell_type": "markdown", - "id": "e973c775", + "id": "a0c3692c", "metadata": { "editable": true }, @@ -325,7 +331,7 @@ }, { "cell_type": "markdown", - "id": "f88a33f4", + "id": "8d13a9ff", "metadata": { "editable": true }, @@ -335,7 +341,7 @@ }, { "cell_type": "markdown", - "id": "5bc7214d", + "id": "31c3dd91", "metadata": { "editable": true }, @@ -347,7 +353,7 @@ }, { "cell_type": "markdown", - "id": "561d4c91", + "id": "87a6b291", "metadata": { "editable": true }, @@ -364,7 +370,7 @@ }, { "cell_type": "markdown", - "id": "0eda2d1e", + "id": "3bb9eed3", "metadata": { "editable": true }, @@ -378,7 +384,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "c95dbc32", + "id": "ba123060", "metadata": { "collapsed": false, "editable": true @@ -405,7 +411,7 @@ }, { "cell_type": "markdown", - "id": "77079d25", + "id": "46fdfd39", "metadata": { "editable": true }, @@ -416,7 +422,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "cd0ecc44", + "id": "36e47fde", "metadata": { "collapsed": false, "editable": true @@ -456,7 +462,7 @@ }, { "cell_type": "markdown", - "id": "41d896dc", + "id": "53878874", "metadata": { "editable": true }, @@ -476,7 +482,7 @@ }, { "cell_type": "markdown", - "id": "f594852c", + "id": "87546753", "metadata": { "editable": true }, @@ -488,7 +494,7 @@ }, { "cell_type": "markdown", - "id": "b78264f2", + "id": "d8a4669d", "metadata": { "editable": true }, @@ -499,7 +505,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "d05eff95", + "id": "b6523ecb", "metadata": { "collapsed": false, "editable": true @@ -533,7 +539,7 @@ }, { "cell_type": "markdown", - "id": "2fae32c6", + "id": "2fce573f", "metadata": { "editable": true }, @@ -543,7 +549,7 @@ }, { "cell_type": "markdown", - "id": "8631011a", + "id": "840991d3", "metadata": { "editable": true }, @@ -556,7 +562,7 @@ }, { "cell_type": "markdown", - "id": "8e7701de", + "id": "4df7aac6", "metadata": { "editable": true }, @@ -568,7 +574,7 @@ }, { "cell_type": "markdown", - "id": "aea7230b", + "id": "3f3acc05", "metadata": { "editable": true }, @@ -578,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "c5029213", + "id": "2d1218f6", "metadata": { "editable": true }, @@ -591,7 +597,7 @@ }, { "cell_type": "markdown", - "id": "535ad70e", + "id": "01624e31", "metadata": { "editable": true }, @@ -601,7 +607,7 @@ }, { "cell_type": "markdown", - "id": "159ae526", + "id": "616fe159", "metadata": { "editable": true }, @@ -613,7 +619,7 @@ }, { "cell_type": "markdown", - "id": "2f25dfa4", + "id": "27cf2acd", "metadata": { "editable": true }, @@ -628,7 +634,7 @@ }, { "cell_type": "markdown", - "id": "9d187806", + "id": "2ff1e0d2", "metadata": { "editable": true }, @@ -641,7 +647,7 @@ }, { "cell_type": "markdown", - "id": "6c692db5", + "id": "aef1996b", "metadata": { "editable": true }, @@ -655,7 +661,7 @@ }, { "cell_type": "markdown", - "id": "3404e0b4", + "id": "75a44cc9", "metadata": { "editable": true }, @@ -667,7 +673,7 @@ }, { "cell_type": "markdown", - "id": "4bc49666", + "id": "e2ba6037", "metadata": { "editable": true }, @@ -677,7 +683,7 @@ }, { "cell_type": "markdown", - "id": "3f08dfa1", + "id": "95517053", "metadata": { "editable": true }, @@ -690,7 +696,7 @@ }, { "cell_type": "markdown", - "id": "30bcc0cd", + "id": "5f15d4ea", "metadata": { "editable": true }, @@ -702,7 +708,7 @@ }, { "cell_type": "markdown", - "id": "d311a7d8", + "id": "68579bd0", "metadata": { "editable": true }, @@ -714,7 +720,7 @@ }, { "cell_type": "markdown", - "id": "32c46b73", + "id": "53887938", "metadata": { "editable": true }, @@ -726,7 +732,7 @@ }, { "cell_type": "markdown", - "id": "3cceda87", + "id": "469c8c35", "metadata": { "editable": true }, @@ -738,7 +744,7 @@ }, { "cell_type": "markdown", - "id": "10a16675", + "id": "a7521a38", "metadata": { "editable": true }, @@ -752,7 +758,7 @@ }, { "cell_type": "markdown", - "id": "6dba20bf", + "id": "cdfddb46", "metadata": { "editable": true }, @@ -764,7 +770,7 @@ }, { "cell_type": "markdown", - "id": "9325f924", + "id": "e87c8606", "metadata": { "editable": true }, @@ -774,7 +780,7 @@ }, { "cell_type": "markdown", - "id": "745d61e9", + "id": "3e06d524", "metadata": { "editable": true }, @@ -786,7 +792,7 @@ }, { "cell_type": "markdown", - "id": "3866af64", + "id": "798d77a9", "metadata": { "editable": true }, @@ -796,7 +802,7 @@ }, { "cell_type": "markdown", - "id": "710de288", + "id": "8808de88", "metadata": { "editable": true }, @@ -808,7 +814,7 @@ }, { "cell_type": "markdown", - "id": "5893fba0", + "id": "c3825a07", "metadata": { "editable": true }, @@ -820,7 +826,7 @@ }, { "cell_type": "markdown", - "id": "5ef8c59f", + "id": "868b6a19", "metadata": { "editable": true }, @@ -830,7 +836,7 @@ }, { "cell_type": "markdown", - "id": "79b257ae", + "id": "eaba5f13", "metadata": { "editable": true }, @@ -841,7 +847,7 @@ }, { "cell_type": "markdown", - "id": "46270996", + "id": "c9cc77c3", "metadata": { "editable": true }, @@ -853,7 +859,7 @@ }, { "cell_type": "markdown", - "id": "785aebe9", + "id": "4b3e1232", "metadata": { "editable": true }, @@ -869,7 +875,7 @@ }, { "cell_type": "markdown", - "id": "443f3a3f", + "id": "d59ed944", "metadata": { "editable": true }, @@ -882,7 +888,7 @@ }, { "cell_type": "markdown", - "id": "cfb8d4c0", + "id": "e81dd0be", "metadata": { "editable": true }, @@ -894,7 +900,7 @@ }, { "cell_type": "markdown", - "id": "2be1ca1f", + "id": "f26a385c", "metadata": { "editable": true }, @@ -904,7 +910,7 @@ }, { "cell_type": "markdown", - "id": "9ed0aec2", + "id": "aa00d97a", "metadata": { "editable": true }, @@ -916,7 +922,7 @@ }, { "cell_type": "markdown", - "id": "943b1cd5", + "id": "17ba8d1c", "metadata": { "editable": true }, @@ -926,7 +932,7 @@ }, { "cell_type": "markdown", - "id": "4a8a6bff", + "id": "8cf9dc8f", "metadata": { "editable": true }, @@ -938,7 +944,7 @@ }, { "cell_type": "markdown", - "id": "b0371965", + "id": "f0c43d20", "metadata": { "editable": true }, @@ -950,7 +956,7 @@ }, { "cell_type": "markdown", - "id": "38e7b497", + "id": "5b119d73", "metadata": { "editable": true }, @@ -966,7 +972,7 @@ }, { "cell_type": "markdown", - "id": "682f90db", + "id": "1efba158", "metadata": { "editable": true }, @@ -978,7 +984,7 @@ }, { "cell_type": "markdown", - "id": "b5fb7771", + "id": "e5943a40", "metadata": { "editable": true }, @@ -990,7 +996,7 @@ }, { "cell_type": "markdown", - "id": "0c228525", + "id": "167a1a9e", "metadata": { "editable": true }, @@ -1000,7 +1006,7 @@ }, { "cell_type": "markdown", - "id": "08a400af", + "id": "924b5c9b", "metadata": { "editable": true }, @@ -1012,7 +1018,7 @@ }, { "cell_type": "markdown", - "id": "ec0808b1", + "id": "56ac0a65", "metadata": { "editable": true }, @@ -1022,7 +1028,7 @@ }, { "cell_type": "markdown", - "id": "92d3d92f", + "id": "4323c860", "metadata": { "editable": true }, @@ -1034,7 +1040,7 @@ }, { "cell_type": "markdown", - "id": "07486e29", + "id": "12ef13b2", "metadata": { "editable": true }, @@ -1051,7 +1057,7 @@ }, { "cell_type": "markdown", - "id": "9868d4f4", + "id": "d33c685e", "metadata": { "editable": true }, @@ -1063,7 +1069,7 @@ }, { "cell_type": "markdown", - "id": "1793a225", + "id": "b30d436d", "metadata": { "editable": true }, @@ -1075,7 +1081,7 @@ }, { "cell_type": "markdown", - "id": "c7d3fa52", + "id": "37e47a2a", "metadata": { "editable": true }, @@ -1085,7 +1091,7 @@ }, { "cell_type": "markdown", - "id": "47894977", + "id": "3f87ff41", "metadata": { "editable": true }, @@ -1097,7 +1103,7 @@ }, { "cell_type": "markdown", - "id": "d1a11dc8", + "id": "b14d10ff", "metadata": { "editable": true }, @@ -1107,7 +1113,7 @@ }, { "cell_type": "markdown", - "id": "a6cf82de", + "id": "672b64b5", "metadata": { "editable": true }, @@ -1119,7 +1125,7 @@ }, { "cell_type": "markdown", - "id": "bbf7ccc7", + "id": "ef321df6", "metadata": { "editable": true }, @@ -1129,7 +1135,7 @@ }, { "cell_type": "markdown", - "id": "bc6dbb83", + "id": "e67e683d", "metadata": { "editable": true }, @@ -1141,7 +1147,7 @@ }, { "cell_type": "markdown", - "id": "ef5a8292", + "id": "b9bd1d3f", "metadata": { "editable": true }, @@ -1151,7 +1157,7 @@ }, { "cell_type": "markdown", - "id": "d77bf73a", + "id": "b705fb0b", "metadata": { "editable": true }, @@ -1166,7 +1172,7 @@ }, { "cell_type": "markdown", - "id": "265ce93f", + "id": "03679605", "metadata": { "editable": true }, @@ -1178,7 +1184,7 @@ }, { "cell_type": "markdown", - "id": "c19e720c", + "id": "b58e9ee7", "metadata": { "editable": true }, @@ -1188,7 +1194,7 @@ }, { "cell_type": "markdown", - "id": "8cad2962", + "id": "42d8e1b3", "metadata": { "editable": true }, @@ -1200,7 +1206,7 @@ }, { "cell_type": "markdown", - "id": "bf6316a2", + "id": "db624348", "metadata": { "editable": true }, @@ -1212,7 +1218,7 @@ }, { "cell_type": "markdown", - "id": "78413ab6", + "id": "8d37cd9f", "metadata": { "editable": true }, @@ -1224,7 +1230,7 @@ }, { "cell_type": "markdown", - "id": "a77ca41b", + "id": "e88bc71d", "metadata": { "editable": true }, @@ -1234,7 +1240,7 @@ }, { "cell_type": "markdown", - "id": "67b3f67e", + "id": "7bee2106", "metadata": { "editable": true }, @@ -1246,7 +1252,7 @@ }, { "cell_type": "markdown", - "id": "d3571c8b", + "id": "fc1dc754", "metadata": { "editable": true }, @@ -1258,7 +1264,7 @@ }, { "cell_type": "markdown", - "id": "943f822e", + "id": "e5c6d86f", "metadata": { "editable": true }, @@ -1270,7 +1276,7 @@ }, { "cell_type": "markdown", - "id": "2ec0ba01", + "id": "1413c7cf", "metadata": { "editable": true }, @@ -1280,7 +1286,7 @@ }, { "cell_type": "markdown", - "id": "85bf4c9b", + "id": "1b181b9b", "metadata": { "editable": true }, @@ -1292,7 +1298,7 @@ }, { "cell_type": "markdown", - "id": "c81d05f4", + "id": "ca27bd28", "metadata": { "editable": true }, @@ -1302,7 +1308,7 @@ }, { "cell_type": "markdown", - "id": "ce36390d", + "id": "f2ce6515", "metadata": { "editable": true }, @@ -1316,7 +1322,7 @@ }, { "cell_type": "markdown", - "id": "a47dd697", + "id": "bbe27e9c", "metadata": { "editable": true }, @@ -1326,7 +1332,7 @@ }, { "cell_type": "markdown", - "id": "15fe5a0c", + "id": "3ddfa9f9", "metadata": { "editable": true }, @@ -1338,7 +1344,7 @@ }, { "cell_type": "markdown", - "id": "62d7f0e9", + "id": "2d31b195", "metadata": { "editable": true }, @@ -1350,7 +1356,7 @@ }, { "cell_type": "markdown", - "id": "f2d4576f", + "id": "4b092e9b", "metadata": { "editable": true }, @@ -1360,7 +1366,7 @@ }, { "cell_type": "markdown", - "id": "fb2bc0ff", + "id": "7c845085", "metadata": { "editable": true }, @@ -1372,7 +1378,7 @@ }, { "cell_type": "markdown", - "id": "807ac009", + "id": "0fb000c5", "metadata": { "editable": true }, @@ -1382,7 +1388,7 @@ }, { "cell_type": "markdown", - "id": "fe72889b", + "id": "bd4d8817", "metadata": { "editable": true }, @@ -1394,7 +1400,7 @@ }, { "cell_type": "markdown", - "id": "48844e10", + "id": "d0d10a69", "metadata": { "editable": true }, @@ -1406,7 +1412,7 @@ }, { "cell_type": "markdown", - "id": "72a4bc4c", + "id": "d9db36da", "metadata": { "editable": true }, @@ -1416,7 +1422,7 @@ }, { "cell_type": "markdown", - "id": "cfe1db6f", + "id": "a409bd8a", "metadata": { "editable": true }, @@ -1428,7 +1434,7 @@ }, { "cell_type": "markdown", - "id": "05b6e6f6", + "id": "c3be5f03", "metadata": { "editable": true }, @@ -1438,7 +1444,7 @@ }, { "cell_type": "markdown", - "id": "0e58f37a", + "id": "6533b635", "metadata": { "editable": true }, @@ -1450,7 +1456,7 @@ }, { "cell_type": "markdown", - "id": "78593de5", + "id": "3c6b3074", "metadata": { "editable": true }, @@ -1462,7 +1468,7 @@ }, { "cell_type": "markdown", - "id": "26b08a56", + "id": "747eecf9", "metadata": { "editable": true }, @@ -1472,7 +1478,7 @@ }, { "cell_type": "markdown", - "id": "3df661a4", + "id": "87542c6e", "metadata": { "editable": true }, @@ -1484,7 +1490,7 @@ }, { "cell_type": "markdown", - "id": "68460e68", + "id": "13f97260", "metadata": { "editable": true }, @@ -1497,7 +1503,7 @@ }, { "cell_type": "markdown", - "id": "6c660112", + "id": "bc02bc84", "metadata": { "editable": true }, @@ -1509,7 +1515,7 @@ }, { "cell_type": "markdown", - "id": "6ff54cf1", + "id": "9f9d8193", "metadata": { "editable": true }, @@ -1521,7 +1527,7 @@ }, { "cell_type": "markdown", - "id": "f3f88f94", + "id": "ba404c5e", "metadata": { "editable": true }, @@ -1533,7 +1539,7 @@ }, { "cell_type": "markdown", - "id": "6918fc7c", + "id": "9af7054a", "metadata": { "editable": true }, @@ -1543,7 +1549,7 @@ }, { "cell_type": "markdown", - "id": "1b2321a3", + "id": "eb7fd44a", "metadata": { "editable": true }, @@ -1555,7 +1561,7 @@ }, { "cell_type": "markdown", - "id": "d5e21457", + "id": "041b60b2", "metadata": { "editable": true }, @@ -1565,7 +1571,7 @@ }, { "cell_type": "markdown", - "id": "e03c3617", + "id": "f95dc455", "metadata": { "editable": true }, @@ -1577,7 +1583,7 @@ }, { "cell_type": "markdown", - "id": "6c4d9a4a", + "id": "28176b93", "metadata": { "editable": true }, @@ -1587,7 +1593,7 @@ }, { "cell_type": "markdown", - "id": "584bef2e", + "id": "b6f75bf7", "metadata": { "editable": true }, @@ -1599,7 +1605,7 @@ }, { "cell_type": "markdown", - "id": "87273242", + "id": "f6ad6840", "metadata": { "editable": true }, @@ -1609,7 +1615,7 @@ }, { "cell_type": "markdown", - "id": "f7e8277f", + "id": "fe93ca4d", "metadata": { "editable": true }, @@ -1622,7 +1628,7 @@ }, { "cell_type": "markdown", - "id": "e7167c28", + "id": "4d7921c1", "metadata": { "editable": true }, @@ -1634,7 +1640,7 @@ }, { "cell_type": "markdown", - "id": "6a3fa7b2", + "id": "d106e526", "metadata": { "editable": true }, @@ -1646,7 +1652,7 @@ }, { "cell_type": "markdown", - "id": "92c33067", + "id": "2ab11733", "metadata": { "editable": true }, @@ -1656,7 +1662,7 @@ }, { "cell_type": "markdown", - "id": "1b7be5cc", + "id": "c871f92c", "metadata": { "editable": true }, @@ -1668,7 +1674,7 @@ }, { "cell_type": "markdown", - "id": "9e2db51b", + "id": "4fc85aac", "metadata": { "editable": true }, @@ -1685,7 +1691,7 @@ }, { "cell_type": "markdown", - "id": "7ea5c023", + "id": "c66960ef", "metadata": { "editable": true }, @@ -1697,7 +1703,7 @@ }, { "cell_type": "markdown", - "id": "efff1845", + "id": "6c742a89", "metadata": { "editable": true }, @@ -1709,7 +1715,7 @@ }, { "cell_type": "markdown", - "id": "80496d8c", + "id": "5a6fe016", "metadata": { "editable": true }, @@ -1719,7 +1725,7 @@ }, { "cell_type": "markdown", - "id": "bb33463a", + "id": "63472c22", "metadata": { "editable": true }, @@ -1731,7 +1737,7 @@ }, { "cell_type": "markdown", - "id": "f16b1457", + "id": "3ede415b", "metadata": { "editable": true }, @@ -1741,7 +1747,7 @@ }, { "cell_type": "markdown", - "id": "4bee5a43", + "id": "90397204", "metadata": { "editable": true }, @@ -1753,7 +1759,7 @@ }, { "cell_type": "markdown", - "id": "17176990", + "id": "acd78254", "metadata": { "editable": true }, @@ -1763,7 +1769,7 @@ }, { "cell_type": "markdown", - "id": "ec2c8f34", + "id": "114e58b4", "metadata": { "editable": true }, @@ -1775,7 +1781,7 @@ }, { "cell_type": "markdown", - "id": "5934144d", + "id": "dc251844", "metadata": { "editable": true }, @@ -1785,7 +1791,7 @@ }, { "cell_type": "markdown", - "id": "fe097c51", + "id": "2fb1ac98", "metadata": { "editable": true }, @@ -1800,7 +1806,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "92cd1361", + "id": "8c64672d", "metadata": { "collapsed": false, "editable": true @@ -1862,7 +1868,7 @@ }, { "cell_type": "markdown", - "id": "9e4426f4", + "id": "a87e0be8", "metadata": { "editable": true }, @@ -1872,7 +1878,7 @@ }, { "cell_type": "markdown", - "id": "5f2b6935", + "id": "ac5bf925", "metadata": { "editable": true }, @@ -1883,7 +1889,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "409bbeac", + "id": "078628d7", "metadata": { "collapsed": false, "editable": true @@ -1950,7 +1956,7 @@ }, { "cell_type": "markdown", - "id": "67a24e77", + "id": "f035fb09", "metadata": { "editable": true }, @@ -1961,7 +1967,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "8da16a74", + "id": "acb14894", "metadata": { "collapsed": false, "editable": true @@ -2050,7 +2056,7 @@ }, { "cell_type": "markdown", - "id": "9d2563fa", + "id": "391b11f6", "metadata": { "editable": true }, @@ -2060,962 +2066,850 @@ }, { "cell_type": "markdown", - "id": "1232037e", + "id": "242933c8", "metadata": { "editable": true }, "source": [ - "## Linking the regression analysis with a statistical interpretation\n", + "## Important technicalities: More on Rescaling data\n", "\n", - "We will now couple the discussions of ordinary least squares, Ridge\n", - "and Lasso regression with a statistical interpretation, that is we\n", - "move from a linear algebra analysis to a statistical analysis. In\n", - "particular, we will focus on what the regularization terms can result\n", - "in. We will amongst other things show that the regularization\n", - "parameter can reduce considerably the variance of the parameters\n", - "$\\beta$.\n", + "When you are comparing your own code with for example **Scikit-Learn**'s\n", + "library, there are some technicalities to keep in mind. The examples\n", + "here demonstrate some of these aspects with potential pitfalls.\n", + "\n", + "The discussion here focuses on the role of the intercept, how we can\n", + "set up the design matrix, what scaling we should use and other topics\n", + "which tend confuse us.\n", + "\n", + "The intercept can be interpreted as the expected value of our\n", + "target/output variables when all other predictors are set to zero.\n", + "Thus, if we cannot assume that the expected outputs/targets are zero\n", + "when all predictors are zero (the columns in the design matrix), it\n", + "may be a bad idea to implement a model which penalizes the intercept.\n", + "Furthermore, in for example Ridge and Lasso regression, the default solutions\n", + "from the library **Scikit-Learn** (when not shrinking $\\beta_0$) for the unknown parameters\n", + "$\\boldsymbol{\\beta}$, are derived under the assumption that both $\\boldsymbol{y}$ and\n", + "$\\boldsymbol{X}$ are zero centered, that is we subtract the mean values.\n", + "\n", + "If our predictors represent different scales, then it is important to\n", + "standardize the design matrix $\\boldsymbol{X}$ by subtracting the mean of each\n", + "column from the corresponding column and dividing the column with its\n", + "standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,\n", + "the results may differ. \n", "\n", "The\n", - "advantage of doing linear regression is that we actually end up with\n", - "analytical expressions for several statistical quantities. \n", - "Standard least squares and Ridge regression allow us to\n", - "derive quantities like the variance and other expectation values in a\n", - "rather straightforward way.\n", + "[Standardscaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html)\n", + "function in **Scikit-Learn** does this for us. For the data sets we\n", + "have been studying in our various examples, the data are in many cases\n", + "already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a\n", + "survey of your data, with a critical assessment of them in case you need to scale the data.\n", "\n", - "It is assumed that $\\varepsilon_i\n", - "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", - "independent, i.e.:" + "If you need to scale the data, not doing so will give an *unfair*\n", + "penalization of the parameters since their magnitude depends on the\n", + "scale of their corresponding predictor.\n", + "\n", + "Suppose as an example that you \n", + "you have an input variable given by the heights of different persons.\n", + "Human height might be measured in inches or meters or\n", + "kilometers. If measured in kilometers, a standard linear regression\n", + "model with this predictor would probably give a much bigger\n", + "coefficient term, than if measured in millimeters.\n", + "This can clearly lead to problems in evaluating the cost/loss functions.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0e71daa5", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" ] }, { "cell_type": "markdown", - "id": "e50205be", + "id": "1f0eb0d0", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "2ffc4acb", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mbox{Cov}(\\varepsilon_{i_1},\n", - "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", - "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", - "\\end{align*}\n", + "C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70eccda9", + "id": "2511c028", "metadata": { "editable": true }, "source": [ - "The randomness of $\\varepsilon_i$ implies that\n", - "$\\mathbf{y}_i$ is also a random variable. In particular,\n", - "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", - "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", - "non-random scalar. To specify the parameters of the distribution of\n", - "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", "\n", - "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", - "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", - "row number $i$ and perform a sum over all values $p$." + "What we have done is to single out the $\\beta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\beta_0$, we want the derivative to obey" ] }, { "cell_type": "markdown", - "id": "e6bdcbf2", - "metadata": { - "editable": true - }, - "source": [ - "## Assumptions made\n", - "\n", - "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", - "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", - "which describe our data" - ] - }, - { - "cell_type": "markdown", - "id": "d74471b8", + "id": "908beb7f", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "\\frac{\\partial C}{\\partial \\beta_j} = 0,\n", "$$" ] }, { "cell_type": "markdown", - "id": "f7b178d9", + "id": "5fae2a94", "metadata": { "editable": true }, "source": [ - "We approximate this function with our model from the solution of the linear regression equations, that is our\n", - "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + "for all $j$. For $\\beta_0$ we have" ] }, { "cell_type": "markdown", - "id": "8ea16c62", + "id": "d7729a27", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n", "$$" ] }, { "cell_type": "markdown", - "id": "4a8d7bfd", + "id": "7fbb2ded", "metadata": { "editable": true }, "source": [ - "## Expectation value and variance\n", - "\n", - "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + "Multiplying away the constant $2/n$, we obtain" ] }, { "cell_type": "markdown", - "id": "5367d943", + "id": "8cef22ce", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mathbb{E}(y_i) & =\n", - "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", - "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", - "\\end{align*}\n", + "\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c7eefd85", + "id": "c38ab3a0", "metadata": { "editable": true }, "source": [ - "while\n", - "its variance is" + "Let us specialize first to the case where we have only two parameters $\\beta_0$ and $\\beta_1$.\n", + "Our result for $\\beta_0$ simplifies then to" ] }, { "cell_type": "markdown", - "id": "27ce343a", + "id": "c47e7bbe", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", - "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", - "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", - "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", - "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", - "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", - "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", - "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", - "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", - "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", - "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", - "\\end{align*}\n", + "n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c67ea20b", + "id": "06962abd", "metadata": { "editable": true }, "source": [ - "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", - "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + "We obtain then" ] }, { "cell_type": "markdown", - "id": "b4da473a", - "metadata": { - "editable": true - }, - "source": [ - "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", - "\n", - "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" - ] - }, - { - "cell_type": "markdown", - "id": "af812d38", + "id": "14c0a845", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "9d3aaff5", + "id": "5f50f223", "metadata": { "editable": true }, "source": [ - "This means that the estimator of the regression parameters is unbiased.\n", - "\n", - "We can also calculate the variance\n", - "\n", - "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + "If we define" ] }, { "cell_type": "markdown", - "id": "eebea8b7", + "id": "188ed44d", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{eqnarray*}\n", - "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", - "\\\\\n", - "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", - "\\\\\n", - "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", - "% \\\\\n", - "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", - "\\\\\n", - "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", - "\\end{eqnarray*}\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", "$$" ] }, { "cell_type": "markdown", - "id": "714f281a", + "id": "0b0db7b8", "metadata": { "editable": true }, "source": [ - "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", - "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", - "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", - "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", - "variance of the estimate of the $j$-th regression coefficient:\n", - "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", - "construct a confidence interval for the estimates.\n", - "\n", - "In a similar way, we can obtain analytical expressions for say the\n", - "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", - "when we employ Ridge regression, allowing us again to define a confidence interval. \n", - "\n", - "It is rather straightforward to show that" + "and the mean value of the outputs as" ] }, { "cell_type": "markdown", - "id": "66015ca6", + "id": "e7550136", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", "$$" ] }, { "cell_type": "markdown", - "id": "267f01c0", + "id": "c7cc678d", "metadata": { "editable": true }, "source": [ - "We see clearly that \n", - "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", - "\n", - "We can also compute the variance as" + "we have" ] }, { "cell_type": "markdown", - "id": "998de4b6", + "id": "9c9abcb2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\beta_0 = \\mu_y - \\beta_1\\mu_{\\boldsymbol{x}_1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "06a8d452", + "id": "2336bf55", "metadata": { "editable": true }, "source": [ - "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", - "\n", - "With this, we can compute the difference" + "In the general case with more parameters than $\\beta_0$ and $\\beta_1$, we have" ] }, { "cell_type": "markdown", - "id": "6cd3ed9b", + "id": "e0111655", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70f9aa69", + "id": "709c3af5", "metadata": { "editable": true }, "source": [ - "The difference is non-negative definite since each component of the\n", - "matrix product is non-negative definite. \n", - "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + "We can rewrite the latter equation as" ] }, { "cell_type": "markdown", - "id": "3bb18388", - "metadata": { - "editable": true - }, - "source": [ - "## Deriving OLS from a probability distribution\n", - "\n", - "Our basic assumption when we derived the OLS equations was to assume\n", - "that our output is determined by a given continuous function\n", - "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", - "distribution with zero mean value and an undetermined variance\n", - "$\\sigma^2$.\n", - "\n", - "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", - "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", - "the design matrix are not stochastic variables, we can assume that the\n", - "probability distribution of our targets is also a normal distribution\n", - "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", - "single output $y_i$ is given by the Gaussian distribution" - ] - }, - { - "cell_type": "markdown", - "id": "84b546af", + "id": "a9ed060b", "metadata": { "editable": true }, "source": [ "$$\n", - "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\beta_j,\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e90935e", + "id": "deb61dec", "metadata": { "editable": true }, "source": [ - "## Independent and Identically Distrubuted (iid)\n", - "\n", - "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", - "We define this distribution as" + "where we have defined" ] }, { "cell_type": "markdown", - "id": "b40b33f5", + "id": "dc60f20f", "metadata": { "editable": true }, "source": [ "$$\n", - "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", "$$" ] }, { "cell_type": "markdown", - "id": "23e0c7ac", + "id": "b43a095d", "metadata": { "editable": true }, "source": [ - "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", "\n", - "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" ] }, { "cell_type": "markdown", - "id": "dcd4ed9c", + "id": "050949a2", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n", "$$" ] }, { "cell_type": "markdown", - "id": "5c37adb5", + "id": "7c288c56", "metadata": { "editable": true }, "source": [ - "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", - "in case we have a simple one-dimensional input and output case" + "If we minimize with respect to $\\boldsymbol{\\beta}$ we have then" ] }, { "cell_type": "markdown", - "id": "27e4eea9", + "id": "72d25a93", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", "$$" ] }, { "cell_type": "markdown", - "id": "5dd95110", + "id": "6e4fe379", "metadata": { "editable": true }, "source": [ - "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", - "We can now rewrite the above probability as" + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\beta}^T\\boldsymbol{\\beta}$ to the cost function and get then" ] }, { "cell_type": "markdown", - "id": "bc7cfda7", + "id": "b0f0e6e7", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "2bf6dbf4", + "id": "6cfb015d", "metadata": { "editable": true }, "source": [ - "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "bfbf5d1d", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_beta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_beta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "beta = fit_beta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True beta: {true_beta}\")\n", + "print(f\"Fitted beta: {beta}\")\n", + "print(f\"Sklearn fitted beta: {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ beta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "beta = fit_beta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ beta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted beta (wiothout intercept): {beta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted beta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ beta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "e8ff9386", + "id": "7dc13bd7", "metadata": { "editable": true }, "source": [ - "## Maximum Likelihood Estimation (MLE)\n", + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", "\n", - "In statistics, maximum likelihood estimation (MLE) is a method of\n", - "estimating the parameters of an assumed probability distribution,\n", - "given some observed data. This is achieved by maximizing a likelihood\n", - "function so that, under the assumed statistical model, the observed\n", - "data is the most probable. \n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\beta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", "\n", - "We will assume here that our events are given by the above Gaussian\n", - "distribution and we will determine the optimal parameters $\\beta$ by\n", - "maximizing the above PDF. However, computing the derivatives of a\n", - "product function is cumbersome and can easily lead to overflow and/or\n", - "underflowproblems, with potentials for loss of numerical precision.\n", - "\n", - "In practice, it is more convenient to maximize the logarithm of the\n", - "PDF because it is a monotonically increasing function of the argument.\n", - "Alternatively, and this will be our option, we will minimize the\n", - "negative of the logarithm since this is a monotonically decreasing\n", - "function.\n", - "\n", - "Note also that maximization/minimization of the logarithm of the PDF\n", - "is equivalent to the maximization/minimization of the function itself." + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" ] }, { "cell_type": "markdown", - "id": "cd665e76", - "metadata": { - "editable": true - }, - "source": [ - "## A new Cost Function\n", - "\n", - "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" - ] - }, - { - "cell_type": "markdown", - "id": "144c5c88", + "id": "3ce23f71", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\beta_j^2,\n", "$$" ] }, { "cell_type": "markdown", - "id": "76c7c5c6", + "id": "f34b4a50", "metadata": { "editable": true }, "source": [ - "which becomes" + "but when we take out the intercept, this equation becomes" ] }, { "cell_type": "markdown", - "id": "f78405b0", + "id": "6c535531", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\beta_j^2.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dd7b69b5", + "id": "dc9bd9cc", "metadata": { "editable": true }, "source": [ - "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + "For Lasso regression we have" ] }, { "cell_type": "markdown", - "id": "a1430a20", + "id": "b5911fe2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\beta_j\\vert.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dfcd0c7b", + "id": "4d4103f1", "metadata": { "editable": true }, "source": [ - "which leads to the well-known OLS equation for the optimal paramters $\\beta$" - ] - }, - { - "cell_type": "markdown", - "id": "043d37a6", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "28fe8edc", - "metadata": { - "editable": true - }, - "source": [ - "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." - ] - }, - { - "cell_type": "markdown", - "id": "b600d21c", - "metadata": { - "editable": true - }, - "source": [ - "## More basic Statistics and Bayes' theorem\n", + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", "\n", - "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", - "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "eecec942", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", "\n", - "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", "\n", - "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", - "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", "\n", - "**Union of events is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "d513cf2c", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "d18008b5", - "metadata": { - "editable": true - }, - "source": [ - "**The product rule (aka joint probability) is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "22f55ec4", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "29e1bff6", - "metadata": { - "editable": true - }, - "source": [ - "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", "\n", - "If we have independent events then $p(X,Y)=p(X)p(Y)$." - ] - }, - { - "cell_type": "markdown", - "id": "333869a5", - "metadata": { - "editable": true - }, - "source": [ - "## Marginal Probability\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" - ] - }, - { - "cell_type": "markdown", - "id": "bf8de993", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1c5349d1", - "metadata": { - "editable": true - }, - "source": [ - "## Conditional Probability\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "The conditional probability, if $p(Y) > 0$, is" - ] - }, - { - "cell_type": "markdown", - "id": "dbede038", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "c1334a9c", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeBeta\n", + " ypredictOwnRidge = X_test @ OwnRidgeBeta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta)\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", "\n", - "If we combine the conditional probability with the marginal probability and the standard product rule, we have" - ] - }, - { - "cell_type": "markdown", - "id": "ac8dcf04", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1992e354", - "metadata": { - "editable": true - }, - "source": [ - "which we can rewrite as" - ] - }, - { - "cell_type": "markdown", - "id": "24e5179f", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "42979e5a", - "metadata": { - "editable": true - }, - "source": [ - "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." - ] - }, - { - "cell_type": "markdown", - "id": "86a8cd07", - "metadata": { - "editable": true - }, - "source": [ - "## Interpretations of Bayes' Theorem\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", "\n", - "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", - "evaluated for the observed data $Y$ and can be viewed as a function of\n", - "the parameter space represented by $X$. This function is not\n", - "necesseraly normalized and is normally called the likelihood function.\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7d67706b", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "598d20bd", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", "\n", - "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", "\n", - "Let us try to illustrate Bayes' theorem through an example." - ] - }, - { - "cell_type": "markdown", - "id": "b9a4f9a9", - "metadata": { - "editable": true - }, - "source": [ - "## Example of Usage of Bayes' theorem\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "Let us suppose that you are undergoing a series of mammography scans in\n", - "order to rule out possible breast cancer cases. We define the\n", - "sensitivity for a positive event by the variable $X$. It takes binary\n", - "values with $X=1$ representing a positive event and $X=0$ being a\n", - "negative event. We reserve $Y$ as a classification parameter for\n", - "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", "\n", - "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", "\n", - "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" - ] - }, - { - "cell_type": "markdown", - "id": "888c64ec", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=1) =0.8.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "5086b046", - "metadata": { - "editable": true - }, - "source": [ - "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", - "It is however not correct, as the following Bayesian analysis shows." - ] - }, - { - "cell_type": "markdown", - "id": "609ea2b0", - "metadata": { - "editable": true - }, - "source": [ - "## Doing it correctly\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", - "Let us assume that the prior probability in the population as a whole is" + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta) #Intercept is given by mean of target variable\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "687bfdf3", + "id": "cec80003", "metadata": { "editable": true }, "source": [ - "$$\n", - "p(Y=1) =0.004.\n", - "$$" + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\beta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." ] }, { "cell_type": "markdown", - "id": "c2af1d2d", + "id": "16cdcffb", "metadata": { "editable": true }, "source": [ - "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" - ] - }, - { - "cell_type": "markdown", - "id": "b8ec8d93", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=0) =0.1.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "48a6fb5b", - "metadata": { - "editable": true - }, - "source": [ - "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" - ] - }, - { - "cell_type": "markdown", - "id": "ba37d4fd", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "824f1789", - "metadata": { - "editable": true - }, - "source": [ - "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" - ] - }, - { - "cell_type": "markdown", - "id": "07f89fd5", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem and Ridge and Lasso Regression\n", + "## Test Function for what happens with OLS, Ridge and Lasso\n", "\n", "Hitherto we have discussed Ridge and Lasso regression in terms of a\n", "linear analysis. This may to many of you feel rather technical and\n", "perhaps not that intuitive. The question is whether we can develop a\n", "more intuitive way of understanding what Ridge and Lasso express.\n", "\n", - "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit." - ] - }, - { - "cell_type": "markdown", - "id": "8fcd73d9", - "metadata": { - "editable": true - }, - "source": [ - "## Test Function for what happens with OLS, Ridge and Lasso\n", + "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. \n", "\n", "We will play around with a study of the values for the optimal\n", "parameters $\\boldsymbol{\\beta}$ using OLS, Ridge and Lasso regression. For\n", @@ -3028,8 +2922,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "id": "4402e113", + "execution_count": 11, + "id": "e04dce1d", "metadata": { "collapsed": false, "editable": true @@ -3056,10 +2950,9 @@ "X = np.zeros((len(x),Maxpolydegree))\n", "X[:,0] = 1.0\n", "\n", - "for polydegree in range(1, Maxpolydegree):\n", - " for degree in range(polydegree):\n", - " X[:,degree] = x**(degree)\n", "\n", + "for polydegree in range(1,Maxpolydegree):\n", + " X[:,polydegree] = x**(polydegree)\n", "\n", "# We split the data in test and training data\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", @@ -3103,7 +2996,7 @@ }, { "cell_type": "markdown", - "id": "b700e45e", + "id": "2ae202c3", "metadata": { "editable": true }, @@ -3113,43 +3006,475 @@ }, { "cell_type": "markdown", - "id": "69659d9d", + "id": "4186f9f0", "metadata": { "editable": true }, "source": [ - "## Invoking Bayes' theorem\n", + "## Linking the regression analysis with a statistical interpretation\n", "\n", - "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "We will now couple the discussions of ordinary least squares, Ridge\n", + "and Lasso regression with a statistical interpretation, that is we\n", + "move from a linear algebra analysis to a statistical analysis. In\n", + "particular, we will focus on what the regularization terms can result\n", + "in. We will amongst other things show that the regularization\n", + "parameter can reduce considerably the variance of the parameters\n", + "$\\beta$.\n", "\n", - "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + "The\n", + "advantage of doing linear regression is that we actually end up with\n", + "analytical expressions for several statistical quantities. \n", + "Standard least squares and Ridge regression allow us to\n", + "derive quantities like the variance and other expectation values in a\n", + "rather straightforward way.\n", + "\n", + "It is assumed that $\\varepsilon_i\n", + "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", + "independent, i.e.:" ] }, { "cell_type": "markdown", - "id": "80b423eb", + "id": "7293d2ab", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "\\begin{align*} \n", + "\\mbox{Cov}(\\varepsilon_{i_1},\n", + "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", + "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", + "\\end{align*}\n", "$$" ] }, { "cell_type": "markdown", - "id": "de547949", + "id": "efee3703", "metadata": { "editable": true }, "source": [ - "is given by" + "The randomness of $\\varepsilon_i$ implies that\n", + "$\\mathbf{y}_i$ is also a random variable. In particular,\n", + "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", + "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", + "non-random scalar. To specify the parameters of the distribution of\n", + "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "\n", + "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", + "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", + "row number $i$ and perform a sum over all values $p$." ] }, { "cell_type": "markdown", - "id": "99d9476a", + "id": "85d2303d", + "metadata": { + "editable": true + }, + "source": [ + "## Assumptions made\n", + "\n", + "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", + "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", + "which describe our data" + ] + }, + { + "cell_type": "markdown", + "id": "c1a189c0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0b63e61f", + "metadata": { + "editable": true + }, + "source": [ + "We approximate this function with our model from the solution of the linear regression equations, that is our\n", + "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + ] + }, + { + "cell_type": "markdown", + "id": "14acfe14", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "681408c6", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance\n", + "\n", + "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + ] + }, + { + "cell_type": "markdown", + "id": "d956e33c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mathbb{E}(y_i) & =\n", + "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", + "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "992bee5b", + "metadata": { + "editable": true + }, + "source": [ + "while\n", + "its variance is" + ] + }, + { + "cell_type": "markdown", + "id": "fd75d1af", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", + "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", + "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", + "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", + "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", + "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", + "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", + "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", + "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", + "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", + "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f702411d", + "metadata": { + "editable": true + }, + "source": [ + "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", + "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + ] + }, + { + "cell_type": "markdown", + "id": "b86e729d", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", + "\n", + "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" + ] + }, + { + "cell_type": "markdown", + "id": "5af888a2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3d21a65e", + "metadata": { + "editable": true + }, + "source": [ + "This means that the estimator of the regression parameters is unbiased.\n", + "\n", + "We can also calculate the variance\n", + "\n", + "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + ] + }, + { + "cell_type": "markdown", + "id": "7d8043ab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{eqnarray*}\n", + "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", + "\\\\\n", + "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", + "\\\\\n", + "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", + "% \\\\\n", + "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", + "\\\\\n", + "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", + "\\end{eqnarray*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a126ec7", + "metadata": { + "editable": true + }, + "source": [ + "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", + "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", + "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", + "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", + "variance of the estimate of the $j$-th regression coefficient:\n", + "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", + "construct a confidence interval for the estimates.\n", + "\n", + "In a similar way, we can obtain analytical expressions for say the\n", + "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", + "when we employ Ridge regression, allowing us again to define a confidence interval. \n", + "\n", + "It is rather straightforward to show that" + ] + }, + { + "cell_type": "markdown", + "id": "7287992a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2803f793", + "metadata": { + "editable": true + }, + "source": [ + "We see clearly that \n", + "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", + "\n", + "We can also compute the variance as" + ] + }, + { + "cell_type": "markdown", + "id": "d7443c20", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a0d32eb0", + "metadata": { + "editable": true + }, + "source": [ + "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", + "\n", + "With this, we can compute the difference" + ] + }, + { + "cell_type": "markdown", + "id": "cbd0143e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6de88a4", + "metadata": { + "editable": true + }, + "source": [ + "The difference is non-negative definite since each component of the\n", + "matrix product is non-negative definite. \n", + "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + ] + }, + { + "cell_type": "markdown", + "id": "55630c9f", + "metadata": { + "editable": true + }, + "source": [ + "## Deriving OLS from a probability distribution\n", + "\n", + "Our basic assumption when we derived the OLS equations was to assume\n", + "that our output is determined by a given continuous function\n", + "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", + "distribution with zero mean value and an undetermined variance\n", + "$\\sigma^2$.\n", + "\n", + "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", + "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", + "the design matrix are not stochastic variables, we can assume that the\n", + "probability distribution of our targets is also a normal distribution\n", + "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", + "single output $y_i$ is given by the Gaussian distribution" + ] + }, + { + "cell_type": "markdown", + "id": "745e6352", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ec8c116", + "metadata": { + "editable": true + }, + "source": [ + "## Independent and Identically Distrubuted (iid)\n", + "\n", + "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", + "We define this distribution as" + ] + }, + { + "cell_type": "markdown", + "id": "f92be14e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "edbdd556", + "metadata": { + "editable": true + }, + "source": [ + "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "\n", + "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "6a6903ca", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "57680896", + "metadata": { + "editable": true + }, + "source": [ + "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", + "in case we have a simple one-dimensional input and output case" + ] + }, + { + "cell_type": "markdown", + "id": "9309640f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67fb3df9", + "metadata": { + "editable": true + }, + "source": [ + "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", + "We can now rewrite the above probability as" + ] + }, + { + "cell_type": "markdown", + "id": "894f0fb7", "metadata": { "editable": true }, @@ -3161,7 +3486,508 @@ }, { "cell_type": "markdown", - "id": "aa5613ea", + "id": "13f3f705", + "metadata": { + "editable": true + }, + "source": [ + "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + ] + }, + { + "cell_type": "markdown", + "id": "6bf3edfa", + "metadata": { + "editable": true + }, + "source": [ + "## Maximum Likelihood Estimation (MLE)\n", + "\n", + "In statistics, maximum likelihood estimation (MLE) is a method of\n", + "estimating the parameters of an assumed probability distribution,\n", + "given some observed data. This is achieved by maximizing a likelihood\n", + "function so that, under the assumed statistical model, the observed\n", + "data is the most probable. \n", + "\n", + "We will assume here that our events are given by the above Gaussian\n", + "distribution and we will determine the optimal parameters $\\beta$ by\n", + "maximizing the above PDF. However, computing the derivatives of a\n", + "product function is cumbersome and can easily lead to overflow and/or\n", + "underflowproblems, with potentials for loss of numerical precision.\n", + "\n", + "In practice, it is more convenient to maximize the logarithm of the\n", + "PDF because it is a monotonically increasing function of the argument.\n", + "Alternatively, and this will be our option, we will minimize the\n", + "negative of the logarithm since this is a monotonically decreasing\n", + "function.\n", + "\n", + "Note also that maximization/minimization of the logarithm of the PDF\n", + "is equivalent to the maximization/minimization of the function itself." + ] + }, + { + "cell_type": "markdown", + "id": "be42ac0c", + "metadata": { + "editable": true + }, + "source": [ + "## A new Cost Function\n", + "\n", + "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" + ] + }, + { + "cell_type": "markdown", + "id": "b6fbab08", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "33ef1983", + "metadata": { + "editable": true + }, + "source": [ + "which becomes" + ] + }, + { + "cell_type": "markdown", + "id": "7d6504da", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "03d8775c", + "metadata": { + "editable": true + }, + "source": [ + "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + ] + }, + { + "cell_type": "markdown", + "id": "176d1dab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e20e33bf", + "metadata": { + "editable": true + }, + "source": [ + "which leads to the well-known OLS equation for the optimal paramters $\\beta$" + ] + }, + { + "cell_type": "markdown", + "id": "94f73eae", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f5d2cf20", + "metadata": { + "editable": true + }, + "source": [ + "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." + ] + }, + { + "cell_type": "markdown", + "id": "c32b8f92", + "metadata": { + "editable": true + }, + "source": [ + "## More basic Statistics and Bayes' theorem\n", + "\n", + "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", + "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "\n", + "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "\n", + "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", + "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", + "\n", + "**Union of events is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "a3779fd2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "37e13b18", + "metadata": { + "editable": true + }, + "source": [ + "**The product rule (aka joint probability) is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "bd8b41f0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "71b22e26", + "metadata": { + "editable": true + }, + "source": [ + "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "\n", + "If we have independent events then $p(X,Y)=p(X)p(Y)$." + ] + }, + { + "cell_type": "markdown", + "id": "5f6645d2", + "metadata": { + "editable": true + }, + "source": [ + "## Marginal Probability\n", + "\n", + "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" + ] + }, + { + "cell_type": "markdown", + "id": "7349fe65", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a6205b4f", + "metadata": { + "editable": true + }, + "source": [ + "## Conditional Probability\n", + "\n", + "The conditional probability, if $p(Y) > 0$, is" + ] + }, + { + "cell_type": "markdown", + "id": "43cb130d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d8f1813", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem\n", + "\n", + "If we combine the conditional probability with the marginal probability and the standard product rule, we have" + ] + }, + { + "cell_type": "markdown", + "id": "d2746770", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f3659d1d", + "metadata": { + "editable": true + }, + "source": [ + "which we can rewrite as" + ] + }, + { + "cell_type": "markdown", + "id": "8dfd9ec7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3f247ad1", + "metadata": { + "editable": true + }, + "source": [ + "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." + ] + }, + { + "cell_type": "markdown", + "id": "a2615306", + "metadata": { + "editable": true + }, + "source": [ + "## Interpretations of Bayes' Theorem\n", + "\n", + "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", + "evaluated for the observed data $Y$ and can be viewed as a function of\n", + "the parameter space represented by $X$. This function is not\n", + "necesseraly normalized and is normally called the likelihood function.\n", + "\n", + "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "\n", + "Let us try to illustrate Bayes' theorem through an example." + ] + }, + { + "cell_type": "markdown", + "id": "91eae9e8", + "metadata": { + "editable": true + }, + "source": [ + "## Example of Usage of Bayes' theorem\n", + "\n", + "Let us suppose that you are undergoing a series of mammography scans in\n", + "order to rule out possible breast cancer cases. We define the\n", + "sensitivity for a positive event by the variable $X$. It takes binary\n", + "values with $X=1$ representing a positive event and $X=0$ being a\n", + "negative event. We reserve $Y$ as a classification parameter for\n", + "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "\n", + "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "\n", + "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "76fe1c46", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=1) =0.8.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "397d9768", + "metadata": { + "editable": true + }, + "source": [ + "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", + "It is however not correct, as the following Bayesian analysis shows." + ] + }, + { + "cell_type": "markdown", + "id": "f2c701f5", + "metadata": { + "editable": true + }, + "source": [ + "## Doing it correctly\n", + "\n", + "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", + "Let us assume that the prior probability in the population as a whole is" + ] + }, + { + "cell_type": "markdown", + "id": "0fdb2639", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1) =0.004.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f9ae9f95", + "metadata": { + "editable": true + }, + "source": [ + "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" + ] + }, + { + "cell_type": "markdown", + "id": "a681f1dc", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=0) =0.1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4d2c6030", + "metadata": { + "editable": true + }, + "source": [ + "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" + ] + }, + { + "cell_type": "markdown", + "id": "67a1c582", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "969f368b", + "metadata": { + "editable": true + }, + "source": [ + "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" + ] + }, + { + "cell_type": "markdown", + "id": "356a70c0", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem and Ridge and Lasso Regression\n", + "\n", + "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "\n", + "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + ] + }, + { + "cell_type": "markdown", + "id": "fa653f05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5dd1081", + "metadata": { + "editable": true + }, + "source": [ + "is given by" + ] + }, + { + "cell_type": "markdown", + "id": "731b21d4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "362ae4e2", "metadata": { "editable": true }, @@ -3171,7 +3997,7 @@ }, { "cell_type": "markdown", - "id": "7dea4e11", + "id": "8ba56ac5", "metadata": { "editable": true }, @@ -3183,7 +4009,7 @@ }, { "cell_type": "markdown", - "id": "99ab1d57", + "id": "658e6364", "metadata": { "editable": true }, @@ -3193,7 +4019,7 @@ }, { "cell_type": "markdown", - "id": "170ee68b", + "id": "28d6553d", "metadata": { "editable": true }, @@ -3205,7 +4031,7 @@ }, { "cell_type": "markdown", - "id": "3388d6a4", + "id": "8f4ec5f0", "metadata": { "editable": true }, @@ -3215,7 +4041,7 @@ }, { "cell_type": "markdown", - "id": "a5bdaa16", + "id": "e72b34ca", "metadata": { "editable": true }, @@ -3231,7 +4057,7 @@ }, { "cell_type": "markdown", - "id": "cbf8e7e6", + "id": "0e656652", "metadata": { "editable": true }, @@ -3243,7 +4069,7 @@ }, { "cell_type": "markdown", - "id": "52c9e48f", + "id": "57606d08", "metadata": { "editable": true }, @@ -3253,7 +4079,7 @@ }, { "cell_type": "markdown", - "id": "1ae37a89", + "id": "cf40e3a9", "metadata": { "editable": true }, @@ -3265,7 +4091,7 @@ }, { "cell_type": "markdown", - "id": "dd8fee27", + "id": "65038f20", "metadata": { "editable": true }, @@ -3278,7 +4104,7 @@ }, { "cell_type": "markdown", - "id": "de603b44", + "id": "deb9890c", "metadata": { "editable": true }, @@ -3290,7 +4116,7 @@ }, { "cell_type": "markdown", - "id": "3f08e4a0", + "id": "1a844813", "metadata": { "editable": true }, @@ -3300,7 +4126,7 @@ }, { "cell_type": "markdown", - "id": "b65ee2f0", + "id": "a32e4148", "metadata": { "editable": true }, @@ -3312,7 +4138,7 @@ }, { "cell_type": "markdown", - "id": "6526b426", + "id": "4050a8ed", "metadata": { "editable": true }, @@ -3322,7 +4148,7 @@ }, { "cell_type": "markdown", - "id": "29190fe5", + "id": "ec31e4e2", "metadata": { "editable": true }, @@ -3334,7 +4160,7 @@ }, { "cell_type": "markdown", - "id": "71b1585c", + "id": "f872c397", "metadata": { "editable": true }, @@ -3346,7 +4172,7 @@ }, { "cell_type": "markdown", - "id": "e18baaa9", + "id": "e8eeb2e5", "metadata": { "editable": true }, @@ -3356,7 +4182,7 @@ }, { "cell_type": "markdown", - "id": "1c58b8db", + "id": "54d2f39d", "metadata": { "editable": true }, @@ -3368,7 +4194,7 @@ }, { "cell_type": "markdown", - "id": "238ee8c5", + "id": "54cbc3a8", "metadata": { "editable": true }, @@ -3380,7 +4206,7 @@ }, { "cell_type": "markdown", - "id": "46389b82", + "id": "7ee728b6", "metadata": { "editable": true }, @@ -3392,7 +4218,7 @@ }, { "cell_type": "markdown", - "id": "0ecbb2d2", + "id": "00e9a0eb", "metadata": { "editable": true }, @@ -3402,7 +4228,7 @@ }, { "cell_type": "markdown", - "id": "27368dfb", + "id": "50f72b22", "metadata": { "editable": true }, @@ -3414,7 +4240,7 @@ }, { "cell_type": "markdown", - "id": "27082f48", + "id": "d53fea2e", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/_build/jupyter_execute/week36.py b/doc/LectureNotes/_build/jupyter_execute/week36.py index 0ecedfda3..7f96e675c 100644 --- a/doc/LectureNotes/_build/jupyter_execute/week36.py +++ b/doc/LectureNotes/_build/jupyter_execute/week36.py @@ -22,15 +22,21 @@ # # * Material for the lecture on Thursday September 7 # -# * Linear Regression and links with Statistics, Resampling methods +# * Technicalities related to scaling and other issues with data handling # -# * Recommended Reading: Goodfellow et al chapter 3 on probability theory, see URL:"" +# * Linear Regression and links with Statistics +# +# * [Recommended Reading: Goodfellow et al chapter 3 on probability theory](https://www.deeplearningbook.org/) # # * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis) +# +# * [Video of lecture](https://youtu.be/Kc20CFK0z7Y) +# +# * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep7.pdf) # ## Material for the active learning sessions Tuesday and Wednesday # -# The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples +# The material here contains a summary from last week and discussion of SVD, Ridge and Lasso regression with examples # ## Linear Regression and the SVD # @@ -844,6 +850,567 @@ plt.show() # ## Material for lecture Thursday September 7 +# ## Important technicalities: More on Rescaling data +# +# When you are comparing your own code with for example **Scikit-Learn**'s +# library, there are some technicalities to keep in mind. The examples +# here demonstrate some of these aspects with potential pitfalls. +# +# The discussion here focuses on the role of the intercept, how we can +# set up the design matrix, what scaling we should use and other topics +# which tend confuse us. +# +# The intercept can be interpreted as the expected value of our +# target/output variables when all other predictors are set to zero. +# Thus, if we cannot assume that the expected outputs/targets are zero +# when all predictors are zero (the columns in the design matrix), it +# may be a bad idea to implement a model which penalizes the intercept. +# Furthermore, in for example Ridge and Lasso regression, the default solutions +# from the library **Scikit-Learn** (when not shrinking $\beta_0$) for the unknown parameters +# $\boldsymbol{\beta}$, are derived under the assumption that both $\boldsymbol{y}$ and +# $\boldsymbol{X}$ are zero centered, that is we subtract the mean values. +# +# If our predictors represent different scales, then it is important to +# standardize the design matrix $\boldsymbol{X}$ by subtracting the mean of each +# column from the corresponding column and dividing the column with its +# standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library, +# the results may differ. +# +# The +# [Standardscaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html) +# function in **Scikit-Learn** does this for us. For the data sets we +# have been studying in our various examples, the data are in many cases +# already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a +# survey of your data, with a critical assessment of them in case you need to scale the data. +# +# If you need to scale the data, not doing so will give an *unfair* +# penalization of the parameters since their magnitude depends on the +# scale of their corresponding predictor. +# +# Suppose as an example that you +# you have an input variable given by the heights of different persons. +# Human height might be measured in inches or meters or +# kilometers. If measured in kilometers, a standard linear regression +# model with this predictor would probably give a much bigger +# coefficient term, than if measured in millimeters. +# This can clearly lead to problems in evaluating the cost/loss functions. +# +# Keep in mind that when you transform your data set before training a model, the same transformation needs to be done +# on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as + +# In[7]: + + +""" +#Model training, we compute the mean value of y and X +y_train_mean = np.mean(y_train) +X_train_mean = np.mean(X_train,axis=0) +X_train = X_train - X_train_mean +y_train = y_train - y_train_mean + +# The we fit our model with the training data +trained_model = some_model.fit(X_train,y_train) + + +#Model prediction, we need also to transform our data set used for the prediction. +X_test = X_test - X_train_mean #Use mean from training data +y_pred = trained_model(X_test) +y_pred = y_pred + y_train_mean +""" + + +# Let us try to understand what this may imply mathematically when we +# subtract the mean values, also known as *zero centering*. For +# simplicity, we will focus on ordinary regression, as done in the above example. +# +# The cost/loss function for regression is + +# $$ +# C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,. +# $$ + +# Recall also that we use the squared value. This expression can lead to an +# increased penalty for higher differences between predicted and +# output/target values. +# +# What we have done is to single out the $\beta_0$ term in the +# definition of the mean squared error (MSE). The design matrix $X$ +# does in this case not contain any intercept column. When we take the +# derivative with respect to $\beta_0$, we want the derivative to obey + +# $$ +# \frac{\partial C}{\partial \beta_j} = 0, +# $$ + +# for all $j$. For $\beta_0$ we have + +# $$ +# \frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right). +# $$ + +# Multiplying away the constant $2/n$, we obtain + +# $$ +# \sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j. +# $$ + +# Let us specialize first to the case where we have only two parameters $\beta_0$ and $\beta_1$. +# Our result for $\beta_0$ simplifies then to + +# $$ +# n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1. +# $$ + +# We obtain then + +# $$ +# \beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}. +# $$ + +# If we define + +# $$ +# \mu_{\boldsymbol{x}_1}=\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}, +# $$ + +# and the mean value of the outputs as + +# $$ +# \mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i, +# $$ + +# we have + +# $$ +# \beta_0 = \mu_y - \beta_1\mu_{\boldsymbol{x}_1}. +# $$ + +# In the general case with more parameters than $\beta_0$ and $\beta_1$, we have + +# $$ +# \beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j. +# $$ + +# We can rewrite the latter equation as + +# $$ +# \beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \sum_{j=1}^{p-1} \mu_{\boldsymbol{x}_j}\beta_j, +# $$ + +# where we have defined + +# $$ +# \mu_{\boldsymbol{x}_j}=\frac{1}{n}\sum_{i=0}^{n-1} X_{ij}, +# $$ + +# the mean value for all elements of the column vector $\boldsymbol{x}_j$. +# +# Replacing $y_i$ with $y_i - y_i - \overline{\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise) + +# $$ +# C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}). +# $$ + +# If we minimize with respect to $\boldsymbol{\beta}$ we have then + +# $$ +# \hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}, +# $$ + +# where $\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}}$ +# and $\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}$. +# +# For Ridge regression we need to add $\lambda \boldsymbol{\beta}^T\boldsymbol{\beta}$ to the cost function and get then + +# $$ +# \hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}. +# $$ + +# What does this mean? And why do we insist on all this? Let us look at some examples. +# +# This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only. +# Note also that we do not split the data into training and test. + +# In[8]: + + +import numpy as np +import matplotlib.pyplot as plt + +from sklearn.linear_model import LinearRegression + + +np.random.seed(2021) + +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n + + +def fit_beta(X, y): + return np.linalg.pinv(X.T @ X) @ X.T @ y + + +true_beta = [2, 0.5, 3.7] + +x = np.linspace(0, 1, 11) +y = np.sum( + np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0 +) + 0.1 * np.random.normal(size=len(x)) + +degree = 3 +X = np.zeros((len(x), degree)) + +# Include the intercept in the design matrix +for p in range(degree): + X[:, p] = x ** p + +beta = fit_beta(X, y) + +# Intercept is included in the design matrix +skl = LinearRegression(fit_intercept=False).fit(X, y) + +print(f"True beta: {true_beta}") +print(f"Fitted beta: {beta}") +print(f"Sklearn fitted beta: {skl.coef_}") +ypredictOwn = X @ beta +ypredictSKL = skl.predict(X) +print(f"MSE with intercept column") +print(MSE(y,ypredictOwn)) +print(f"MSE with intercept column from SKL") +print(MSE(y,ypredictSKL)) + + +plt.figure() +plt.scatter(x, y, label="Data") +plt.plot(x, X @ beta, label="Fit") +plt.plot(x, skl.predict(X), label="Sklearn (fit_intercept=False)") + + +# Do not include the intercept in the design matrix +X = np.zeros((len(x), degree - 1)) + +for p in range(degree - 1): + X[:, p] = x ** (p + 1) + +# Intercept is not included in the design matrix +skl = LinearRegression(fit_intercept=True).fit(X, y) + +# Use centered values for X and y when computing coefficients +y_offset = np.average(y, axis=0) +X_offset = np.average(X, axis=0) + +beta = fit_beta(X - X_offset, y - y_offset) +intercept = np.mean(y_offset - X_offset @ beta) + +print(f"Manual intercept: {intercept}") +print(f"Fitted beta (wiothout intercept): {beta}") +print(f"Sklearn intercept: {skl.intercept_}") +print(f"Sklearn fitted beta (without intercept): {skl.coef_}") +ypredictOwn = X @ beta +ypredictSKL = skl.predict(X) +print(f"MSE with Manual intercept") +print(MSE(y,ypredictOwn+intercept)) +print(f"MSE with Sklearn intercept") +print(MSE(y,ypredictSKL)) + +plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)") +plt.plot(x, skl.predict(X), "--", label="Sklearn (fit_intercept=True)") +plt.grid() +plt.legend() + +plt.show() + + +# The intercept is the value of our output/target variable +# when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). +# +# Printing the MSE, we see first that both methods give the same MSE, as +# they should. However, when we move to for example Ridge regression, +# the way we treat the intercept may give a larger or smaller MSE, +# meaning that the MSE can be penalized by the value of the +# intercept. Not including the intercept in the fit, means that the +# regularization term does not include $\beta_0$. For different values +# of $\lambda$, this may lead to different MSE values. +# +# To remind the reader, the regularization term, with the intercept in Ridge regression, is given by + +# $$ +# \lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2, +# $$ + +# but when we take out the intercept, this equation becomes + +# $$ +# \lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2. +# $$ + +# For Lasso regression we have + +# $$ +# \lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert. +# $$ + +# It means that, when scaling the design matrix and the outputs/targets, +# by subtracting the mean values, we have an optimization problem which +# is not penalized by the intercept. The MSE value can then be smaller +# since it focuses only on the remaining quantities. If we however bring +# back the intercept, we will get a MSE which then contains the +# intercept. +# +# Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set. + +# In[9]: + + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +from sklearn import linear_model + +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n + + +# A seed just to ensure that the random numbers are the same for every run. +# Useful for eventual debugging. +np.random.seed(3155) + +n = 100 +x = np.random.rand(n) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + +Maxpolydegree = 20 +X = np.zeros((n,Maxpolydegree)) +#We include explicitely the intercept column +for degree in range(Maxpolydegree): + X[:,degree] = x**degree +# We split the data in test and training data +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + +p = Maxpolydegree +I = np.eye(p,p) +# Decide which values of lambda to use +nlambdas = 6 +MSEOwnRidgePredict = np.zeros(nlambdas) +MSERidgePredict = np.zeros(nlambdas) +lambdas = np.logspace(-4, 2, nlambdas) +for i in range(nlambdas): + lmb = lambdas[i] + OwnRidgeBeta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train + # Note: we include the intercept column and no scaling + RegRidge = linear_model.Ridge(lmb,fit_intercept=False) + RegRidge.fit(X_train,y_train) + # and then make the prediction + ytildeOwnRidge = X_train @ OwnRidgeBeta + ypredictOwnRidge = X_test @ OwnRidgeBeta + ytildeRidge = RegRidge.predict(X_train) + ypredictRidge = RegRidge.predict(X_test) + MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge) + MSERidgePredict[i] = MSE(y_test,ypredictRidge) + print("Beta values for own Ridge implementation") + print(OwnRidgeBeta) + print("Beta values for Scikit-Learn Ridge implementation") + print(RegRidge.coef_) + print("MSE values for own Ridge implementation") + print(MSEOwnRidgePredict[i]) + print("MSE values for Scikit-Learn Ridge implementation") + print(MSERidgePredict[i]) + +# Now plot the results +plt.figure() +plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test') +plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test') + +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +# The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix. +# We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix. +# What happens if we do not include the intercept in our fit? +# Let us see how we can change this code by zero centering. + +# In[10]: + + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +from sklearn import linear_model +from sklearn.preprocessing import StandardScaler + +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n +# A seed just to ensure that the random numbers are the same for every run. +# Useful for eventual debugging. +np.random.seed(315) + +n = 100 +x = np.random.rand(n) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + +Maxpolydegree = 20 +X = np.zeros((n,Maxpolydegree-1)) + +for degree in range(1,Maxpolydegree): #No intercept column + X[:,degree-1] = x**(degree) + +# We split the data in test and training data +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + +#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable +X_train_mean = np.mean(X_train,axis=0) +#Center by removing mean from each feature +X_train_scaled = X_train - X_train_mean +X_test_scaled = X_test - X_train_mean +#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered) +#Remove the intercept from the training data. +y_scaler = np.mean(y_train) +y_train_scaled = y_train - y_scaler + +p = Maxpolydegree-1 +I = np.eye(p,p) +# Decide which values of lambda to use +nlambdas = 6 +MSEOwnRidgePredict = np.zeros(nlambdas) +MSERidgePredict = np.zeros(nlambdas) + +lambdas = np.logspace(-4, 2, nlambdas) +for i in range(nlambdas): + lmb = lambdas[i] + OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled) + intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data + #Add intercept to prediction + ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler + RegRidge = linear_model.Ridge(lmb) + RegRidge.fit(X_train,y_train) + ypredictRidge = RegRidge.predict(X_test) + MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge) + MSERidgePredict[i] = MSE(y_test,ypredictRidge) + print("Beta values for own Ridge implementation") + print(OwnRidgeBeta) #Intercept is given by mean of target variable + print("Beta values for Scikit-Learn Ridge implementation") + print(RegRidge.coef_) + print('Intercept from own implementation:') + print(intercept_) + print('Intercept from Scikit-Learn Ridge implementation') + print(RegRidge.intercept_) + print("MSE values for own Ridge implementation") + print(MSEOwnRidgePredict[i]) + print("MSE values for Scikit-Learn Ridge implementation") + print(MSERidgePredict[i]) + + +# Now plot the results +plt.figure() +plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test') +plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test') +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +# We see here, when compared to the code which includes explicitely the +# intercept column, that our MSE value is actually smaller. This is +# because the regularization term does not include the intercept value +# $\beta_0$ in the fitting. This applies to Lasso regularization as +# well. It means that our optimization is now done only with the +# centered matrix and/or vector that enter the fitting procedure. + +# ## Test Function for what happens with OLS, Ridge and Lasso +# +# Hitherto we have discussed Ridge and Lasso regression in terms of a +# linear analysis. This may to many of you feel rather technical and +# perhaps not that intuitive. The question is whether we can develop a +# more intuitive way of understanding what Ridge and Lasso express. +# +# Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. +# +# We will play around with a study of the values for the optimal +# parameters $\boldsymbol{\beta}$ using OLS, Ridge and Lasso regression. For +# OLS, you will notice as function of the noise and polynomial degree, +# that the parameters $\beta$ will fluctuate from order to order in the +# polynomial fit and that for larger and larger polynomial degrees of freedom, the parameters will tend to increase in value for OLS. +# +# For Ridge and Lasso regression, the higher order parameters will typically be reduced, providing thereby less fluctuations from one order to another one. + +# In[11]: + + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +from sklearn import linear_model + +def R2(y_data, y_model): + return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2) +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n + +# Make data set. +n = 10000 +x = np.random.rand(n) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.randn(n) + +Maxpolydegree = 5 +X = np.zeros((len(x),Maxpolydegree)) +X[:,0] = 1.0 + + +for polydegree in range(1,Maxpolydegree): + X[:,polydegree] = x**(polydegree) + +# We split the data in test and training data +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) + +# matrix inversion to find beta +OLSbeta = np.linalg.pinv(X_train.T @ X_train) @ X_train.T @ y_train +print(OLSbeta) +ypredictOLS = X_test @ OLSbeta +print("Test MSE OLS") +print(MSE(y_test,ypredictOLS)) +# Repeat now for Lasso and Ridge regression and various values of the regularization parameter using Scikit-Learn +# Decide which values of lambda to use +nlambdas = 4 +MSERidgePredict = np.zeros(nlambdas) +MSELassoPredict = np.zeros(nlambdas) +lambdas = np.logspace(-3, 1, nlambdas) +for i in range(nlambdas): + lmb = lambdas[i] + # Make the fit using Ridge and Lasso + RegRidge = linear_model.Ridge(lmb,fit_intercept=False) + RegRidge.fit(X_train,y_train) + RegLasso = linear_model.Lasso(lmb,fit_intercept=False) + RegLasso.fit(X_train,y_train) + # and then make the prediction + ypredictRidge = RegRidge.predict(X_test) + ypredictLasso = RegLasso.predict(X_test) + # Compute the MSE and print it + MSERidgePredict[i] = MSE(y_test,ypredictRidge) + MSELassoPredict[i] = MSE(y_test,ypredictLasso) + print(lmb,RegRidge.coef_) + print(lmb,RegLasso.coef_) +# Now plot the results +plt.figure() +plt.plot(np.log10(lambdas), MSERidgePredict, 'b', label = 'MSE Ridge Test') +plt.plot(np.log10(lambdas), MSELassoPredict, 'r', label = 'MSE Lasso Test') +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +# How can we understand this? + # ## Linking the regression analysis with a statistical interpretation # # We will now couple the discussions of ordinary least squares, Ridge @@ -1227,95 +1794,6 @@ plt.show() # ## Bayes' Theorem and Ridge and Lasso Regression # -# Hitherto we have discussed Ridge and Lasso regression in terms of a -# linear analysis. This may to many of you feel rather technical and -# perhaps not that intuitive. The question is whether we can develop a -# more intuitive way of understanding what Ridge and Lasso express. -# -# Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. - -# ## Test Function for what happens with OLS, Ridge and Lasso -# -# We will play around with a study of the values for the optimal -# parameters $\boldsymbol{\beta}$ using OLS, Ridge and Lasso regression. For -# OLS, you will notice as function of the noise and polynomial degree, -# that the parameters $\beta$ will fluctuate from order to order in the -# polynomial fit and that for larger and larger polynomial degrees of freedom, the parameters will tend to increase in value for OLS. -# -# For Ridge and Lasso regression, the higher order parameters will typically be reduced, providing thereby less fluctuations from one order to another one. - -# In[7]: - - -import numpy as np -import matplotlib.pyplot as plt -from sklearn.model_selection import train_test_split -from sklearn import linear_model - -def R2(y_data, y_model): - return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2) -def MSE(y_data,y_model): - n = np.size(y_model) - return np.sum((y_data-y_model)**2)/n - -# Make data set. -n = 10000 -x = np.random.rand(n) -y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.randn(n) - -Maxpolydegree = 5 -X = np.zeros((len(x),Maxpolydegree)) -X[:,0] = 1.0 - -for polydegree in range(1, Maxpolydegree): - for degree in range(polydegree): - X[:,degree] = x**(degree) - - -# We split the data in test and training data -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) - -# matrix inversion to find beta -OLSbeta = np.linalg.pinv(X_train.T @ X_train) @ X_train.T @ y_train -print(OLSbeta) -ypredictOLS = X_test @ OLSbeta -print("Test MSE OLS") -print(MSE(y_test,ypredictOLS)) -# Repeat now for Lasso and Ridge regression and various values of the regularization parameter using Scikit-Learn -# Decide which values of lambda to use -nlambdas = 4 -MSERidgePredict = np.zeros(nlambdas) -MSELassoPredict = np.zeros(nlambdas) -lambdas = np.logspace(-3, 1, nlambdas) -for i in range(nlambdas): - lmb = lambdas[i] - # Make the fit using Ridge and Lasso - RegRidge = linear_model.Ridge(lmb,fit_intercept=False) - RegRidge.fit(X_train,y_train) - RegLasso = linear_model.Lasso(lmb,fit_intercept=False) - RegLasso.fit(X_train,y_train) - # and then make the prediction - ypredictRidge = RegRidge.predict(X_test) - ypredictLasso = RegLasso.predict(X_test) - # Compute the MSE and print it - MSERidgePredict[i] = MSE(y_test,ypredictRidge) - MSELassoPredict[i] = MSE(y_test,ypredictLasso) - print(lmb,RegRidge.coef_) - print(lmb,RegLasso.coef_) -# Now plot the results -plt.figure() -plt.plot(np.log10(lambdas), MSERidgePredict, 'b', label = 'MSE Ridge Test') -plt.plot(np.log10(lambdas), MSELassoPredict, 'r', label = 'MSE Lasso Test') -plt.xlabel('log10(lambda)') -plt.ylabel('MSE') -plt.legend() -plt.show() - - -# How can we understand this? - -# ## Invoking Bayes' theorem -# # Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. # # For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\boldsymbol{D}$ (one-dimensional case) diff --git a/doc/LectureNotes/exercisesweek37.ipynb b/doc/LectureNotes/exercisesweek37.ipynb index c565e7b07..39e531a60 100644 --- a/doc/LectureNotes/exercisesweek37.ipynb +++ b/doc/LectureNotes/exercisesweek37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "cfebae03", + "id": "1ff4a4e9", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "353ebb07", + "id": "0569f7a9", "metadata": { "editable": true }, @@ -27,25 +27,34 @@ }, { "cell_type": "markdown", - "id": "c02815dd", + "id": "6f34e507", "metadata": { "editable": true }, "source": [ "## Overarching aims of the exercises this week\n", "\n", - "This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercise is also a part of project 1 and can be reused in the theory part of the project.\n", + "This exercise deals with various mean values and variances in linear\n", + "regression method (here it may be useful to look up chapter 3,\n", + "equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome\n", + "H. Friedman, The Elements of Statistical Learning,\n", + "Springer](https://www.springer.com/gp/book/9780387848570)). The\n", + "exercise is also a part of project 1 and can be reused in the theory\n", + "part of the project.\n", "\n", - "For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "For more discussions on Ridge regression and calculation of\n", + "expectation values, [Wessel van\n", + "Wieringen's](https://arxiv.org/abs/1509.09169) article is highly\n", + "recommended.\n", "\n", - "The assumption we have made is \n", - "that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$\n", - "which describes our data" + "The assumption we have made is that there exists a continuous function\n", + "$f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0,\n", + "\\sigma^2)$ which describes our data" ] }, { "cell_type": "markdown", - "id": "fb152043", + "id": "3ff574a4", "metadata": { "editable": true }, @@ -57,7 +66,7 @@ }, { "cell_type": "markdown", - "id": "9aba7a46", + "id": "096a3165", "metadata": { "editable": true }, @@ -68,7 +77,7 @@ }, { "cell_type": "markdown", - "id": "be52c097", + "id": "cc9fb258", "metadata": { "editable": true }, @@ -80,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "c54979e5", + "id": "daf4eb5d", "metadata": { "editable": true }, @@ -90,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "4b1eb811", + "id": "3b491679", "metadata": { "editable": true }, @@ -102,7 +111,7 @@ }, { "cell_type": "markdown", - "id": "036a5440", + "id": "93a49739", "metadata": { "editable": true }, @@ -114,7 +123,7 @@ }, { "cell_type": "markdown", - "id": "f7fb8094", + "id": "a3fa80db", "metadata": { "editable": true }, @@ -125,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "d7d92995", + "id": "e6e2defd", "metadata": { "editable": true }, @@ -137,7 +146,7 @@ }, { "cell_type": "markdown", - "id": "29eb9701", + "id": "9aa19afa", "metadata": { "editable": true }, @@ -150,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "aa106511", + "id": "5fa82a47", "metadata": { "editable": true }, @@ -162,7 +171,7 @@ }, { "cell_type": "markdown", - "id": "58be1091", + "id": "33099abb", "metadata": { "editable": true }, @@ -172,7 +181,7 @@ }, { "cell_type": "markdown", - "id": "4cb58e41", + "id": "5d03f001", "metadata": { "editable": true }, @@ -184,7 +193,7 @@ }, { "cell_type": "markdown", - "id": "a04df28d", + "id": "94bd151d", "metadata": { "editable": true }, @@ -195,7 +204,7 @@ }, { "cell_type": "markdown", - "id": "c095c96a", + "id": "f0b84941", "metadata": { "editable": true }, @@ -207,20 +216,20 @@ }, { "cell_type": "markdown", - "id": "5d425a79", + "id": "60370a63", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", + "\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\\n", "\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "fb89cfab", + "id": "af251217", "metadata": { "editable": true }, @@ -233,19 +242,19 @@ }, { "cell_type": "markdown", - "id": "aea8801c", + "id": "02e978da", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\mbox{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e885693", + "id": "3656a896", "metadata": { "editable": true }, diff --git a/doc/LectureNotes/week36.ipynb b/doc/LectureNotes/week36.ipynb index bdaee795a..ededd9147 100644 --- a/doc/LectureNotes/week36.ipynb +++ b/doc/LectureNotes/week36.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "7411173e", + "id": "2b900fea", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "f50db913", + "id": "2286e211", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "4a90e380", + "id": "edbcd7f9", "metadata": { "editable": true }, @@ -44,28 +44,34 @@ "\n", "* Material for the lecture on Thursday September 7\n", "\n", - " * Linear Regression and links with Statistics, Resampling methods\n", + " * Technicalities related to scaling and other issues with data handling\n", "\n", - " * Recommended Reading: Goodfellow et al chapter 3 on probability theory, see URL:\"\"\n", + " * Linear Regression and links with Statistics\n", "\n", - " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)" + " * [Recommended Reading: Goodfellow et al chapter 3 on probability theory](https://www.deeplearningbook.org/)\n", + "\n", + " * See also Murphy, sections 2.4 (Gaussian distributions) and 3.2 (Bayesian Statistics, basis)\n", + "\n", + " * [Video of lecture](https://youtu.be/Kc20CFK0z7Y) \n", + "\n", + " * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep7.pdf)" ] }, { "cell_type": "markdown", - "id": "351d36fd", + "id": "5dbf3ff5", "metadata": { "editable": true }, "source": [ "## Material for the active learning sessions Tuesday and Wednesday\n", "\n", - "The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples" + "The material here contains a summary from last week and discussion of SVD, Ridge and Lasso regression with examples" ] }, { "cell_type": "markdown", - "id": "0f9f4d2f", + "id": "2dae8175", "metadata": { "editable": true }, @@ -77,7 +83,7 @@ }, { "cell_type": "markdown", - "id": "9ec36bdd", + "id": "fb3d486d", "metadata": { "editable": true }, @@ -89,7 +95,7 @@ }, { "cell_type": "markdown", - "id": "614493c6", + "id": "5f8492a1", "metadata": { "editable": true }, @@ -99,7 +105,7 @@ }, { "cell_type": "markdown", - "id": "dc4be423", + "id": "a35422b4", "metadata": { "editable": true }, @@ -111,7 +117,7 @@ }, { "cell_type": "markdown", - "id": "b262332c", + "id": "aef923ea", "metadata": { "editable": true }, @@ -121,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "ab1829a4", + "id": "5da8277b", "metadata": { "editable": true }, @@ -138,7 +144,7 @@ }, { "cell_type": "markdown", - "id": "7c82dd9e", + "id": "eb34e530", "metadata": { "editable": true }, @@ -148,7 +154,7 @@ }, { "cell_type": "markdown", - "id": "5bab8721", + "id": "a2a33728", "metadata": { "editable": true }, @@ -160,7 +166,7 @@ }, { "cell_type": "markdown", - "id": "72273a4f", + "id": "9bea1837", "metadata": { "editable": true }, @@ -170,7 +176,7 @@ }, { "cell_type": "markdown", - "id": "d4af03ea", + "id": "af711505", "metadata": { "editable": true }, @@ -182,7 +188,7 @@ }, { "cell_type": "markdown", - "id": "c0d35853", + "id": "f02b3738", "metadata": { "editable": true }, @@ -196,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "db5ea827", + "id": "d3fe4855", "metadata": { "editable": true }, @@ -208,7 +214,7 @@ }, { "cell_type": "markdown", - "id": "a6b341ce", + "id": "7b5fa376", "metadata": { "editable": true }, @@ -230,7 +236,7 @@ }, { "cell_type": "markdown", - "id": "db723709", + "id": "d212be4e", "metadata": { "editable": true }, @@ -242,7 +248,7 @@ }, { "cell_type": "markdown", - "id": "1e600268", + "id": "eca13576", "metadata": { "editable": true }, @@ -257,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "b9452938", + "id": "fbf0e2be", "metadata": { "editable": true }, @@ -269,7 +275,7 @@ }, { "cell_type": "markdown", - "id": "53fea1e6", + "id": "acbd86ec", "metadata": { "editable": true }, @@ -281,7 +287,7 @@ }, { "cell_type": "markdown", - "id": "ed10baad", + "id": "8887e338", "metadata": { "editable": true }, @@ -291,7 +297,7 @@ }, { "cell_type": "markdown", - "id": "a4d0db96", + "id": "e95bcbda", "metadata": { "editable": true }, @@ -303,7 +309,7 @@ }, { "cell_type": "markdown", - "id": "fae3cadc", + "id": "e66200c2", "metadata": { "editable": true }, @@ -313,7 +319,7 @@ }, { "cell_type": "markdown", - "id": "e973c775", + "id": "a0c3692c", "metadata": { "editable": true }, @@ -325,7 +331,7 @@ }, { "cell_type": "markdown", - "id": "f88a33f4", + "id": "8d13a9ff", "metadata": { "editable": true }, @@ -335,7 +341,7 @@ }, { "cell_type": "markdown", - "id": "5bc7214d", + "id": "31c3dd91", "metadata": { "editable": true }, @@ -347,7 +353,7 @@ }, { "cell_type": "markdown", - "id": "561d4c91", + "id": "87a6b291", "metadata": { "editable": true }, @@ -364,7 +370,7 @@ }, { "cell_type": "markdown", - "id": "0eda2d1e", + "id": "3bb9eed3", "metadata": { "editable": true }, @@ -378,7 +384,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "c95dbc32", + "id": "ba123060", "metadata": { "collapsed": false, "editable": true @@ -392,7 +398,7 @@ }, { "cell_type": "markdown", - "id": "77079d25", + "id": "46fdfd39", "metadata": { "editable": true }, @@ -403,7 +409,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "cd0ecc44", + "id": "36e47fde", "metadata": { "collapsed": false, "editable": true @@ -443,7 +449,7 @@ }, { "cell_type": "markdown", - "id": "41d896dc", + "id": "53878874", "metadata": { "editable": true }, @@ -463,7 +469,7 @@ }, { "cell_type": "markdown", - "id": "f594852c", + "id": "87546753", "metadata": { "editable": true }, @@ -475,7 +481,7 @@ }, { "cell_type": "markdown", - "id": "b78264f2", + "id": "d8a4669d", "metadata": { "editable": true }, @@ -486,7 +492,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "d05eff95", + "id": "b6523ecb", "metadata": { "collapsed": false, "editable": true @@ -520,7 +526,7 @@ }, { "cell_type": "markdown", - "id": "2fae32c6", + "id": "2fce573f", "metadata": { "editable": true }, @@ -530,7 +536,7 @@ }, { "cell_type": "markdown", - "id": "8631011a", + "id": "840991d3", "metadata": { "editable": true }, @@ -543,7 +549,7 @@ }, { "cell_type": "markdown", - "id": "8e7701de", + "id": "4df7aac6", "metadata": { "editable": true }, @@ -555,7 +561,7 @@ }, { "cell_type": "markdown", - "id": "aea7230b", + "id": "3f3acc05", "metadata": { "editable": true }, @@ -565,7 +571,7 @@ }, { "cell_type": "markdown", - "id": "c5029213", + "id": "2d1218f6", "metadata": { "editable": true }, @@ -578,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "535ad70e", + "id": "01624e31", "metadata": { "editable": true }, @@ -588,7 +594,7 @@ }, { "cell_type": "markdown", - "id": "159ae526", + "id": "616fe159", "metadata": { "editable": true }, @@ -600,7 +606,7 @@ }, { "cell_type": "markdown", - "id": "2f25dfa4", + "id": "27cf2acd", "metadata": { "editable": true }, @@ -615,7 +621,7 @@ }, { "cell_type": "markdown", - "id": "9d187806", + "id": "2ff1e0d2", "metadata": { "editable": true }, @@ -628,7 +634,7 @@ }, { "cell_type": "markdown", - "id": "6c692db5", + "id": "aef1996b", "metadata": { "editable": true }, @@ -642,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "3404e0b4", + "id": "75a44cc9", "metadata": { "editable": true }, @@ -654,7 +660,7 @@ }, { "cell_type": "markdown", - "id": "4bc49666", + "id": "e2ba6037", "metadata": { "editable": true }, @@ -664,7 +670,7 @@ }, { "cell_type": "markdown", - "id": "3f08dfa1", + "id": "95517053", "metadata": { "editable": true }, @@ -677,7 +683,7 @@ }, { "cell_type": "markdown", - "id": "30bcc0cd", + "id": "5f15d4ea", "metadata": { "editable": true }, @@ -689,7 +695,7 @@ }, { "cell_type": "markdown", - "id": "d311a7d8", + "id": "68579bd0", "metadata": { "editable": true }, @@ -701,7 +707,7 @@ }, { "cell_type": "markdown", - "id": "32c46b73", + "id": "53887938", "metadata": { "editable": true }, @@ -713,7 +719,7 @@ }, { "cell_type": "markdown", - "id": "3cceda87", + "id": "469c8c35", "metadata": { "editable": true }, @@ -725,7 +731,7 @@ }, { "cell_type": "markdown", - "id": "10a16675", + "id": "a7521a38", "metadata": { "editable": true }, @@ -739,7 +745,7 @@ }, { "cell_type": "markdown", - "id": "6dba20bf", + "id": "cdfddb46", "metadata": { "editable": true }, @@ -751,7 +757,7 @@ }, { "cell_type": "markdown", - "id": "9325f924", + "id": "e87c8606", "metadata": { "editable": true }, @@ -761,7 +767,7 @@ }, { "cell_type": "markdown", - "id": "745d61e9", + "id": "3e06d524", "metadata": { "editable": true }, @@ -773,7 +779,7 @@ }, { "cell_type": "markdown", - "id": "3866af64", + "id": "798d77a9", "metadata": { "editable": true }, @@ -783,7 +789,7 @@ }, { "cell_type": "markdown", - "id": "710de288", + "id": "8808de88", "metadata": { "editable": true }, @@ -795,7 +801,7 @@ }, { "cell_type": "markdown", - "id": "5893fba0", + "id": "c3825a07", "metadata": { "editable": true }, @@ -807,7 +813,7 @@ }, { "cell_type": "markdown", - "id": "5ef8c59f", + "id": "868b6a19", "metadata": { "editable": true }, @@ -817,7 +823,7 @@ }, { "cell_type": "markdown", - "id": "79b257ae", + "id": "eaba5f13", "metadata": { "editable": true }, @@ -828,7 +834,7 @@ }, { "cell_type": "markdown", - "id": "46270996", + "id": "c9cc77c3", "metadata": { "editable": true }, @@ -840,7 +846,7 @@ }, { "cell_type": "markdown", - "id": "785aebe9", + "id": "4b3e1232", "metadata": { "editable": true }, @@ -856,7 +862,7 @@ }, { "cell_type": "markdown", - "id": "443f3a3f", + "id": "d59ed944", "metadata": { "editable": true }, @@ -869,7 +875,7 @@ }, { "cell_type": "markdown", - "id": "cfb8d4c0", + "id": "e81dd0be", "metadata": { "editable": true }, @@ -881,7 +887,7 @@ }, { "cell_type": "markdown", - "id": "2be1ca1f", + "id": "f26a385c", "metadata": { "editable": true }, @@ -891,7 +897,7 @@ }, { "cell_type": "markdown", - "id": "9ed0aec2", + "id": "aa00d97a", "metadata": { "editable": true }, @@ -903,7 +909,7 @@ }, { "cell_type": "markdown", - "id": "943b1cd5", + "id": "17ba8d1c", "metadata": { "editable": true }, @@ -913,7 +919,7 @@ }, { "cell_type": "markdown", - "id": "4a8a6bff", + "id": "8cf9dc8f", "metadata": { "editable": true }, @@ -925,7 +931,7 @@ }, { "cell_type": "markdown", - "id": "b0371965", + "id": "f0c43d20", "metadata": { "editable": true }, @@ -937,7 +943,7 @@ }, { "cell_type": "markdown", - "id": "38e7b497", + "id": "5b119d73", "metadata": { "editable": true }, @@ -953,7 +959,7 @@ }, { "cell_type": "markdown", - "id": "682f90db", + "id": "1efba158", "metadata": { "editable": true }, @@ -965,7 +971,7 @@ }, { "cell_type": "markdown", - "id": "b5fb7771", + "id": "e5943a40", "metadata": { "editable": true }, @@ -977,7 +983,7 @@ }, { "cell_type": "markdown", - "id": "0c228525", + "id": "167a1a9e", "metadata": { "editable": true }, @@ -987,7 +993,7 @@ }, { "cell_type": "markdown", - "id": "08a400af", + "id": "924b5c9b", "metadata": { "editable": true }, @@ -999,7 +1005,7 @@ }, { "cell_type": "markdown", - "id": "ec0808b1", + "id": "56ac0a65", "metadata": { "editable": true }, @@ -1009,7 +1015,7 @@ }, { "cell_type": "markdown", - "id": "92d3d92f", + "id": "4323c860", "metadata": { "editable": true }, @@ -1021,7 +1027,7 @@ }, { "cell_type": "markdown", - "id": "07486e29", + "id": "12ef13b2", "metadata": { "editable": true }, @@ -1038,7 +1044,7 @@ }, { "cell_type": "markdown", - "id": "9868d4f4", + "id": "d33c685e", "metadata": { "editable": true }, @@ -1050,7 +1056,7 @@ }, { "cell_type": "markdown", - "id": "1793a225", + "id": "b30d436d", "metadata": { "editable": true }, @@ -1062,7 +1068,7 @@ }, { "cell_type": "markdown", - "id": "c7d3fa52", + "id": "37e47a2a", "metadata": { "editable": true }, @@ -1072,7 +1078,7 @@ }, { "cell_type": "markdown", - "id": "47894977", + "id": "3f87ff41", "metadata": { "editable": true }, @@ -1084,7 +1090,7 @@ }, { "cell_type": "markdown", - "id": "d1a11dc8", + "id": "b14d10ff", "metadata": { "editable": true }, @@ -1094,7 +1100,7 @@ }, { "cell_type": "markdown", - "id": "a6cf82de", + "id": "672b64b5", "metadata": { "editable": true }, @@ -1106,7 +1112,7 @@ }, { "cell_type": "markdown", - "id": "bbf7ccc7", + "id": "ef321df6", "metadata": { "editable": true }, @@ -1116,7 +1122,7 @@ }, { "cell_type": "markdown", - "id": "bc6dbb83", + "id": "e67e683d", "metadata": { "editable": true }, @@ -1128,7 +1134,7 @@ }, { "cell_type": "markdown", - "id": "ef5a8292", + "id": "b9bd1d3f", "metadata": { "editable": true }, @@ -1138,7 +1144,7 @@ }, { "cell_type": "markdown", - "id": "d77bf73a", + "id": "b705fb0b", "metadata": { "editable": true }, @@ -1153,7 +1159,7 @@ }, { "cell_type": "markdown", - "id": "265ce93f", + "id": "03679605", "metadata": { "editable": true }, @@ -1165,7 +1171,7 @@ }, { "cell_type": "markdown", - "id": "c19e720c", + "id": "b58e9ee7", "metadata": { "editable": true }, @@ -1175,7 +1181,7 @@ }, { "cell_type": "markdown", - "id": "8cad2962", + "id": "42d8e1b3", "metadata": { "editable": true }, @@ -1187,7 +1193,7 @@ }, { "cell_type": "markdown", - "id": "bf6316a2", + "id": "db624348", "metadata": { "editable": true }, @@ -1199,7 +1205,7 @@ }, { "cell_type": "markdown", - "id": "78413ab6", + "id": "8d37cd9f", "metadata": { "editable": true }, @@ -1211,7 +1217,7 @@ }, { "cell_type": "markdown", - "id": "a77ca41b", + "id": "e88bc71d", "metadata": { "editable": true }, @@ -1221,7 +1227,7 @@ }, { "cell_type": "markdown", - "id": "67b3f67e", + "id": "7bee2106", "metadata": { "editable": true }, @@ -1233,7 +1239,7 @@ }, { "cell_type": "markdown", - "id": "d3571c8b", + "id": "fc1dc754", "metadata": { "editable": true }, @@ -1245,7 +1251,7 @@ }, { "cell_type": "markdown", - "id": "943f822e", + "id": "e5c6d86f", "metadata": { "editable": true }, @@ -1257,7 +1263,7 @@ }, { "cell_type": "markdown", - "id": "2ec0ba01", + "id": "1413c7cf", "metadata": { "editable": true }, @@ -1267,7 +1273,7 @@ }, { "cell_type": "markdown", - "id": "85bf4c9b", + "id": "1b181b9b", "metadata": { "editable": true }, @@ -1279,7 +1285,7 @@ }, { "cell_type": "markdown", - "id": "c81d05f4", + "id": "ca27bd28", "metadata": { "editable": true }, @@ -1289,7 +1295,7 @@ }, { "cell_type": "markdown", - "id": "ce36390d", + "id": "f2ce6515", "metadata": { "editable": true }, @@ -1303,7 +1309,7 @@ }, { "cell_type": "markdown", - "id": "a47dd697", + "id": "bbe27e9c", "metadata": { "editable": true }, @@ -1313,7 +1319,7 @@ }, { "cell_type": "markdown", - "id": "15fe5a0c", + "id": "3ddfa9f9", "metadata": { "editable": true }, @@ -1325,7 +1331,7 @@ }, { "cell_type": "markdown", - "id": "62d7f0e9", + "id": "2d31b195", "metadata": { "editable": true }, @@ -1337,7 +1343,7 @@ }, { "cell_type": "markdown", - "id": "f2d4576f", + "id": "4b092e9b", "metadata": { "editable": true }, @@ -1347,7 +1353,7 @@ }, { "cell_type": "markdown", - "id": "fb2bc0ff", + "id": "7c845085", "metadata": { "editable": true }, @@ -1359,7 +1365,7 @@ }, { "cell_type": "markdown", - "id": "807ac009", + "id": "0fb000c5", "metadata": { "editable": true }, @@ -1369,7 +1375,7 @@ }, { "cell_type": "markdown", - "id": "fe72889b", + "id": "bd4d8817", "metadata": { "editable": true }, @@ -1381,7 +1387,7 @@ }, { "cell_type": "markdown", - "id": "48844e10", + "id": "d0d10a69", "metadata": { "editable": true }, @@ -1393,7 +1399,7 @@ }, { "cell_type": "markdown", - "id": "72a4bc4c", + "id": "d9db36da", "metadata": { "editable": true }, @@ -1403,7 +1409,7 @@ }, { "cell_type": "markdown", - "id": "cfe1db6f", + "id": "a409bd8a", "metadata": { "editable": true }, @@ -1415,7 +1421,7 @@ }, { "cell_type": "markdown", - "id": "05b6e6f6", + "id": "c3be5f03", "metadata": { "editable": true }, @@ -1425,7 +1431,7 @@ }, { "cell_type": "markdown", - "id": "0e58f37a", + "id": "6533b635", "metadata": { "editable": true }, @@ -1437,7 +1443,7 @@ }, { "cell_type": "markdown", - "id": "78593de5", + "id": "3c6b3074", "metadata": { "editable": true }, @@ -1449,7 +1455,7 @@ }, { "cell_type": "markdown", - "id": "26b08a56", + "id": "747eecf9", "metadata": { "editable": true }, @@ -1459,7 +1465,7 @@ }, { "cell_type": "markdown", - "id": "3df661a4", + "id": "87542c6e", "metadata": { "editable": true }, @@ -1471,7 +1477,7 @@ }, { "cell_type": "markdown", - "id": "68460e68", + "id": "13f97260", "metadata": { "editable": true }, @@ -1484,7 +1490,7 @@ }, { "cell_type": "markdown", - "id": "6c660112", + "id": "bc02bc84", "metadata": { "editable": true }, @@ -1496,7 +1502,7 @@ }, { "cell_type": "markdown", - "id": "6ff54cf1", + "id": "9f9d8193", "metadata": { "editable": true }, @@ -1508,7 +1514,7 @@ }, { "cell_type": "markdown", - "id": "f3f88f94", + "id": "ba404c5e", "metadata": { "editable": true }, @@ -1520,7 +1526,7 @@ }, { "cell_type": "markdown", - "id": "6918fc7c", + "id": "9af7054a", "metadata": { "editable": true }, @@ -1530,7 +1536,7 @@ }, { "cell_type": "markdown", - "id": "1b2321a3", + "id": "eb7fd44a", "metadata": { "editable": true }, @@ -1542,7 +1548,7 @@ }, { "cell_type": "markdown", - "id": "d5e21457", + "id": "041b60b2", "metadata": { "editable": true }, @@ -1552,7 +1558,7 @@ }, { "cell_type": "markdown", - "id": "e03c3617", + "id": "f95dc455", "metadata": { "editable": true }, @@ -1564,7 +1570,7 @@ }, { "cell_type": "markdown", - "id": "6c4d9a4a", + "id": "28176b93", "metadata": { "editable": true }, @@ -1574,7 +1580,7 @@ }, { "cell_type": "markdown", - "id": "584bef2e", + "id": "b6f75bf7", "metadata": { "editable": true }, @@ -1586,7 +1592,7 @@ }, { "cell_type": "markdown", - "id": "87273242", + "id": "f6ad6840", "metadata": { "editable": true }, @@ -1596,7 +1602,7 @@ }, { "cell_type": "markdown", - "id": "f7e8277f", + "id": "fe93ca4d", "metadata": { "editable": true }, @@ -1609,7 +1615,7 @@ }, { "cell_type": "markdown", - "id": "e7167c28", + "id": "4d7921c1", "metadata": { "editable": true }, @@ -1621,7 +1627,7 @@ }, { "cell_type": "markdown", - "id": "6a3fa7b2", + "id": "d106e526", "metadata": { "editable": true }, @@ -1633,7 +1639,7 @@ }, { "cell_type": "markdown", - "id": "92c33067", + "id": "2ab11733", "metadata": { "editable": true }, @@ -1643,7 +1649,7 @@ }, { "cell_type": "markdown", - "id": "1b7be5cc", + "id": "c871f92c", "metadata": { "editable": true }, @@ -1655,7 +1661,7 @@ }, { "cell_type": "markdown", - "id": "9e2db51b", + "id": "4fc85aac", "metadata": { "editable": true }, @@ -1672,7 +1678,7 @@ }, { "cell_type": "markdown", - "id": "7ea5c023", + "id": "c66960ef", "metadata": { "editable": true }, @@ -1684,7 +1690,7 @@ }, { "cell_type": "markdown", - "id": "efff1845", + "id": "6c742a89", "metadata": { "editable": true }, @@ -1696,7 +1702,7 @@ }, { "cell_type": "markdown", - "id": "80496d8c", + "id": "5a6fe016", "metadata": { "editable": true }, @@ -1706,7 +1712,7 @@ }, { "cell_type": "markdown", - "id": "bb33463a", + "id": "63472c22", "metadata": { "editable": true }, @@ -1718,7 +1724,7 @@ }, { "cell_type": "markdown", - "id": "f16b1457", + "id": "3ede415b", "metadata": { "editable": true }, @@ -1728,7 +1734,7 @@ }, { "cell_type": "markdown", - "id": "4bee5a43", + "id": "90397204", "metadata": { "editable": true }, @@ -1740,7 +1746,7 @@ }, { "cell_type": "markdown", - "id": "17176990", + "id": "acd78254", "metadata": { "editable": true }, @@ -1750,7 +1756,7 @@ }, { "cell_type": "markdown", - "id": "ec2c8f34", + "id": "114e58b4", "metadata": { "editable": true }, @@ -1762,7 +1768,7 @@ }, { "cell_type": "markdown", - "id": "5934144d", + "id": "dc251844", "metadata": { "editable": true }, @@ -1772,7 +1778,7 @@ }, { "cell_type": "markdown", - "id": "fe097c51", + "id": "2fb1ac98", "metadata": { "editable": true }, @@ -1787,7 +1793,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "92cd1361", + "id": "8c64672d", "metadata": { "collapsed": false, "editable": true @@ -1849,7 +1855,7 @@ }, { "cell_type": "markdown", - "id": "9e4426f4", + "id": "a87e0be8", "metadata": { "editable": true }, @@ -1859,7 +1865,7 @@ }, { "cell_type": "markdown", - "id": "5f2b6935", + "id": "ac5bf925", "metadata": { "editable": true }, @@ -1870,7 +1876,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "409bbeac", + "id": "078628d7", "metadata": { "collapsed": false, "editable": true @@ -1937,7 +1943,7 @@ }, { "cell_type": "markdown", - "id": "67a24e77", + "id": "f035fb09", "metadata": { "editable": true }, @@ -1948,7 +1954,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "8da16a74", + "id": "acb14894", "metadata": { "collapsed": false, "editable": true @@ -2037,7 +2043,7 @@ }, { "cell_type": "markdown", - "id": "9d2563fa", + "id": "391b11f6", "metadata": { "editable": true }, @@ -2047,962 +2053,850 @@ }, { "cell_type": "markdown", - "id": "1232037e", + "id": "242933c8", "metadata": { "editable": true }, "source": [ - "## Linking the regression analysis with a statistical interpretation\n", + "## Important technicalities: More on Rescaling data\n", "\n", - "We will now couple the discussions of ordinary least squares, Ridge\n", - "and Lasso regression with a statistical interpretation, that is we\n", - "move from a linear algebra analysis to a statistical analysis. In\n", - "particular, we will focus on what the regularization terms can result\n", - "in. We will amongst other things show that the regularization\n", - "parameter can reduce considerably the variance of the parameters\n", - "$\\beta$.\n", + "When you are comparing your own code with for example **Scikit-Learn**'s\n", + "library, there are some technicalities to keep in mind. The examples\n", + "here demonstrate some of these aspects with potential pitfalls.\n", + "\n", + "The discussion here focuses on the role of the intercept, how we can\n", + "set up the design matrix, what scaling we should use and other topics\n", + "which tend confuse us.\n", + "\n", + "The intercept can be interpreted as the expected value of our\n", + "target/output variables when all other predictors are set to zero.\n", + "Thus, if we cannot assume that the expected outputs/targets are zero\n", + "when all predictors are zero (the columns in the design matrix), it\n", + "may be a bad idea to implement a model which penalizes the intercept.\n", + "Furthermore, in for example Ridge and Lasso regression, the default solutions\n", + "from the library **Scikit-Learn** (when not shrinking $\\beta_0$) for the unknown parameters\n", + "$\\boldsymbol{\\beta}$, are derived under the assumption that both $\\boldsymbol{y}$ and\n", + "$\\boldsymbol{X}$ are zero centered, that is we subtract the mean values.\n", + "\n", + "If our predictors represent different scales, then it is important to\n", + "standardize the design matrix $\\boldsymbol{X}$ by subtracting the mean of each\n", + "column from the corresponding column and dividing the column with its\n", + "standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,\n", + "the results may differ. \n", "\n", "The\n", - "advantage of doing linear regression is that we actually end up with\n", - "analytical expressions for several statistical quantities. \n", - "Standard least squares and Ridge regression allow us to\n", - "derive quantities like the variance and other expectation values in a\n", - "rather straightforward way.\n", + "[Standardscaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html)\n", + "function in **Scikit-Learn** does this for us. For the data sets we\n", + "have been studying in our various examples, the data are in many cases\n", + "already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a\n", + "survey of your data, with a critical assessment of them in case you need to scale the data.\n", "\n", - "It is assumed that $\\varepsilon_i\n", - "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", - "independent, i.e.:" + "If you need to scale the data, not doing so will give an *unfair*\n", + "penalization of the parameters since their magnitude depends on the\n", + "scale of their corresponding predictor.\n", + "\n", + "Suppose as an example that you \n", + "you have an input variable given by the heights of different persons.\n", + "Human height might be measured in inches or meters or\n", + "kilometers. If measured in kilometers, a standard linear regression\n", + "model with this predictor would probably give a much bigger\n", + "coefficient term, than if measured in millimeters.\n", + "This can clearly lead to problems in evaluating the cost/loss functions.\n", + "\n", + "Keep in mind that when you transform your data set before training a model, the same transformation needs to be done\n", + "on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0e71daa5", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "#Model training, we compute the mean value of y and X\n", + "y_train_mean = np.mean(y_train)\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "X_train = X_train - X_train_mean\n", + "y_train = y_train - y_train_mean\n", + "\n", + "# The we fit our model with the training data\n", + "trained_model = some_model.fit(X_train,y_train)\n", + "\n", + "\n", + "#Model prediction, we need also to transform our data set used for the prediction.\n", + "X_test = X_test - X_train_mean #Use mean from training data\n", + "y_pred = trained_model(X_test)\n", + "y_pred = y_pred + y_train_mean\n", + "\"\"\"" ] }, { "cell_type": "markdown", - "id": "e50205be", + "id": "1f0eb0d0", + "metadata": { + "editable": true + }, + "source": [ + "Let us try to understand what this may imply mathematically when we\n", + "subtract the mean values, also known as *zero centering*. For\n", + "simplicity, we will focus on ordinary regression, as done in the above example.\n", + "\n", + "The cost/loss function for regression is" + ] + }, + { + "cell_type": "markdown", + "id": "2ffc4acb", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mbox{Cov}(\\varepsilon_{i_1},\n", - "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", - "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", - "\\end{align*}\n", + "C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70eccda9", + "id": "2511c028", "metadata": { "editable": true }, "source": [ - "The randomness of $\\varepsilon_i$ implies that\n", - "$\\mathbf{y}_i$ is also a random variable. In particular,\n", - "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", - "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", - "non-random scalar. To specify the parameters of the distribution of\n", - "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "Recall also that we use the squared value. This expression can lead to an\n", + "increased penalty for higher differences between predicted and\n", + "output/target values.\n", "\n", - "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", - "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", - "row number $i$ and perform a sum over all values $p$." + "What we have done is to single out the $\\beta_0$ term in the\n", + "definition of the mean squared error (MSE). The design matrix $X$\n", + "does in this case not contain any intercept column. When we take the\n", + "derivative with respect to $\\beta_0$, we want the derivative to obey" ] }, { "cell_type": "markdown", - "id": "e6bdcbf2", - "metadata": { - "editable": true - }, - "source": [ - "## Assumptions made\n", - "\n", - "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", - "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", - "which describe our data" - ] - }, - { - "cell_type": "markdown", - "id": "d74471b8", + "id": "908beb7f", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "\\frac{\\partial C}{\\partial \\beta_j} = 0,\n", "$$" ] }, { "cell_type": "markdown", - "id": "f7b178d9", + "id": "5fae2a94", "metadata": { "editable": true }, "source": [ - "We approximate this function with our model from the solution of the linear regression equations, that is our\n", - "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + "for all $j$. For $\\beta_0$ we have" ] }, { "cell_type": "markdown", - "id": "8ea16c62", + "id": "d7729a27", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n", "$$" ] }, { "cell_type": "markdown", - "id": "4a8d7bfd", + "id": "7fbb2ded", "metadata": { "editable": true }, "source": [ - "## Expectation value and variance\n", - "\n", - "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + "Multiplying away the constant $2/n$, we obtain" ] }, { "cell_type": "markdown", - "id": "5367d943", + "id": "8cef22ce", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \n", - "\\mathbb{E}(y_i) & =\n", - "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", - "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", - "\\end{align*}\n", + "\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c7eefd85", + "id": "c38ab3a0", "metadata": { "editable": true }, "source": [ - "while\n", - "its variance is" + "Let us specialize first to the case where we have only two parameters $\\beta_0$ and $\\beta_1$.\n", + "Our result for $\\beta_0$ simplifies then to" ] }, { "cell_type": "markdown", - "id": "27ce343a", + "id": "c47e7bbe", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", - "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", - "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", - "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", - "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", - "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", - "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", - "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", - "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", - "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", - "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", - "\\end{align*}\n", + "n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n", "$$" ] }, { "cell_type": "markdown", - "id": "c67ea20b", + "id": "06962abd", "metadata": { "editable": true }, "source": [ - "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", - "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + "We obtain then" ] }, { "cell_type": "markdown", - "id": "b4da473a", - "metadata": { - "editable": true - }, - "source": [ - "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", - "\n", - "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" - ] - }, - { - "cell_type": "markdown", - "id": "af812d38", + "id": "14c0a845", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "9d3aaff5", + "id": "5f50f223", "metadata": { "editable": true }, "source": [ - "This means that the estimator of the regression parameters is unbiased.\n", - "\n", - "We can also calculate the variance\n", - "\n", - "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + "If we define" ] }, { "cell_type": "markdown", - "id": "eebea8b7", + "id": "188ed44d", "metadata": { "editable": true }, "source": [ "$$\n", - "\\begin{eqnarray*}\n", - "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", - "\\\\\n", - "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", - "\\\\\n", - "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\\\\n", - "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "% \\\\\n", - "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", - "% \\\\\n", - "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", - "\\\\\n", - "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", - "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", - "\\end{eqnarray*}\n", + "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", "$$" ] }, { "cell_type": "markdown", - "id": "714f281a", + "id": "0b0db7b8", "metadata": { "editable": true }, "source": [ - "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", - "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", - "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", - "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", - "variance of the estimate of the $j$-th regression coefficient:\n", - "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", - "construct a confidence interval for the estimates.\n", - "\n", - "In a similar way, we can obtain analytical expressions for say the\n", - "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", - "when we employ Ridge regression, allowing us again to define a confidence interval. \n", - "\n", - "It is rather straightforward to show that" + "and the mean value of the outputs as" ] }, { "cell_type": "markdown", - "id": "66015ca6", + "id": "e7550136", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", "$$" ] }, { "cell_type": "markdown", - "id": "267f01c0", + "id": "c7cc678d", "metadata": { "editable": true }, "source": [ - "We see clearly that \n", - "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", - "\n", - "We can also compute the variance as" + "we have" ] }, { "cell_type": "markdown", - "id": "998de4b6", + "id": "9c9abcb2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "\\beta_0 = \\mu_y - \\beta_1\\mu_{\\boldsymbol{x}_1}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "06a8d452", + "id": "2336bf55", "metadata": { "editable": true }, "source": [ - "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", - "\n", - "With this, we can compute the difference" + "In the general case with more parameters than $\\beta_0$ and $\\beta_1$, we have" ] }, { "cell_type": "markdown", - "id": "6cd3ed9b", + "id": "e0111655", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n", "$$" ] }, { "cell_type": "markdown", - "id": "70f9aa69", + "id": "709c3af5", "metadata": { "editable": true }, "source": [ - "The difference is non-negative definite since each component of the\n", - "matrix product is non-negative definite. \n", - "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + "We can rewrite the latter equation as" ] }, { "cell_type": "markdown", - "id": "3bb18388", - "metadata": { - "editable": true - }, - "source": [ - "## Deriving OLS from a probability distribution\n", - "\n", - "Our basic assumption when we derived the OLS equations was to assume\n", - "that our output is determined by a given continuous function\n", - "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", - "distribution with zero mean value and an undetermined variance\n", - "$\\sigma^2$.\n", - "\n", - "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", - "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", - "the design matrix are not stochastic variables, we can assume that the\n", - "probability distribution of our targets is also a normal distribution\n", - "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", - "single output $y_i$ is given by the Gaussian distribution" - ] - }, - { - "cell_type": "markdown", - "id": "84b546af", + "id": "a9ed060b", "metadata": { "editable": true }, "source": [ "$$\n", - "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\beta_j,\n", "$$" ] }, { "cell_type": "markdown", - "id": "0e90935e", + "id": "deb61dec", "metadata": { "editable": true }, "source": [ - "## Independent and Identically Distrubuted (iid)\n", - "\n", - "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", - "We define this distribution as" + "where we have defined" ] }, { "cell_type": "markdown", - "id": "b40b33f5", + "id": "dc60f20f", "metadata": { "editable": true }, "source": [ "$$\n", - "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", "$$" ] }, { "cell_type": "markdown", - "id": "23e0c7ac", + "id": "b43a095d", "metadata": { "editable": true }, "source": [ - "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", "\n", - "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" ] }, { "cell_type": "markdown", - "id": "dcd4ed9c", + "id": "050949a2", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n", "$$" ] }, { "cell_type": "markdown", - "id": "5c37adb5", + "id": "7c288c56", "metadata": { "editable": true }, "source": [ - "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", - "in case we have a simple one-dimensional input and output case" + "If we minimize with respect to $\\boldsymbol{\\beta}$ we have then" ] }, { "cell_type": "markdown", - "id": "27e4eea9", + "id": "72d25a93", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", "$$" ] }, { "cell_type": "markdown", - "id": "5dd95110", + "id": "6e4fe379", "metadata": { "editable": true }, "source": [ - "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", - "We can now rewrite the above probability as" + "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", + "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", + "\n", + "For Ridge regression we need to add $\\lambda \\boldsymbol{\\beta}^T\\boldsymbol{\\beta}$ to the cost function and get then" ] }, { "cell_type": "markdown", - "id": "bc7cfda7", + "id": "b0f0e6e7", "metadata": { "editable": true }, "source": [ "$$\n", - "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", "$$" ] }, { "cell_type": "markdown", - "id": "2bf6dbf4", + "id": "6cfb015d", "metadata": { "editable": true }, "source": [ - "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + "What does this mean? And why do we insist on all this? Let us look at some examples.\n", + "\n", + "This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (*code example thanks to Øyvind Sigmundson Schøyen*). Here our scaling of the data is done by subtracting the mean values only.\n", + "Note also that we do not split the data into training and test." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "bfbf5d1d", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "\n", + "np.random.seed(2021)\n", + "\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "def fit_beta(X, y):\n", + " return np.linalg.pinv(X.T @ X) @ X.T @ y\n", + "\n", + "\n", + "true_beta = [2, 0.5, 3.7]\n", + "\n", + "x = np.linspace(0, 1, 11)\n", + "y = np.sum(\n", + " np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0\n", + ") + 0.1 * np.random.normal(size=len(x))\n", + "\n", + "degree = 3\n", + "X = np.zeros((len(x), degree))\n", + "\n", + "# Include the intercept in the design matrix\n", + "for p in range(degree):\n", + " X[:, p] = x ** p\n", + "\n", + "beta = fit_beta(X, y)\n", + "\n", + "# Intercept is included in the design matrix\n", + "skl = LinearRegression(fit_intercept=False).fit(X, y)\n", + "\n", + "print(f\"True beta: {true_beta}\")\n", + "print(f\"Fitted beta: {beta}\")\n", + "print(f\"Sklearn fitted beta: {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with intercept column\")\n", + "print(MSE(y,ypredictOwn))\n", + "print(f\"MSE with intercept column from SKL\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "\n", + "plt.figure()\n", + "plt.scatter(x, y, label=\"Data\")\n", + "plt.plot(x, X @ beta, label=\"Fit\")\n", + "plt.plot(x, skl.predict(X), label=\"Sklearn (fit_intercept=False)\")\n", + "\n", + "\n", + "# Do not include the intercept in the design matrix\n", + "X = np.zeros((len(x), degree - 1))\n", + "\n", + "for p in range(degree - 1):\n", + " X[:, p] = x ** (p + 1)\n", + "\n", + "# Intercept is not included in the design matrix\n", + "skl = LinearRegression(fit_intercept=True).fit(X, y)\n", + "\n", + "# Use centered values for X and y when computing coefficients\n", + "y_offset = np.average(y, axis=0)\n", + "X_offset = np.average(X, axis=0)\n", + "\n", + "beta = fit_beta(X - X_offset, y - y_offset)\n", + "intercept = np.mean(y_offset - X_offset @ beta)\n", + "\n", + "print(f\"Manual intercept: {intercept}\")\n", + "print(f\"Fitted beta (wiothout intercept): {beta}\")\n", + "print(f\"Sklearn intercept: {skl.intercept_}\")\n", + "print(f\"Sklearn fitted beta (without intercept): {skl.coef_}\")\n", + "ypredictOwn = X @ beta\n", + "ypredictSKL = skl.predict(X)\n", + "print(f\"MSE with Manual intercept\")\n", + "print(MSE(y,ypredictOwn+intercept))\n", + "print(f\"MSE with Sklearn intercept\")\n", + "print(MSE(y,ypredictSKL))\n", + "\n", + "plt.plot(x, X @ beta + intercept, \"--\", label=\"Fit (manual intercept)\")\n", + "plt.plot(x, skl.predict(X), \"--\", label=\"Sklearn (fit_intercept=True)\")\n", + "plt.grid()\n", + "plt.legend()\n", + "\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "e8ff9386", + "id": "7dc13bd7", "metadata": { "editable": true }, "source": [ - "## Maximum Likelihood Estimation (MLE)\n", + "The intercept is the value of our output/target variable\n", + "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", "\n", - "In statistics, maximum likelihood estimation (MLE) is a method of\n", - "estimating the parameters of an assumed probability distribution,\n", - "given some observed data. This is achieved by maximizing a likelihood\n", - "function so that, under the assumed statistical model, the observed\n", - "data is the most probable. \n", + "Printing the MSE, we see first that both methods give the same MSE, as\n", + "they should. However, when we move to for example Ridge regression,\n", + "the way we treat the intercept may give a larger or smaller MSE,\n", + "meaning that the MSE can be penalized by the value of the\n", + "intercept. Not including the intercept in the fit, means that the\n", + "regularization term does not include $\\beta_0$. For different values\n", + "of $\\lambda$, this may lead to different MSE values. \n", "\n", - "We will assume here that our events are given by the above Gaussian\n", - "distribution and we will determine the optimal parameters $\\beta$ by\n", - "maximizing the above PDF. However, computing the derivatives of a\n", - "product function is cumbersome and can easily lead to overflow and/or\n", - "underflowproblems, with potentials for loss of numerical precision.\n", - "\n", - "In practice, it is more convenient to maximize the logarithm of the\n", - "PDF because it is a monotonically increasing function of the argument.\n", - "Alternatively, and this will be our option, we will minimize the\n", - "negative of the logarithm since this is a monotonically decreasing\n", - "function.\n", - "\n", - "Note also that maximization/minimization of the logarithm of the PDF\n", - "is equivalent to the maximization/minimization of the function itself." + "To remind the reader, the regularization term, with the intercept in Ridge regression, is given by" ] }, { "cell_type": "markdown", - "id": "cd665e76", - "metadata": { - "editable": true - }, - "source": [ - "## A new Cost Function\n", - "\n", - "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" - ] - }, - { - "cell_type": "markdown", - "id": "144c5c88", + "id": "3ce23f71", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\beta_j^2,\n", "$$" ] }, { "cell_type": "markdown", - "id": "76c7c5c6", + "id": "f34b4a50", "metadata": { "editable": true }, "source": [ - "which becomes" + "but when we take out the intercept, this equation becomes" ] }, { "cell_type": "markdown", - "id": "f78405b0", + "id": "6c535531", "metadata": { "editable": true }, "source": [ "$$\n", - "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\beta_j^2.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dd7b69b5", + "id": "dc9bd9cc", "metadata": { "editable": true }, "source": [ - "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + "For Lasso regression we have" ] }, { "cell_type": "markdown", - "id": "a1430a20", + "id": "b5911fe2", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\beta_j\\vert.\n", "$$" ] }, { "cell_type": "markdown", - "id": "dfcd0c7b", + "id": "4d4103f1", "metadata": { "editable": true }, "source": [ - "which leads to the well-known OLS equation for the optimal paramters $\\beta$" - ] - }, - { - "cell_type": "markdown", - "id": "043d37a6", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "28fe8edc", - "metadata": { - "editable": true - }, - "source": [ - "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." - ] - }, - { - "cell_type": "markdown", - "id": "b600d21c", - "metadata": { - "editable": true - }, - "source": [ - "## More basic Statistics and Bayes' theorem\n", + "It means that, when scaling the design matrix and the outputs/targets,\n", + "by subtracting the mean values, we have an optimization problem which\n", + "is not penalized by the intercept. The MSE value can then be smaller\n", + "since it focuses only on the remaining quantities. If we however bring\n", + "back the intercept, we will get a MSE which then contains the\n", + "intercept.\n", "\n", - "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", - "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "Armed with this wisdom, we attempt first to simply set the intercept equal to **False** in our implementation of Ridge regression for our well-known vanilla data set." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "eecec942", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", "\n", - "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", "\n", - "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", - "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", "\n", - "**Union of events is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "d513cf2c", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "d18008b5", - "metadata": { - "editable": true - }, - "source": [ - "**The product rule (aka joint probability) is given by.**" - ] - }, - { - "cell_type": "markdown", - "id": "22f55ec4", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "29e1bff6", - "metadata": { - "editable": true - }, - "source": [ - "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", "\n", - "If we have independent events then $p(X,Y)=p(X)p(Y)$." - ] - }, - { - "cell_type": "markdown", - "id": "333869a5", - "metadata": { - "editable": true - }, - "source": [ - "## Marginal Probability\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" - ] - }, - { - "cell_type": "markdown", - "id": "bf8de993", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1c5349d1", - "metadata": { - "editable": true - }, - "source": [ - "## Conditional Probability\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree))\n", + "#We include explicitely the intercept column\n", + "for degree in range(Maxpolydegree):\n", + " X[:,degree] = x**degree\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "The conditional probability, if $p(Y) > 0$, is" - ] - }, - { - "cell_type": "markdown", - "id": "dbede038", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "c1334a9c", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem\n", + "p = Maxpolydegree\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # Note: we include the intercept column and no scaling\n", + " RegRidge = linear_model.Ridge(lmb,fit_intercept=False)\n", + " RegRidge.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeOwnRidge = X_train @ OwnRidgeBeta\n", + " ypredictOwnRidge = X_test @ OwnRidgeBeta\n", + " ytildeRidge = RegRidge.predict(X_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta)\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", "\n", - "If we combine the conditional probability with the marginal probability and the standard product rule, we have" - ] - }, - { - "cell_type": "markdown", - "id": "ac8dcf04", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "1992e354", - "metadata": { - "editable": true - }, - "source": [ - "which we can rewrite as" - ] - }, - { - "cell_type": "markdown", - "id": "24e5179f", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "42979e5a", - "metadata": { - "editable": true - }, - "source": [ - "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." - ] - }, - { - "cell_type": "markdown", - "id": "86a8cd07", - "metadata": { - "editable": true - }, - "source": [ - "## Interpretations of Bayes' Theorem\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'r', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g', label = 'MSE Ridge Test')\n", "\n", - "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", - "evaluated for the observed data $Y$ and can be viewed as a function of\n", - "the parameter space represented by $X$. This function is not\n", - "necesseraly normalized and is normally called the likelihood function.\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7d67706b", + "metadata": { + "editable": true + }, + "source": [ + "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", + "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", + "What happens if we do not include the intercept in our fit?\n", + "Let us see how we can change this code by zero centering." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "598d20bd", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "from sklearn.preprocessing import StandardScaler\n", "\n", - "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(315)\n", "\n", - "Let us try to illustrate Bayes' theorem through an example." - ] - }, - { - "cell_type": "markdown", - "id": "b9a4f9a9", - "metadata": { - "editable": true - }, - "source": [ - "## Example of Usage of Bayes' theorem\n", + "n = 100\n", + "x = np.random.rand(n)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)\n", "\n", - "Let us suppose that you are undergoing a series of mammography scans in\n", - "order to rule out possible breast cancer cases. We define the\n", - "sensitivity for a positive event by the variable $X$. It takes binary\n", - "values with $X=1$ representing a positive event and $X=0$ being a\n", - "negative event. We reserve $Y$ as a classification parameter for\n", - "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "Maxpolydegree = 20\n", + "X = np.zeros((n,Maxpolydegree-1))\n", "\n", - "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "for degree in range(1,Maxpolydegree): #No intercept column\n", + " X[:,degree-1] = x**(degree)\n", "\n", - "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" - ] - }, - { - "cell_type": "markdown", - "id": "888c64ec", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=1) =0.8.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "5086b046", - "metadata": { - "editable": true - }, - "source": [ - "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", - "It is however not correct, as the following Bayesian analysis shows." - ] - }, - { - "cell_type": "markdown", - "id": "609ea2b0", - "metadata": { - "editable": true - }, - "source": [ - "## Doing it correctly\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", "\n", - "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", - "Let us assume that the prior probability in the population as a whole is" + "#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable\n", + "X_train_mean = np.mean(X_train,axis=0)\n", + "#Center by removing mean from each feature\n", + "X_train_scaled = X_train - X_train_mean \n", + "X_test_scaled = X_test - X_train_mean\n", + "#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)\n", + "#Remove the intercept from the training data.\n", + "y_scaler = np.mean(y_train) \n", + "y_train_scaled = y_train - y_scaler \n", + "\n", + "p = Maxpolydegree-1\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 6\n", + "MSEOwnRidgePredict = np.zeros(nlambdas)\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "\n", + "lambdas = np.logspace(-4, 2, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)\n", + " intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data\n", + " #Add intercept to prediction\n", + " ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler \n", + " RegRidge = linear_model.Ridge(lmb)\n", + " RegRidge.fit(X_train,y_train)\n", + " ypredictRidge = RegRidge.predict(X_test)\n", + " MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)\n", + " MSERidgePredict[i] = MSE(y_test,ypredictRidge)\n", + " print(\"Beta values for own Ridge implementation\")\n", + " print(OwnRidgeBeta) #Intercept is given by mean of target variable\n", + " print(\"Beta values for Scikit-Learn Ridge implementation\")\n", + " print(RegRidge.coef_)\n", + " print('Intercept from own implementation:')\n", + " print(intercept_)\n", + " print('Intercept from Scikit-Learn Ridge implementation')\n", + " print(RegRidge.intercept_)\n", + " print(\"MSE values for own Ridge implementation\")\n", + " print(MSEOwnRidgePredict[i])\n", + " print(\"MSE values for Scikit-Learn Ridge implementation\")\n", + " print(MSERidgePredict[i])\n", + "\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" ] }, { "cell_type": "markdown", - "id": "687bfdf3", + "id": "cec80003", "metadata": { "editable": true }, "source": [ - "$$\n", - "p(Y=1) =0.004.\n", - "$$" + "We see here, when compared to the code which includes explicitely the\n", + "intercept column, that our MSE value is actually smaller. This is\n", + "because the regularization term does not include the intercept value\n", + "$\\beta_0$ in the fitting. This applies to Lasso regularization as\n", + "well. It means that our optimization is now done only with the\n", + "centered matrix and/or vector that enter the fitting procedure." ] }, { "cell_type": "markdown", - "id": "c2af1d2d", + "id": "16cdcffb", "metadata": { "editable": true }, "source": [ - "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" - ] - }, - { - "cell_type": "markdown", - "id": "b8ec8d93", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(X=1\\vert Y=0) =0.1.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "48a6fb5b", - "metadata": { - "editable": true - }, - "source": [ - "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" - ] - }, - { - "cell_type": "markdown", - "id": "ba37d4fd", - "metadata": { - "editable": true - }, - "source": [ - "$$\n", - "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", - "$$" - ] - }, - { - "cell_type": "markdown", - "id": "824f1789", - "metadata": { - "editable": true - }, - "source": [ - "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" - ] - }, - { - "cell_type": "markdown", - "id": "07f89fd5", - "metadata": { - "editable": true - }, - "source": [ - "## Bayes' Theorem and Ridge and Lasso Regression\n", + "## Test Function for what happens with OLS, Ridge and Lasso\n", "\n", "Hitherto we have discussed Ridge and Lasso regression in terms of a\n", "linear analysis. This may to many of you feel rather technical and\n", "perhaps not that intuitive. The question is whether we can develop a\n", "more intuitive way of understanding what Ridge and Lasso express.\n", "\n", - "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit." - ] - }, - { - "cell_type": "markdown", - "id": "8fcd73d9", - "metadata": { - "editable": true - }, - "source": [ - "## Test Function for what happens with OLS, Ridge and Lasso\n", + "Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. \n", "\n", "We will play around with a study of the values for the optimal\n", "parameters $\\boldsymbol{\\beta}$ using OLS, Ridge and Lasso regression. For\n", @@ -3015,8 +2909,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "id": "4402e113", + "execution_count": 11, + "id": "e04dce1d", "metadata": { "collapsed": false, "editable": true @@ -3043,10 +2937,9 @@ "X = np.zeros((len(x),Maxpolydegree))\n", "X[:,0] = 1.0\n", "\n", - "for polydegree in range(1, Maxpolydegree):\n", - " for degree in range(polydegree):\n", - " X[:,degree] = x**(degree)\n", "\n", + "for polydegree in range(1,Maxpolydegree):\n", + " X[:,polydegree] = x**(polydegree)\n", "\n", "# We split the data in test and training data\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", @@ -3090,7 +2983,7 @@ }, { "cell_type": "markdown", - "id": "b700e45e", + "id": "2ae202c3", "metadata": { "editable": true }, @@ -3100,43 +2993,475 @@ }, { "cell_type": "markdown", - "id": "69659d9d", + "id": "4186f9f0", "metadata": { "editable": true }, "source": [ - "## Invoking Bayes' theorem\n", + "## Linking the regression analysis with a statistical interpretation\n", "\n", - "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "We will now couple the discussions of ordinary least squares, Ridge\n", + "and Lasso regression with a statistical interpretation, that is we\n", + "move from a linear algebra analysis to a statistical analysis. In\n", + "particular, we will focus on what the regularization terms can result\n", + "in. We will amongst other things show that the regularization\n", + "parameter can reduce considerably the variance of the parameters\n", + "$\\beta$.\n", "\n", - "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + "The\n", + "advantage of doing linear regression is that we actually end up with\n", + "analytical expressions for several statistical quantities. \n", + "Standard least squares and Ridge regression allow us to\n", + "derive quantities like the variance and other expectation values in a\n", + "rather straightforward way.\n", + "\n", + "It is assumed that $\\varepsilon_i\n", + "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", + "independent, i.e.:" ] }, { "cell_type": "markdown", - "id": "80b423eb", + "id": "7293d2ab", "metadata": { "editable": true }, "source": [ "$$\n", - "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "\\begin{align*} \n", + "\\mbox{Cov}(\\varepsilon_{i_1},\n", + "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", + "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", + "\\end{align*}\n", "$$" ] }, { "cell_type": "markdown", - "id": "de547949", + "id": "efee3703", "metadata": { "editable": true }, "source": [ - "is given by" + "The randomness of $\\varepsilon_i$ implies that\n", + "$\\mathbf{y}_i$ is also a random variable. In particular,\n", + "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", + "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", + "non-random scalar. To specify the parameters of the distribution of\n", + "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "\n", + "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", + "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", + "row number $i$ and perform a sum over all values $p$." ] }, { "cell_type": "markdown", - "id": "99d9476a", + "id": "85d2303d", + "metadata": { + "editable": true + }, + "source": [ + "## Assumptions made\n", + "\n", + "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", + "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", + "which describe our data" + ] + }, + { + "cell_type": "markdown", + "id": "c1a189c0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "0b63e61f", + "metadata": { + "editable": true + }, + "source": [ + "We approximate this function with our model from the solution of the linear regression equations, that is our\n", + "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + ] + }, + { + "cell_type": "markdown", + "id": "14acfe14", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "681408c6", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance\n", + "\n", + "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + ] + }, + { + "cell_type": "markdown", + "id": "d956e33c", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mathbb{E}(y_i) & =\n", + "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", + "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "992bee5b", + "metadata": { + "editable": true + }, + "source": [ + "while\n", + "its variance is" + ] + }, + { + "cell_type": "markdown", + "id": "fd75d1af", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", + "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", + "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", + "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", + "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", + "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", + "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", + "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", + "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", + "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", + "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f702411d", + "metadata": { + "editable": true + }, + "source": [ + "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", + "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." + ] + }, + { + "cell_type": "markdown", + "id": "b86e729d", + "metadata": { + "editable": true + }, + "source": [ + "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", + "\n", + "With the OLS expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}}$ we can evaluate the expectation value" + ] + }, + { + "cell_type": "markdown", + "id": "5af888a2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3d21a65e", + "metadata": { + "editable": true + }, + "source": [ + "This means that the estimator of the regression parameters is unbiased.\n", + "\n", + "We can also calculate the variance\n", + "\n", + "The variance of the optimal value $\\boldsymbol{\\hat{\\beta}}$ is" + ] + }, + { + "cell_type": "markdown", + "id": "7d8043ab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\begin{eqnarray*}\n", + "\\mbox{Var}(\\boldsymbol{\\hat{\\beta}}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", + "\\\\\n", + "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", + "\\\\\n", + "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", + "% \\\\\n", + "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", + "\\\\\n", + "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", + "\\end{eqnarray*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4a126ec7", + "metadata": { + "editable": true + }, + "source": [ + "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", + "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", + "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", + "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", + "variance of the estimate of the $j$-th regression coefficient:\n", + "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 [(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} $. This may be used to\n", + "construct a confidence interval for the estimates.\n", + "\n", + "In a similar way, we can obtain analytical expressions for say the\n", + "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", + "when we employ Ridge regression, allowing us again to define a confidence interval. \n", + "\n", + "It is rather straightforward to show that" + ] + }, + { + "cell_type": "markdown", + "id": "7287992a", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "2803f793", + "metadata": { + "editable": true + }, + "source": [ + "We see clearly that \n", + "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", + "\n", + "We can also compute the variance as" + ] + }, + { + "cell_type": "markdown", + "id": "d7443c20", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a0d32eb0", + "metadata": { + "editable": true + }, + "source": [ + "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", + "\n", + "With this, we can compute the difference" + ] + }, + { + "cell_type": "markdown", + "id": "cbd0143e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e6de88a4", + "metadata": { + "editable": true + }, + "source": [ + "The difference is non-negative definite since each component of the\n", + "matrix product is non-negative definite. \n", + "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below." + ] + }, + { + "cell_type": "markdown", + "id": "55630c9f", + "metadata": { + "editable": true + }, + "source": [ + "## Deriving OLS from a probability distribution\n", + "\n", + "Our basic assumption when we derived the OLS equations was to assume\n", + "that our output is determined by a given continuous function\n", + "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", + "distribution with zero mean value and an undetermined variance\n", + "$\\sigma^2$.\n", + "\n", + "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", + "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", + "the design matrix are not stochastic variables, we can assume that the\n", + "probability distribution of our targets is also a normal distribution\n", + "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", + "single output $y_i$ is given by the Gaussian distribution" + ] + }, + { + "cell_type": "markdown", + "id": "745e6352", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4ec8c116", + "metadata": { + "editable": true + }, + "source": [ + "## Independent and Identically Distrubuted (iid)\n", + "\n", + "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", + "We define this distribution as" + ] + }, + { + "cell_type": "markdown", + "id": "f92be14e", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "edbdd556", + "metadata": { + "editable": true + }, + "source": [ + "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "\n", + "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "6a6903ca", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "57680896", + "metadata": { + "editable": true + }, + "source": [ + "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", + "in case we have a simple one-dimensional input and output case" + ] + }, + { + "cell_type": "markdown", + "id": "9309640f", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "67fb3df9", + "metadata": { + "editable": true + }, + "source": [ + "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", + "We can now rewrite the above probability as" + ] + }, + { + "cell_type": "markdown", + "id": "894f0fb7", "metadata": { "editable": true }, @@ -3148,7 +3473,508 @@ }, { "cell_type": "markdown", - "id": "aa5613ea", + "id": "13f3f705", + "metadata": { + "editable": true + }, + "source": [ + "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." + ] + }, + { + "cell_type": "markdown", + "id": "6bf3edfa", + "metadata": { + "editable": true + }, + "source": [ + "## Maximum Likelihood Estimation (MLE)\n", + "\n", + "In statistics, maximum likelihood estimation (MLE) is a method of\n", + "estimating the parameters of an assumed probability distribution,\n", + "given some observed data. This is achieved by maximizing a likelihood\n", + "function so that, under the assumed statistical model, the observed\n", + "data is the most probable. \n", + "\n", + "We will assume here that our events are given by the above Gaussian\n", + "distribution and we will determine the optimal parameters $\\beta$ by\n", + "maximizing the above PDF. However, computing the derivatives of a\n", + "product function is cumbersome and can easily lead to overflow and/or\n", + "underflowproblems, with potentials for loss of numerical precision.\n", + "\n", + "In practice, it is more convenient to maximize the logarithm of the\n", + "PDF because it is a monotonically increasing function of the argument.\n", + "Alternatively, and this will be our option, we will minimize the\n", + "negative of the logarithm since this is a monotonically decreasing\n", + "function.\n", + "\n", + "Note also that maximization/minimization of the logarithm of the PDF\n", + "is equivalent to the maximization/minimization of the function itself." + ] + }, + { + "cell_type": "markdown", + "id": "be42ac0c", + "metadata": { + "editable": true + }, + "source": [ + "## A new Cost Function\n", + "\n", + "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" + ] + }, + { + "cell_type": "markdown", + "id": "b6fbab08", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "33ef1983", + "metadata": { + "editable": true + }, + "source": [ + "which becomes" + ] + }, + { + "cell_type": "markdown", + "id": "7d6504da", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "03d8775c", + "metadata": { + "editable": true + }, + "source": [ + "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + ] + }, + { + "cell_type": "markdown", + "id": "176d1dab", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "e20e33bf", + "metadata": { + "editable": true + }, + "source": [ + "which leads to the well-known OLS equation for the optimal paramters $\\beta$" + ] + }, + { + "cell_type": "markdown", + "id": "94f73eae", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f5d2cf20", + "metadata": { + "editable": true + }, + "source": [ + "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." + ] + }, + { + "cell_type": "markdown", + "id": "c32b8f92", + "metadata": { + "editable": true + }, + "source": [ + "## More basic Statistics and Bayes' theorem\n", + "\n", + "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", + "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "\n", + "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "\n", + "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", + "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", + "\n", + "**Union of events is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "a3779fd2", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "37e13b18", + "metadata": { + "editable": true + }, + "source": [ + "**The product rule (aka joint probability) is given by.**" + ] + }, + { + "cell_type": "markdown", + "id": "bd8b41f0", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "71b22e26", + "metadata": { + "editable": true + }, + "source": [ + "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "\n", + "If we have independent events then $p(X,Y)=p(X)p(Y)$." + ] + }, + { + "cell_type": "markdown", + "id": "5f6645d2", + "metadata": { + "editable": true + }, + "source": [ + "## Marginal Probability\n", + "\n", + "The marginal probability is defined in terms of only one of the set of variables $X,Y$. For a discrete probability we have" + ] + }, + { + "cell_type": "markdown", + "id": "7349fe65", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "a6205b4f", + "metadata": { + "editable": true + }, + "source": [ + "## Conditional Probability\n", + "\n", + "The conditional probability, if $p(Y) > 0$, is" + ] + }, + { + "cell_type": "markdown", + "id": "43cb130d", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "9d8f1813", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem\n", + "\n", + "If we combine the conditional probability with the marginal probability and the standard product rule, we have" + ] + }, + { + "cell_type": "markdown", + "id": "d2746770", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f3659d1d", + "metadata": { + "editable": true + }, + "source": [ + "which we can rewrite as" + ] + }, + { + "cell_type": "markdown", + "id": "8dfd9ec7", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "3f247ad1", + "metadata": { + "editable": true + }, + "source": [ + "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." + ] + }, + { + "cell_type": "markdown", + "id": "a2615306", + "metadata": { + "editable": true + }, + "source": [ + "## Interpretations of Bayes' Theorem\n", + "\n", + "The quantity $p(Y\\vert X)$ on the right-hand side of the theorem is\n", + "evaluated for the observed data $Y$ and can be viewed as a function of\n", + "the parameter space represented by $X$. This function is not\n", + "necesseraly normalized and is normally called the likelihood function.\n", + "\n", + "The function $p(X)$ on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution.\n", + "\n", + "Let us try to illustrate Bayes' theorem through an example." + ] + }, + { + "cell_type": "markdown", + "id": "91eae9e8", + "metadata": { + "editable": true + }, + "source": [ + "## Example of Usage of Bayes' theorem\n", + "\n", + "Let us suppose that you are undergoing a series of mammography scans in\n", + "order to rule out possible breast cancer cases. We define the\n", + "sensitivity for a positive event by the variable $X$. It takes binary\n", + "values with $X=1$ representing a positive event and $X=0$ being a\n", + "negative event. We reserve $Y$ as a classification parameter for\n", + "either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing).\n", + "\n", + "We let $Y=1$ represent the the case of having breast cancer and $Y=0$ as not.\n", + "\n", + "Let us assume that if you have breast cancer, the test will be positive with a probability of $0.8$, that is we have" + ] + }, + { + "cell_type": "markdown", + "id": "76fe1c46", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=1) =0.8.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "397d9768", + "metadata": { + "editable": true + }, + "source": [ + "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", + "It is however not correct, as the following Bayesian analysis shows." + ] + }, + { + "cell_type": "markdown", + "id": "f2c701f5", + "metadata": { + "editable": true + }, + "source": [ + "## Doing it correctly\n", + "\n", + "If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number.\n", + "Let us assume that the prior probability in the population as a whole is" + ] + }, + { + "cell_type": "markdown", + "id": "0fdb2639", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1) =0.004.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "f9ae9f95", + "metadata": { + "editable": true + }, + "source": [ + "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" + ] + }, + { + "cell_type": "markdown", + "id": "a681f1dc", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(X=1\\vert Y=0) =0.1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "4d2c6030", + "metadata": { + "editable": true + }, + "source": [ + "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" + ] + }, + { + "cell_type": "markdown", + "id": "67a1c582", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "969f368b", + "metadata": { + "editable": true + }, + "source": [ + "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" + ] + }, + { + "cell_type": "markdown", + "id": "356a70c0", + "metadata": { + "editable": true + }, + "source": [ + "## Bayes' Theorem and Ridge and Lasso Regression\n", + "\n", + "Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. \n", + "\n", + "For ordinary least squares we postulated that the maximum likelihood for the doamin of events $\\boldsymbol{D}$ (one-dimensional case)" + ] + }, + { + "cell_type": "markdown", + "id": "fa653f05", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "d5dd1081", + "metadata": { + "editable": true + }, + "source": [ + "is given by" + ] + }, + { + "cell_type": "markdown", + "id": "731b21d4", + "metadata": { + "editable": true + }, + "source": [ + "$$\n", + "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "362ae4e2", "metadata": { "editable": true }, @@ -3158,7 +3984,7 @@ }, { "cell_type": "markdown", - "id": "7dea4e11", + "id": "8ba56ac5", "metadata": { "editable": true }, @@ -3170,7 +3996,7 @@ }, { "cell_type": "markdown", - "id": "99ab1d57", + "id": "658e6364", "metadata": { "editable": true }, @@ -3180,7 +4006,7 @@ }, { "cell_type": "markdown", - "id": "170ee68b", + "id": "28d6553d", "metadata": { "editable": true }, @@ -3192,7 +4018,7 @@ }, { "cell_type": "markdown", - "id": "3388d6a4", + "id": "8f4ec5f0", "metadata": { "editable": true }, @@ -3202,7 +4028,7 @@ }, { "cell_type": "markdown", - "id": "a5bdaa16", + "id": "e72b34ca", "metadata": { "editable": true }, @@ -3218,7 +4044,7 @@ }, { "cell_type": "markdown", - "id": "cbf8e7e6", + "id": "0e656652", "metadata": { "editable": true }, @@ -3230,7 +4056,7 @@ }, { "cell_type": "markdown", - "id": "52c9e48f", + "id": "57606d08", "metadata": { "editable": true }, @@ -3240,7 +4066,7 @@ }, { "cell_type": "markdown", - "id": "1ae37a89", + "id": "cf40e3a9", "metadata": { "editable": true }, @@ -3252,7 +4078,7 @@ }, { "cell_type": "markdown", - "id": "dd8fee27", + "id": "65038f20", "metadata": { "editable": true }, @@ -3265,7 +4091,7 @@ }, { "cell_type": "markdown", - "id": "de603b44", + "id": "deb9890c", "metadata": { "editable": true }, @@ -3277,7 +4103,7 @@ }, { "cell_type": "markdown", - "id": "3f08e4a0", + "id": "1a844813", "metadata": { "editable": true }, @@ -3287,7 +4113,7 @@ }, { "cell_type": "markdown", - "id": "b65ee2f0", + "id": "a32e4148", "metadata": { "editable": true }, @@ -3299,7 +4125,7 @@ }, { "cell_type": "markdown", - "id": "6526b426", + "id": "4050a8ed", "metadata": { "editable": true }, @@ -3309,7 +4135,7 @@ }, { "cell_type": "markdown", - "id": "29190fe5", + "id": "ec31e4e2", "metadata": { "editable": true }, @@ -3321,7 +4147,7 @@ }, { "cell_type": "markdown", - "id": "71b1585c", + "id": "f872c397", "metadata": { "editable": true }, @@ -3333,7 +4159,7 @@ }, { "cell_type": "markdown", - "id": "e18baaa9", + "id": "e8eeb2e5", "metadata": { "editable": true }, @@ -3343,7 +4169,7 @@ }, { "cell_type": "markdown", - "id": "1c58b8db", + "id": "54d2f39d", "metadata": { "editable": true }, @@ -3355,7 +4181,7 @@ }, { "cell_type": "markdown", - "id": "238ee8c5", + "id": "54cbc3a8", "metadata": { "editable": true }, @@ -3367,7 +4193,7 @@ }, { "cell_type": "markdown", - "id": "46389b82", + "id": "7ee728b6", "metadata": { "editable": true }, @@ -3379,7 +4205,7 @@ }, { "cell_type": "markdown", - "id": "0ecbb2d2", + "id": "00e9a0eb", "metadata": { "editable": true }, @@ -3389,7 +4215,7 @@ }, { "cell_type": "markdown", - "id": "27368dfb", + "id": "50f72b22", "metadata": { "editable": true }, @@ -3401,7 +4227,7 @@ }, { "cell_type": "markdown", - "id": "27082f48", + "id": "d53fea2e", "metadata": { "editable": true }, diff --git a/doc/src/week37/exercisesweek37.do.txt b/doc/src/week37/exercisesweek37.do.txt index f6c803e74..556182276 100644 --- a/doc/src/week37/exercisesweek37.do.txt +++ b/doc/src/week37/exercisesweek37.do.txt @@ -6,14 +6,24 @@ DATE: Deadline is Sunday September 17 at midnight ===== Overarching aims of the exercises this week ===== -This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of "Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer":"https://www.springer.com/gp/book/9780387848570"). The exercise is also a part of project 1 and can be reused in the theory part of the project. +This exercise deals with various mean values and variances in linear +regression method (here it may be useful to look up chapter 3, +equation (3.8) of "Trevor Hastie, Robert Tibshirani, Jerome +H. Friedman, The Elements of Statistical Learning, +Springer":"https://www.springer.com/gp/book/9780387848570"). The +exercise is also a part of project 1 and can be reused in the theory +part of the project. -For more discussions on Ridge regression and calculation of expectation values, "Wessel van Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly recommended. +For more discussions on Ridge regression and calculation of +expectation values, "Wessel van +Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly +recommended. -The assumption we have made is -that there exists a continuous function $f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim N(0, \sigma^2)$ -which describes our data +The assumption we have made is that there exists a continuous function +$f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim N(0, +\sigma^2)$ which describes our data + !bt \[ \bm{y} = f(\bm{x})+\bm{\varepsilon} @@ -70,7 +80,7 @@ A given parameter $\beta_j$ is given by the diagonal matrix element of the above Show that !bt \[ -\mathbb{E} \big[ \bm{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\ +\mathbb{E} \big[ \hat{\bm{\beta}}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\ \bm{\beta}^{\mathrm{OLS}}. \] !et @@ -82,7 +92,7 @@ Show also that the variance is !bt \[ -\mbox{Var}[\bm{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +\mbox{Var}[\hat{\bm{\beta}}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, \] !et and it is easy to see that if the parameter $\lambda$ goes to infinity then the variance of Ridge parameters $\bm{\beta}$ goes to zero.