From 33123d4bf2d398919a7b416ac2d643cc5496f2ad Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Tue, 19 Sep 2023 14:46:00 +0200 Subject: [PATCH] updating typos --- doc/LectureNotes/exercisesweek38.ipynb | 30 +- .../2023/Project1/html/._Project1-bs000.html | 4 +- .../2023/Project1/html/Project1-bs.html | 4 +- doc/Projects/2023/Project1/html/Project1.html | 4 +- .../2023/Project1/ipynb/Project1.ipynb | 104 +- .../Project1/ipynb/ipynb-Project1-src.tar.gz | Bin 193 -> 193 bytes doc/Projects/2023/Project1/pdf/Project1.p.tex | 4 +- doc/Projects/2023/Project1/pdf/Project1.pdf | Bin 270659 -> 270659 bytes doc/Projects/2023/Project1/pdf/Project1.tex | 4 +- doc/pub/week35/ipynb/week35.ipynb | 1787 +++++------------ doc/pub/week38/ipynb/week38.ipynb | 8 +- .../Projects/2023/Project1/Project1.do.txt | 4 +- doc/src/week38/exercisesweek38.do.txt | 4 +- 13 files changed, 549 insertions(+), 1408 deletions(-) diff --git a/doc/LectureNotes/exercisesweek38.ipynb b/doc/LectureNotes/exercisesweek38.ipynb index 9f34b9108..bc0c0a7fb 100644 --- a/doc/LectureNotes/exercisesweek38.ipynb +++ b/doc/LectureNotes/exercisesweek38.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "d8f7704f", + "id": "ce05d309", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "9e388b87", + "id": "5b7c1442", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "80d7df2a", + "id": "02e09b06", "metadata": { "editable": true }, @@ -45,7 +45,7 @@ }, { "cell_type": "markdown", - "id": "80e18e80", + "id": "babb7346", "metadata": { "editable": true }, @@ -57,7 +57,7 @@ }, { "cell_type": "markdown", - "id": "3a74059f", + "id": "a5ee908c", "metadata": { "editable": true }, @@ -76,7 +76,7 @@ }, { "cell_type": "markdown", - "id": "1f92f219", + "id": "afa3df2c", "metadata": { "editable": true }, @@ -88,7 +88,7 @@ }, { "cell_type": "markdown", - "id": "416cd63a", + "id": "9665df80", "metadata": { "editable": true }, @@ -102,19 +102,19 @@ }, { "cell_type": "markdown", - "id": "d65d4905", + "id": "25391aba", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[y]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", "$$" ] }, { "cell_type": "markdown", - "id": "d102dec2", + "id": "b6e8652c", "metadata": { "editable": true }, @@ -124,19 +124,19 @@ }, { "cell_type": "markdown", - "id": "869dc9d9", + "id": "77755270", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathrm{Bias}[y]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", + "\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", "$$" ] }, { "cell_type": "markdown", - "id": "fee09cce", + "id": "fa94db90", "metadata": { "editable": true }, @@ -146,7 +146,7 @@ }, { "cell_type": "markdown", - "id": "b6fb6522", + "id": "4ecda624", "metadata": { "editable": true }, @@ -158,7 +158,7 @@ }, { "cell_type": "markdown", - "id": "6fa010b9", + "id": "2dcb3e9a", "metadata": { "editable": true }, diff --git a/doc/Projects/2023/Project1/html/._Project1-bs000.html b/doc/Projects/2023/Project1/html/._Project1-bs000.html index 905d9da71..927bc0413 100644 --- a/doc/Projects/2023/Project1/html/._Project1-bs000.html +++ b/doc/Projects/2023/Project1/html/._Project1-bs000.html @@ -489,12 +489,12 @@ term which measures the deviation from the true data and the mean value of the m That is, show that

$$ -\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2, +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, $$

with

$$ -\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], $$

and

diff --git a/doc/Projects/2023/Project1/html/Project1-bs.html b/doc/Projects/2023/Project1/html/Project1-bs.html index 905d9da71..927bc0413 100644 --- a/doc/Projects/2023/Project1/html/Project1-bs.html +++ b/doc/Projects/2023/Project1/html/Project1-bs.html @@ -489,12 +489,12 @@ term which measures the deviation from the true data and the mean value of the m That is, show that

$$ -\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2, +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, $$

with

$$ -\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], $$

and

diff --git a/doc/Projects/2023/Project1/html/Project1.html b/doc/Projects/2023/Project1/html/Project1.html index 9e087eae6..d2226502a 100644 --- a/doc/Projects/2023/Project1/html/Project1.html +++ b/doc/Projects/2023/Project1/html/Project1.html @@ -525,12 +525,12 @@ term which measures the deviation from the true data and the mean value of the m That is, show that

$$ -\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2, +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, $$

with

$$ -\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right], $$

and

diff --git a/doc/Projects/2023/Project1/ipynb/Project1.ipynb b/doc/Projects/2023/Project1/ipynb/Project1.ipynb index e28db853b..2a2b5d80e 100644 --- a/doc/Projects/2023/Project1/ipynb/Project1.ipynb +++ b/doc/Projects/2023/Project1/ipynb/Project1.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "ee871b85", + "id": "f8900c95", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "de647a13", + "id": "e1595afd", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "f0094ea8", + "id": "8d3e72b9", "metadata": { "editable": true }, @@ -63,7 +63,7 @@ }, { "cell_type": "markdown", - "id": "9d47e07f", + "id": "8293cb06", "metadata": { "editable": true }, @@ -85,7 +85,7 @@ }, { "cell_type": "markdown", - "id": "13b3c63c", + "id": "e49db9ae", "metadata": { "editable": true }, @@ -100,7 +100,7 @@ }, { "cell_type": "markdown", - "id": "588eb391", + "id": "7108aee4", "metadata": { "editable": true }, @@ -133,7 +133,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "2b565e88", + "id": "2f54fcda", "metadata": { "collapsed": false, "editable": true @@ -185,7 +185,7 @@ }, { "cell_type": "markdown", - "id": "5a80d8fa", + "id": "91ce0ca4", "metadata": { "editable": true }, @@ -207,7 +207,7 @@ }, { "cell_type": "markdown", - "id": "2b8a75d3", + "id": "3bf84fef", "metadata": { "editable": true }, @@ -220,7 +220,7 @@ }, { "cell_type": "markdown", - "id": "de8e68ca", + "id": "568afb54", "metadata": { "editable": true }, @@ -232,7 +232,7 @@ }, { "cell_type": "markdown", - "id": "70a2795e", + "id": "a44eb2fc", "metadata": { "editable": true }, @@ -244,7 +244,7 @@ }, { "cell_type": "markdown", - "id": "07d74c94", + "id": "5603e809", "metadata": { "editable": true }, @@ -254,7 +254,7 @@ }, { "cell_type": "markdown", - "id": "cb7a1688", + "id": "7d6f55ea", "metadata": { "editable": true }, @@ -266,7 +266,7 @@ }, { "cell_type": "markdown", - "id": "0c273378", + "id": "5407a481", "metadata": { "editable": true }, @@ -295,7 +295,7 @@ }, { "cell_type": "markdown", - "id": "b6aa07e9", + "id": "aa8fbc15", "metadata": { "editable": true }, @@ -313,7 +313,7 @@ }, { "cell_type": "markdown", - "id": "4c297a7f", + "id": "5d805d8c", "metadata": { "editable": true }, @@ -330,7 +330,7 @@ }, { "cell_type": "markdown", - "id": "9b65abbd", + "id": "2dde585d", "metadata": { "editable": true }, @@ -346,7 +346,7 @@ }, { "cell_type": "markdown", - "id": "9f2c97bc", + "id": "ed0dd5d4", "metadata": { "editable": true }, @@ -358,7 +358,7 @@ }, { "cell_type": "markdown", - "id": "1390caea", + "id": "e0429f21", "metadata": { "editable": true }, @@ -369,7 +369,7 @@ }, { "cell_type": "markdown", - "id": "757a87d4", + "id": "3daec21b", "metadata": { "editable": true }, @@ -381,7 +381,7 @@ }, { "cell_type": "markdown", - "id": "cf182a2a", + "id": "2d2524c6", "metadata": { "editable": true }, @@ -393,7 +393,7 @@ }, { "cell_type": "markdown", - "id": "c7f15f3d", + "id": "ff089285", "metadata": { "editable": true }, @@ -405,7 +405,7 @@ }, { "cell_type": "markdown", - "id": "70c8c4e9", + "id": "3d168dd6", "metadata": { "editable": true }, @@ -416,7 +416,7 @@ }, { "cell_type": "markdown", - "id": "3879a5b4", + "id": "f8be4ad2", "metadata": { "editable": true }, @@ -428,7 +428,7 @@ }, { "cell_type": "markdown", - "id": "17191387", + "id": "ec6b847d", "metadata": { "editable": true }, @@ -441,7 +441,7 @@ }, { "cell_type": "markdown", - "id": "a7e3a698", + "id": "1a9533ed", "metadata": { "editable": true }, @@ -453,7 +453,7 @@ }, { "cell_type": "markdown", - "id": "93fc4a12", + "id": "b9da1087", "metadata": { "editable": true }, @@ -463,7 +463,7 @@ }, { "cell_type": "markdown", - "id": "85d336ad", + "id": "4f84b574", "metadata": { "editable": true }, @@ -475,7 +475,7 @@ }, { "cell_type": "markdown", - "id": "869ed780", + "id": "3077c7a2", "metadata": { "editable": true }, @@ -486,7 +486,7 @@ }, { "cell_type": "markdown", - "id": "88502fe8", + "id": "f82bcde0", "metadata": { "editable": true }, @@ -519,7 +519,7 @@ }, { "cell_type": "markdown", - "id": "db4583da", + "id": "3ff30b63", "metadata": { "editable": true }, @@ -531,7 +531,7 @@ }, { "cell_type": "markdown", - "id": "da3a0dfc", + "id": "61c75d7b", "metadata": { "editable": true }, @@ -550,7 +550,7 @@ }, { "cell_type": "markdown", - "id": "07b53a45", + "id": "829664c0", "metadata": { "editable": true }, @@ -562,7 +562,7 @@ }, { "cell_type": "markdown", - "id": "1d8bc8d4", + "id": "c3462fff", "metadata": { "editable": true }, @@ -576,19 +576,19 @@ }, { "cell_type": "markdown", - "id": "7f06bfd3", + "id": "10d717b5", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[y]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n", "$$" ] }, { "cell_type": "markdown", - "id": "5624047e", + "id": "5858f4a4", "metadata": { "editable": true }, @@ -598,19 +598,19 @@ }, { "cell_type": "markdown", - "id": "1b26ef35", + "id": "c3b75de4", "metadata": { "editable": true }, "source": [ "$$\n", - "\\mathrm{Bias}[y]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", + "\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n", "$$" ] }, { "cell_type": "markdown", - "id": "6319fcec", + "id": "b8cc130e", "metadata": { "editable": true }, @@ -620,7 +620,7 @@ }, { "cell_type": "markdown", - "id": "2253bb98", + "id": "1ae109c5", "metadata": { "editable": true }, @@ -632,7 +632,7 @@ }, { "cell_type": "markdown", - "id": "d08baf03", + "id": "6bd24bc3", "metadata": { "editable": true }, @@ -651,7 +651,7 @@ }, { "cell_type": "markdown", - "id": "d06c2f80", + "id": "4b962286", "metadata": { "editable": true }, @@ -676,7 +676,7 @@ }, { "cell_type": "markdown", - "id": "2a957aee", + "id": "8a4f9316", "metadata": { "editable": true }, @@ -704,7 +704,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "1c0d6323", + "id": "96232c73", "metadata": { "collapsed": false, "editable": true @@ -716,7 +716,7 @@ }, { "cell_type": "markdown", - "id": "183f2666", + "id": "67545b6f", "metadata": { "editable": true }, @@ -728,7 +728,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "3ccf3ab9", + "id": "bad8a459", "metadata": { "collapsed": false, "editable": true @@ -754,7 +754,7 @@ }, { "cell_type": "markdown", - "id": "cf6cdda9", + "id": "1e6a8f3f", "metadata": { "editable": true }, @@ -779,7 +779,7 @@ }, { "cell_type": "markdown", - "id": "f28a3bdf", + "id": "3e942cfc", "metadata": { "editable": true }, @@ -793,7 +793,7 @@ }, { "cell_type": "markdown", - "id": "ed920879", + "id": "5451b425", "metadata": { "editable": true }, @@ -823,7 +823,7 @@ }, { "cell_type": "markdown", - "id": "f2bef519", + "id": "b0aabc96", "metadata": { "editable": true }, @@ -845,7 +845,7 @@ }, { "cell_type": "markdown", - "id": "e1891d3f", + "id": "a6299b8f", "metadata": { "editable": true }, diff --git a/doc/Projects/2023/Project1/ipynb/ipynb-Project1-src.tar.gz b/doc/Projects/2023/Project1/ipynb/ipynb-Project1-src.tar.gz index da81f803396e9e2bcaf351a81e7535691914029c..6e22948d1b9718f59b0c42c972f959126ecbd5cc 100644 GIT binary patch literal 193 zcmV;y06za8iwFSwmkDJ61MSaC3c@fD2H>uHia9|^nm*QoUAPd6c!AWWHq~Zol7hXx zeSoeMH${Yeo1bBZVdju+w)-rwck8W&5Rya@OqnL}oFrW88Ko)EC?+AJEKL~@%3>CJ zK-N3yr8k!A@szf1gfc?Cn;XZf>cgJp6?o>KIM&j@E+1@-3KUvVAYLHX*oc#*f$RoV vK#9UkP0-@jOD%w_0eD$TD^c;Q-)TH+-kQMw^*fH^IL?uHia9}EVhBjWm?j?J zdMCZ~#xgyg($)=^hRb(z<5*RF*t5I>&-@d|S{T^ngRPYWsVpy)xL_e{bW+5x)CZ2*^c7R&Dm1oWUHjo0(05-5|;MkYF+hXoXSQu@Yka}^wS$tT2S66MhDVocblyaH-$0qOcw&*TDe_sFP>Yd3(JW|C{ z!C?XS;~zZ5bd`khOazKGvTwVxy7QdtTmS5~1rC&3%5&kscVLr&jLHT!|MVm_Ss0dC znL1IJF=DaeOl4{!j<#cgnN7`NMD!-IE09_9ezDsK8^yA;-}b)~oe&o@qLD1MD$pE> zz>Xva-KBaW@lO#pJ__`gjwOEWiBd5kQX&m&Jx10tOq=Lhvu(}Rg01m0Thn2=*Ox19 zY?L48EH%AcZ!TBDrpuVFxMf`Tgs(i4KQ|hGUBreb1Plol3A}%_^cz2ZxD@HK@Z-E~4|^JS+q?^-@Txh`$a#ooPK(?XRng>K zvGLQ}rrr}z|ETJ+J^JCkhBL^Pn_?F+B2F6;Ro;_aDmEFsq# z)D*cZB5OMs<_*m@#m{Blm30-9)LPJgWHyK@B~c}lK!O^S%y7yWBI+;-7_CNB-fK8Z z#LR1S20jdjgGSLAKyhwbM> z-W08$?2B%T)^C>^{{p&AXZ5sDiVLRIOjX`EM@n(}Z_7JU^<2*eEYp&WfO4Di zF%~EUQ^_16OvM6Ulsy)C*p_R548U9O8pR$D@5WgO+DW?Hkn52(Djz*9M$d2j3kuXg zQx_o!bZLa!2*W_E@Wo{h581Hve4hJCLf$DTVENV7stGZ~x}KXFQ}!F7#S z_;v8J7nzBGq@f{OPFS2blYVI-0V@bK{T+C(U%I#ALqlG9TOTTN)f+s2IP(7bzAN|T zF9a}?!Nx<73{io8TQp7GoEV)>!1EM@FT=ec5H)8&wXIich|XxY8~sgy6sjCyA8^ZY9~A8^_;xN$h8S$@s9nB`Y6u{U08DN8kAA7<4E zU|qbuzW996n}m@}0eIJc8~_rqUhgkHd`uP__=rGC0+M=gqy8`_~iaf4Q2iIDv$Ko3-ghmJu6) z(t}U*6Gh+vL|{>Wiv!3u=?H<3qckM-$9tE`i*=8iJ)GPfis&f=px5(bo-947CIeI- zvI69Te1E^ADXeAOT4#$!WfcS%>9fpx0H`4B1J-Dv3dGS7-_{2|Sz{i8kg z*bWzln&99o0g@*mA2r{AhJ?iFJcep%B|&Z~tWiTwP*5C|<+v$dmcz)STl~1CZ-hNy zG#H$HKI9Dc^0q4gInDO*~8c6kXfmD8#Kx)52AXP4q z%9jLEI}fBMuq_+whuUx7f(dJqx z00t54aZGS;Obybf>P6lf_enY3J8jT ziogI9?g=TlU#|G_yw`)#j9Moz^2;wWKjF)l}!ntQ$hsRz&ugW6v?=y1CSZ#{4xku`-cKO&N!x3)r2ib0z6LFt`=;nz z_0^uOo<#~ioss7L<~GsX)~Gcs)THfiag6^fgK-7aYF_Zb1nQz=>lJKBEunA)Ht#M| zy=?O86Bf(#ZhVj9UfkfvqaU~30c*G;^`Y%~omuH(m)pUQ@88aQ6kcuZtgjq@FVfaE zb{-zf4i#~n{=wpiZb26ra#V;9s#SB{8V-Srbo)nlUQUcqt;)U{m7(U9=}9dK)J?uA zR<(oZo|b+y-c3q^rlHr-_}Qdm0Q1=vPh^Mk#Q?a4RLL^YUp|BaS|X>rrf}n|Uy?p% zOLEoSv8wlFzVp3q{NTDC8$a%UsVhA1!yYA}z|l)XLC4n{DpyNRN&17zVagBJLi2ws)Ko6y;HNe8`c(C z0-1Qa_6>@PcV$v6>#l{x@8iKcFp&xX0l=*>lcs2~Vb%*Bu4zGi7==K8#s5O*bKNia z?7LquxUF}qn1Y;B0c=R{<{1~*^*uF1SDrXn2gCBy;*q*`IB^{`gMADPm45uw;oiA7 z#HE!#ZU0Y-`jh{S?|59G90^m4=57&&|M(tIjgH;%x*P_1ZzArt>+aiES6681_gjcb z>z3U?oq4!nWqsv#yyB^UWhoY;;{x-?e7!BJ;)fz{&;+hF_4?|EV%;5@;;%JWz}0_l z4&`ohwe9x1tMxX&hpkA4v)_SMR+lTC_~__HKa&spne*ZL_LlbO|50}Sobj?Z1xpfG zhmaStGxjwy1;ToGR8n+*|Q%yDgGcnfE+>N|xef?7a9h_g{p3Q7_}{$-}a zW=}3Vdmbr3EInm$9eR9uNnOUR?`MORm7(cZ`zJMir3-?6V0T6dvj=K6u1C#8&j)iO z^(bpNbecSr5MPgfeoUbVfyH5$H_@C0#{>9Jo%*k9{uTaG-KqDR=%p#}-u0KJbcC)? zJ{+M#4WmF1kdvMzDXxXb{O;c)=)(KL2if5-4p02af@^1i6E2_PXV)){7u;(8qZNds zjbHNIj=+$2cQqW`cJHT{m%>3h!x=I(9~~r2`d}3@k_h#Ge=PcOh_X`A@f0A4MJSQ` zb-+{Z2-5ZVMhD=bCplT;*&cS`&zd^)zXy10D6_bapnw)5wiZLg#t-Y9xM4)gn<9GI z&IOb$X*SPk4pO`DT?L5Khwbrypk)6~3`h(J|3J{waKohn#q`b1_v`Xg*{xi_jGQ?d z`EWH>5{{}JiUMi^HVK4KQf4d>P%le-QXyj|G*BS_7x!O&goEEghu=a0hu=a1x8Fho zkI*PIG&M9fD=;!AFfb=63NKA>WJFbJVZG{MKm@wFf~CpI7CH6G&Mm& WMlv!qG(kBvG(kBqLAT%@1Y-irS%xP7 delta 3444 zcmV-)4U6)_!Vts45U>L_e~;t15x)1YFp!HIS+m8rw2L`pH#2!)H%KylNH7@$YFpjb zjxBj5dAq%b{q3nDi+Y*v?qCK1auO+(i^XC+zA8FdJS>vMAFdL5etUa$^G>Qo#xf(! z;`V;QRmvZ&a_MRUE9QZ94<*yLT_7Txv7f7?IbyffK|N2*vV zI4t0P{Jp1`u97gGi9oSN_H9>I51w;<@1Nbaz=3j0c`h9I4s0@zQQ5%eAD+Y}3&S!i zQzr^DMl4pGsZ348(RM5_v#D8(h~7kY1u|>iFLobcqga;q+x}Og6XIe2U4wrI0nBuuPtavI}CL|0j%cS&_ ze=M3J!c9n~V@pmrS|&}uq&rrKiB;X~^WE-5F&M&TQ{El@F~f_;vfKKYxi9j{zx$yw~u659~(b_OjQ1St@Y9i?LXf8C*=%y(QL{pZ%%%;EAqDU!}h0R z-W08$?2B%T)^C>^{{p4Q ziUkV6R5FJMQ?bAoWsgN3x8)jt1Mt?nMzP1kyK@$Tc9JeP9fYGo#Z9c%FjrWw;jvqUH>!w)JWa(HZS_1WvK@2!>k$s ztgE-TS3h0#CSfE~0Nyo!2Y>{u*ZZpvACtufJ|d8kfTSKB=^pU_inCk1zxtamS*9r} za7us>JlU;SMc^(-B?0DV0LbiwYEm))t`qtkJ2V?)wQSz>!F ziFk1=EOlxy=Ed}rK9_#L{`JK9U$15>&LAP+W^HEKd%vF>rRhm*Tw5j|x9^m=~GlcgusWPs{J zR)Bnv?+-hg!dk|yb+KqvRzZM~KFhoZfC|DsVvQE6KpdU$ZGH5UHRdrKG{s|6c10jY zS#VU+X99dUM8Jp793LJE`WGGoWe9;BYe-qJQkR!i8`u(mf4`HagvI%CU0g!M${k4>f*(Vd#XY=THy@z*P|?&6=w1$O zkk@mMfLcIpO1+{>5mDr<&*PdDBdTwrsRkmoQ8T-B&6gN7Qw)j?05CwS3Ep2)I2KA; zzN8X9R!M-}^RzW-YhCgwiI6eTw1aKbrEH>{@IBd=b}DlzmA;KO>t)iL#D|DmLbT)R z%em!$#|wUGzyP2qW=WF0{(&W1V68$M$^LDf})?n3;PV)1Er8W;jM*Iu9gBvASp6by*&ljYl| z!!9qWRr*L!KN)JKS*0N|3t@rUwf!A`gB;ySQTd_p25WspN!nhUCH>E!^mS*^9GaqY z6<2$)Viqa*bVi!{tJ_3#U!&HrIFq)&xiS9l3&s^qt9g+F6R3qwtyi!iwS;07*u1+= z^|HyUPgo$+yYbzPdvS*!Pk!8XN37dU)O)sWc$WiefqU`Of#Z@q_DhZ2Y)?r>^k4k9(Aa0!K6DOKpIGF`2fnlzXN1pWC*)^S+vT zAD>%2-9`v$A|ldKmyT+M>|FbaWxivNku=Q>{S z*>}8Pa9{6MF$FoN0@#q?Z8I*g>-%Vit~qhA4u<8Y#S?YvaN@dV274D6D*gD!y|x0g?YSVWqspzyyB^UWhs`S6BAtLv3cd~|f9pUH>)%=vJAe@}b#|0275&UnF_f+Y#8 zL&yu+1^b$x9Zyrn!A2;ggK7B$mqNgBmWE#WHAuy?<#!GE3`BE(gTb=gg!WuYF(%D| zbp=z$S6#8b2D9B(L6SbEChHuUxKk~)i9-@^tgD?_8N_D^c|N*4tC!0wC^W{=chT#p)vo)6|m z>Pyye=rnmOA-N60(Th^xz3VSZ=?GnK zd^kdf8b*O2ASXRbQd|p<`Q1N8(1rJf53 z>wu>`5TxtzjSj#=Pja%xvpw#@A2fC7c@OZ`P-bx-K>;mBY%PX}jUU!Ial?p~H%0V< zoeL;i(rlj79He&Py9yAe58LDaJ<0yx7myeb{(+$9;f6~Eis{>$@7Lw0vRk=;898$_ z^5HtHBpg*86$R7;Y!V2eq|8_%pk9{vq(a6_XrMs;4+WJF\n", @@ -15,9 +13,7 @@ { "cell_type": "markdown", "id": "342c9c77", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "# Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", @@ -28,9 +24,7 @@ { "cell_type": "markdown", "id": "8d89c4f2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Plans for week 35\n", "\n", @@ -54,9 +48,7 @@ { "cell_type": "markdown", "id": "4dc0b391", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "### Reading recommendations:\n", "\n", @@ -70,9 +62,7 @@ { "cell_type": "markdown", "id": "22518518", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week\n", "\n", @@ -104,9 +94,7 @@ { "cell_type": "markdown", "id": "40a10f3c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The equations for ordinary least squares\n", "\n", @@ -120,9 +108,7 @@ { "cell_type": "markdown", "id": "76f1c739", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_i=f(x_i)+\\epsilon_i,\n", @@ -132,9 +118,7 @@ { "cell_type": "markdown", "id": "2710298f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or in general" ] @@ -142,9 +126,7 @@ { "cell_type": "markdown", "id": "f1e8bd8f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y}=f(\\boldsymbol{x})+\\boldsymbol{\\epsilon},\n", @@ -154,9 +136,7 @@ { "cell_type": "markdown", "id": "79e795b0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{\\epsilon}$ represents some noise which is normally assumed to\n", "be distributed via a normal probability distribution with zero mean\n", @@ -175,9 +155,7 @@ { "cell_type": "markdown", "id": "c868570e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\beta},\n", @@ -187,9 +165,7 @@ { "cell_type": "markdown", "id": "855e1dbf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and in order to find the optimal parameters $\\beta_i$ we defined a function which\n", "gives a measure of the spread between the values $y_i$ (which\n", @@ -200,9 +176,7 @@ { "cell_type": "markdown", "id": "e76339ee", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The cost/loss function\n", "\n", @@ -212,9 +186,7 @@ { "cell_type": "markdown", "id": "706dd49c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", @@ -224,9 +196,7 @@ { "cell_type": "markdown", "id": "d068e28e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or using the matrix $\\boldsymbol{X}$ and in a more compact matrix-vector notation as" ] @@ -234,9 +204,7 @@ { "cell_type": "markdown", "id": "bb734579", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", @@ -246,9 +214,7 @@ { "cell_type": "markdown", "id": "aba5029f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This function represents one of many possible ways to define the so-called cost function.\n", "\n", @@ -259,9 +225,7 @@ { "cell_type": "markdown", "id": "6ac8d12b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{2n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2,\n", @@ -271,9 +235,7 @@ { "cell_type": "markdown", "id": "7d3ac82a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "since when taking the first derivative with respect to the unknown parameters $\\beta$, the factor of $2$ cancels out." ] @@ -281,9 +243,7 @@ { "cell_type": "markdown", "id": "1eedadb6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "\n", @@ -293,9 +253,7 @@ { "cell_type": "markdown", "id": "61c29b5e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\},\n", @@ -305,9 +263,7 @@ { "cell_type": "markdown", "id": "4cf9eba7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "can be linked to the variance of the quantity $y_i$ if we interpret the latter as the mean value. \n", "When linking (see the discussions next week) with the maximum likelihood approach below, we will indeed interpret $y_i$ as a mean value" @@ -316,9 +272,7 @@ { "cell_type": "markdown", "id": "0761eee3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_{i}=\\langle y_i \\rangle = \\beta_0x_{i,0}+\\beta_1x_{i,1}+\\beta_2x_{i,2}+\\dots+\\beta_{n-1}x_{i,n-1}+\\epsilon_i,\n", @@ -328,9 +282,7 @@ { "cell_type": "markdown", "id": "ead917bf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\langle y_i \\rangle$ is the mean value. Keep in mind also that\n", "till now we have treated $y_i$ as the exact value. Normally, the\n", @@ -347,9 +299,7 @@ { "cell_type": "markdown", "id": "92fe0780", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -360,9 +310,7 @@ { "cell_type": "markdown", "id": "396bf1e2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In practical terms it means we will require" ] @@ -370,9 +318,7 @@ { "cell_type": "markdown", "id": "6d5307de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = \\frac{\\partial }{\\partial \\beta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)^2\\right]=0,\n", @@ -382,9 +328,7 @@ { "cell_type": "markdown", "id": "45b68b6b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which results in" ] @@ -392,9 +336,7 @@ { "cell_type": "markdown", "id": "8bec3f2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_{ij}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)\\right]=0,\n", @@ -404,9 +346,7 @@ { "cell_type": "markdown", "id": "80dc90da", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or in a matrix-vector form as (multiplying away the factor $-2/n$, see derivation below)" ] @@ -414,9 +354,7 @@ { "cell_type": "markdown", "id": "64e3c687", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}^T} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right).\n", @@ -426,9 +364,7 @@ { "cell_type": "markdown", "id": "510d4d4c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "We can rewrite, see the derivations below," @@ -437,9 +373,7 @@ { "cell_type": "markdown", "id": "f0b1c164", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}^T} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right),\n", @@ -449,9 +383,7 @@ { "cell_type": "markdown", "id": "05896ad6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "as" ] @@ -459,9 +391,7 @@ { "cell_type": "markdown", "id": "bfa041c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{y} = \\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta},\n", @@ -471,9 +401,7 @@ { "cell_type": "markdown", "id": "ae4e69aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and if the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ is invertible we have the solution" ] @@ -481,9 +409,7 @@ { "cell_type": "markdown", "id": "22a53165", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta} =\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -493,9 +419,7 @@ { "cell_type": "markdown", "id": "02a2b1b8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We note also that since our design matrix is defined as $\\boldsymbol{X}\\in\n", "{\\mathbb{R}}^{n\\times p}$, the product $\\boldsymbol{X}^T\\boldsymbol{X} \\in\n", @@ -513,9 +437,7 @@ { "cell_type": "markdown", "id": "d1470cae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Some useful matrix and vector expressions\n", "\n", @@ -540,9 +462,7 @@ { "cell_type": "markdown", "id": "fbc63e64", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y}=f(\\boldsymbol{x}).\n", @@ -552,9 +472,7 @@ { "cell_type": "markdown", "id": "5475f4f0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Jacobian\n", "\n", @@ -564,9 +482,7 @@ { "cell_type": "markdown", "id": "7ccd0445", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{J}=\\frac{\\partial \\boldsymbol{y}}{\\partial \\boldsymbol{x}}=\\begin{bmatrix} \\frac{\\partial y_0}{\\partial x_0} & \\frac{\\partial y_0}{\\partial x_1} & \\frac{\\partial y_0}{\\partial x_2} & \\dots & \\dots & \\frac{\\partial y_0}{\\partial x_{n-1}} \\\\ \\frac{\\partial y_1}{\\partial x_0} & \\frac{\\partial y_1}{\\partial x_1} & \\frac{\\partial y_1}{\\partial x_2} & \\dots & \\dots & \\frac{\\partial y_1}{\\partial x_{n-1}} \\\\\n", @@ -580,9 +496,7 @@ { "cell_type": "markdown", "id": "c40947ac", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is an $m\\times n$ matrix. If $\\boldsymbol{x}$ is a scalar, then the\n", "Jacobian is only a single-column vector, or an $m\\times 1$ matrix. If\n", @@ -598,9 +512,7 @@ { "cell_type": "markdown", "id": "18e29324", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Derivatives, example 1\n", "\n", @@ -610,9 +522,7 @@ { "cell_type": "markdown", "id": "6069e0fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_i = \\sum_{j=0}^{n-1}a_{ij}x_j,\n", @@ -622,9 +532,7 @@ { "cell_type": "markdown", "id": "2796afe6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\forall i=0,1,2,\\dots,m-1$. The individual matrix elements of $\\boldsymbol{A}$ are given by the symbol $a_{ij}$.\n", "It follows that the partial derivatives of $y_i$ with respect to $x_k$" @@ -633,9 +541,7 @@ { "cell_type": "markdown", "id": "2b66e25f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial y_i }{\\partial x_k}= a_{ik} \\forall i=0,1,2,\\dots,m-1.\n", @@ -645,9 +551,7 @@ { "cell_type": "markdown", "id": "86f9add1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "From this we have, using the definition of the Jacobian" ] @@ -655,9 +559,7 @@ { "cell_type": "markdown", "id": "ae97bea9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\boldsymbol{y} }{\\partial \\boldsymbol{x}}= \\boldsymbol{A}.\n", @@ -667,9 +569,7 @@ { "cell_type": "markdown", "id": "14510558", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 2\n", "\n", @@ -681,9 +581,7 @@ { "cell_type": "markdown", "id": "fbb54b4a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{y}^T\\boldsymbol{A}\\boldsymbol{x},\n", @@ -693,9 +591,7 @@ { "cell_type": "markdown", "id": "553845ce", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{y}$ a vector of length $m$, $\\boldsymbol{A}$ an $m\\times n$ matrix and $\\boldsymbol{x}$ a vector of length $n$. We assume also that $\\boldsymbol{A}$ does not depend on any of the two vectors.\n", "In order to find the derivative of $\\alpha$ with respect to the two vectors, we define an intermediate vector $\\boldsymbol{z}$. We define first\n", @@ -705,9 +601,7 @@ { "cell_type": "markdown", "id": "e79fdc3c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{z}^T\\boldsymbol{x},\n", @@ -717,9 +611,7 @@ { "cell_type": "markdown", "id": "0713d61a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which means that (using our previous example) we have" ] @@ -727,9 +619,7 @@ { "cell_type": "markdown", "id": "2e522237", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = \\boldsymbol{z}=bm{A}^T\\boldsymbol{y}.\n", @@ -739,9 +629,7 @@ { "cell_type": "markdown", "id": "4baadc8b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Note that the resulting vector elements are the same for $\\boldsymbol{z}^T$ and $\\boldsymbol{z}$, the only difference is that one if just the transpose of the other.\n", "\n", @@ -751,9 +639,7 @@ { "cell_type": "markdown", "id": "6e0fee06", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{y}} = \\boldsymbol{z}^T=\\boldsymbol{x}^T\\boldsymbol{A}^T.\n", @@ -763,9 +649,7 @@ { "cell_type": "markdown", "id": "1d85d1c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 3\n", "\n", @@ -777,9 +661,7 @@ { "cell_type": "markdown", "id": "a6643a5f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{x}^T\\boldsymbol{A}\\boldsymbol{x},\n", @@ -789,9 +671,7 @@ { "cell_type": "markdown", "id": "f15d9044", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{x}$ a vector of length $n$.\n", "\n", @@ -801,9 +681,7 @@ { "cell_type": "markdown", "id": "658745bf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\sum_{i=0}^{n-1}\\sum_{j=0}^{n-1}x_i a_{ij}x_j,\n", @@ -813,9 +691,7 @@ { "cell_type": "markdown", "id": "ba8bc828", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "taking the derivative of $\\alpha$ with respect to a given component $x_k$ we get the two sums" ] @@ -823,9 +699,7 @@ { "cell_type": "markdown", "id": "38512288", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial x_k} = \\sum_{i=0}^{n-1}a_{ik}x_i+\\sum_{j=0}^{n-1}a_{kj}x_j,\n", @@ -835,9 +709,7 @@ { "cell_type": "markdown", "id": "0a571ae1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $\\forall k =0,1,2,\\dots,n-1$. We identify these sums as" ] @@ -845,9 +717,7 @@ { "cell_type": "markdown", "id": "f26f46f8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = \\boldsymbol{x}^T\\left(\\boldsymbol{A}^T+\\boldsymbol{A}\\right).\n", @@ -857,9 +727,7 @@ { "cell_type": "markdown", "id": "9e6c5896", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If the matrix $\\boldsymbol{A}$ is symmetric, that is $\\boldsymbol{A}=\\boldsymbol{A}^T$, we have" ] @@ -867,9 +735,7 @@ { "cell_type": "markdown", "id": "fa516251", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = 2\\boldsymbol{x}^T\\boldsymbol{A}.\n", @@ -879,9 +745,7 @@ { "cell_type": "markdown", "id": "10db7624", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 4\n", "\n", @@ -891,9 +755,7 @@ { "cell_type": "markdown", "id": "7cdeda2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{y}^T\\boldsymbol{x},\n", @@ -903,9 +765,7 @@ { "cell_type": "markdown", "id": "2b9a8aed", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where both $\\boldsymbol{y}$ and $\\boldsymbol{x}$ have the same length $n$, or if we\n", "wish to think of them as column vectors, they have dimensions $n\\times\n", @@ -918,9 +778,7 @@ { "cell_type": "markdown", "id": "758969d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\sum_{i=0}^{n-1}y_ix_i,\n", @@ -930,9 +788,7 @@ { "cell_type": "markdown", "id": "d7203c02", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the partial derivative" ] @@ -940,9 +796,7 @@ { "cell_type": "markdown", "id": "16f3ca20", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial z_k} = \\sum_{i=0}^{n-1}\\left(x_i\\frac{\\partial y_i}{\\partial z_k}+y_i\\frac{\\partial x_i}{\\partial z_k}\\right),\n", @@ -952,9 +806,7 @@ { "cell_type": "markdown", "id": "f7caf01e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $\\forall k =0,1,2,\\dots,n-1$. We can rewrite the partial derivative in a more compact form as" ] @@ -962,9 +814,7 @@ { "cell_type": "markdown", "id": "b1e5ad17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{z}} = \\boldsymbol{x}^T\\frac{\\partial \\boldsymbol{y}}{\\partial \\boldsymbol{z}}+\\boldsymbol{y}^T\\frac{\\partial \\boldsymbol{x}}{\\partial \\boldsymbol{z}},\n", @@ -974,9 +824,7 @@ { "cell_type": "markdown", "id": "587c347e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and if $\\boldsymbol{y}=\\boldsymbol{x}$ we have" ] @@ -984,9 +832,7 @@ { "cell_type": "markdown", "id": "5279b1cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{z}} = 2\\boldsymbol{x}^T\\frac{\\partial \\boldsymbol{x}}{\\partial \\boldsymbol{z}}.\n", @@ -996,9 +842,7 @@ { "cell_type": "markdown", "id": "0c0d18ed", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The mean squared error and its derivative\n", "\n", @@ -1008,9 +852,7 @@ { "cell_type": "markdown", "id": "ba1b34c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", @@ -1020,9 +862,7 @@ { "cell_type": "markdown", "id": "b859b644", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or using the design/feature matrix $\\boldsymbol{X}$ we have the more compact matrix-vector" ] @@ -1030,9 +870,7 @@ { "cell_type": "markdown", "id": "a6451ed1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", @@ -1042,9 +880,7 @@ { "cell_type": "markdown", "id": "6879be83", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We note that the design matrix $\\boldsymbol{X}$ does not depend on the unknown parameters defined by the vector $\\boldsymbol{\\beta}$.\n", "We are now interested in minimizing the cost function with respect to the unknown parameters $\\boldsymbol{\\beta}$.\n", @@ -1055,9 +891,7 @@ { "cell_type": "markdown", "id": "bda465e5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{w}=\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta},\n", @@ -1067,9 +901,7 @@ { "cell_type": "markdown", "id": "aeb97168", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which depends on $\\boldsymbol{\\beta}$. We rewrite the cost function as" ] @@ -1077,9 +909,7 @@ { "cell_type": "markdown", "id": "d5339f3b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\boldsymbol{w}^T\\boldsymbol{w},\n", @@ -1089,9 +919,7 @@ { "cell_type": "markdown", "id": "924aee00", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with partial derivative" ] @@ -1099,9 +927,7 @@ { "cell_type": "markdown", "id": "856a72bf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=\\frac{2}{n}\\boldsymbol{w}^T\\frac{\\partial \\boldsymbol{w}}{\\partial \\boldsymbol{\\beta}},\n", @@ -1111,9 +937,7 @@ { "cell_type": "markdown", "id": "ad9fe745", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using that" ] @@ -1121,9 +945,7 @@ { "cell_type": "markdown", "id": "5a72fd42", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\boldsymbol{w}}{\\partial \\boldsymbol{\\beta}}=-\\boldsymbol{X},\n", @@ -1133,9 +955,7 @@ { "cell_type": "markdown", "id": "569524a1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we used the result from example two above. Inserting the last expression we obtain" ] @@ -1143,9 +963,7 @@ { "cell_type": "markdown", "id": "4e8d401f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-\\frac{2}{n}\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\boldsymbol{X},\n", @@ -1155,9 +973,7 @@ { "cell_type": "markdown", "id": "24230b58", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or as" ] @@ -1165,9 +981,7 @@ { "cell_type": "markdown", "id": "fe2c5d6e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}^T}=-\\frac{2}{n}\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right).\n", @@ -1177,9 +991,7 @@ { "cell_type": "markdown", "id": "9b68b468", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Other useful relations\n", "\n", @@ -1189,9 +1001,7 @@ { "cell_type": "markdown", "id": "bf6a5077", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial (\\boldsymbol{b}^T\\boldsymbol{a})}{\\partial \\boldsymbol{a}} = \\boldsymbol{b},\n", @@ -1201,9 +1011,7 @@ { "cell_type": "markdown", "id": "ddf09a27", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial tr(\\boldsymbol{B}\\boldsymbol{A})}{\\partial \\boldsymbol{A}} = \\boldsymbol{B}^T,\n", @@ -1213,9 +1021,7 @@ { "cell_type": "markdown", "id": "e1665513", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\log{\\vert\\boldsymbol{A}\\vert}}{\\partial \\boldsymbol{A}} = (\\boldsymbol{A}^{-1})^T.\n", @@ -1225,9 +1031,7 @@ { "cell_type": "markdown", "id": "e754746c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Meet the Hessian Matrix\n", "\n", @@ -1241,9 +1045,7 @@ { "cell_type": "markdown", "id": "922d0d8c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial}{\\partial \\boldsymbol{\\beta}}\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}^T} =\\frac{\\partial}{\\partial \\boldsymbol{\\beta}}\\left[-\\frac{2}{n}\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right]=\\frac{2}{n}\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -1253,9 +1055,7 @@ { "cell_type": "markdown", "id": "acf89848", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The Hessian matrix plays an important role and is defined here as" ] @@ -1263,9 +1063,7 @@ { "cell_type": "markdown", "id": "12d22c90", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{H}=\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -1275,9 +1073,7 @@ { "cell_type": "markdown", "id": "9dbcb6d7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For ordinary least squares, it is inversely proportional (derivation\n", "next week) with the variance of the optimal parameters\n", @@ -1293,9 +1089,7 @@ { "cell_type": "markdown", "id": "b797b390", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "\n", @@ -1305,9 +1099,7 @@ { "cell_type": "markdown", "id": "b78ab217", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon} = \\boldsymbol{y}-\\boldsymbol{\\tilde{y}} = \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta},\n", @@ -1317,9 +1109,7 @@ { "cell_type": "markdown", "id": "e0a0542d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and with" ] @@ -1327,9 +1117,7 @@ { "cell_type": "markdown", "id": "fd640b28", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", @@ -1339,9 +1127,7 @@ { "cell_type": "markdown", "id": "6a073b35", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have" ] @@ -1349,9 +1135,7 @@ { "cell_type": "markdown", "id": "f7daf28e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{\\epsilon}=\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", @@ -1361,9 +1145,7 @@ { "cell_type": "markdown", "id": "d8e03fdc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that the solution for $\\boldsymbol{\\beta}$ is the one which minimizes the residuals." ] @@ -1371,9 +1153,7 @@ { "cell_type": "markdown", "id": "058aed61", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example relevant for the exercises\n", "\n", @@ -1385,9 +1165,7 @@ { "cell_type": "markdown", "id": "c7d35183", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_i = \\beta_0+\\beta_1x_i+\\beta_2x_i^2+\\beta_3x_i^3+\\beta_4x_i^4.\n", @@ -1397,9 +1175,7 @@ { "cell_type": "markdown", "id": "bad6fd9a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have five predictors/features. The first is the intercept $\\beta_0$. The other terms are $\\beta_i$ with $i=1,2,3,4$. Furthermore we have $n$ entries for each predictor. It means that our design matrix is an \n", "$n\\times p$ matrix $\\boldsymbol{X}$." @@ -1408,9 +1184,7 @@ { "cell_type": "markdown", "id": "1bd57074", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Own code for Ordinary Least Squares\n", "\n", @@ -1421,10 +1195,7 @@ "cell_type": "code", "execution_count": 1, "id": "9895174d", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# matrix inversion to find beta\n", @@ -1448,9 +1219,7 @@ { "cell_type": "markdown", "id": "0d7ad6dc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Alternatively, you can use the least squares functionality in **Numpy** as" ] @@ -1459,10 +1228,7 @@ "cell_type": "code", "execution_count": 2, "id": "e52ec127", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "fit = np.linalg.lstsq(X, y, rcond =None)[0]\n", @@ -1472,9 +1238,7 @@ { "cell_type": "markdown", "id": "c66acc10", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Adding error analysis and training set up\n", "\n", @@ -1486,10 +1250,7 @@ "cell_type": "code", "execution_count": 3, "id": "85f5f060", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def R2(y_data, y_model):\n", @@ -1499,9 +1260,7 @@ { "cell_type": "markdown", "id": "ef359561", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and we would be using it as" ] @@ -1510,10 +1269,7 @@ "cell_type": "code", "execution_count": 4, "id": "21b8437f", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(R2(y,ytilde))" @@ -1522,9 +1278,7 @@ { "cell_type": "markdown", "id": "b1c814a4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can easily add our **MSE** score as" ] @@ -1533,10 +1287,7 @@ "cell_type": "code", "execution_count": 5, "id": "de8f1db6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def MSE(y_data,y_model):\n", @@ -1549,9 +1300,7 @@ { "cell_type": "markdown", "id": "421ff6ee", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and finally the relative error as" ] @@ -1560,10 +1309,7 @@ "cell_type": "code", "execution_count": 6, "id": "77d5b744", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def RelativeError(y_data,y_model):\n", @@ -1574,9 +1320,7 @@ { "cell_type": "markdown", "id": "0afffa89", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Splitting our Data in Training and Test data\n", "\n", @@ -1595,9 +1339,7 @@ { "cell_type": "markdown", "id": "83fc18dd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The complete code with a simple data set" ] @@ -1606,10 +1348,7 @@ "cell_type": "code", "execution_count": 7, "id": "c908e069", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -1659,9 +1398,7 @@ { "cell_type": "markdown", "id": "f69cf8d1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Making your own test-train splitting" ] @@ -1670,10 +1407,7 @@ "cell_type": "code", "execution_count": 8, "id": "03acb6ba", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# equivalently in numpy\n", @@ -1695,9 +1429,7 @@ { "cell_type": "markdown", "id": "0d8db71d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "But since **scikit-learn** has its own function for doing this and since\n", "it interfaces easily with **tensorflow** and other libraries, we\n", @@ -1707,9 +1439,7 @@ { "cell_type": "markdown", "id": "29fc4792", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Reducing the number of degrees of freedom, overarching view\n", "\n", @@ -1736,9 +1466,7 @@ { "cell_type": "markdown", "id": "5d2323e6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Preprocessing our data\n", "\n", @@ -1761,9 +1489,7 @@ { "cell_type": "markdown", "id": "084a05ce", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Functionality in Scikit-Learn\n", "\n", @@ -1781,9 +1507,7 @@ { "cell_type": "markdown", "id": "2cf236cf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More preprocessing\n", "\n", @@ -1808,9 +1532,7 @@ { "cell_type": "markdown", "id": "f7ef3d03", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Frequently used scaling functions\n", "\n", @@ -1821,9 +1543,7 @@ { "cell_type": "markdown", "id": "30df9a47", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n", @@ -1833,9 +1553,7 @@ { "cell_type": "markdown", "id": "fa957ecb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard deviation, respectively, of the feature $x_j$.\n", "This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one." @@ -1844,9 +1562,7 @@ { "cell_type": "markdown", "id": "c6b8f467", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example of own Standard scaling\n", "\n", @@ -1860,10 +1576,7 @@ "cell_type": "code", "execution_count": 9, "id": "a2480cf9", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import sklearn.linear_model as skl\n", @@ -1894,9 +1607,7 @@ { "cell_type": "markdown", "id": "7643608b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives." ] @@ -1904,9 +1615,7 @@ { "cell_type": "markdown", "id": "5e6e489d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Min-Max Scaling\n", "\n", @@ -1919,9 +1628,7 @@ { "cell_type": "markdown", "id": "ff073975", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "x_j^{(i)} \\rightarrow (b-a)\\frac{x_j^{(i)} - \\min(x_j)}{\\max(x_j) - \\min(x_j)} - a\n", @@ -1931,9 +1638,7 @@ { "cell_type": "markdown", "id": "6a176186", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively." ] @@ -1941,9 +1646,7 @@ { "cell_type": "markdown", "id": "784ddba6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Testing the Means Squared Error as function of Complexity\n", "\n", @@ -1957,10 +1660,7 @@ "cell_type": "code", "execution_count": 10, "id": "83686834", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.random.seed()\n", @@ -1974,9 +1674,7 @@ { "cell_type": "markdown", "id": "dd641a05", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $y$ is the function we want to fit with a given polynomial.\n", "\n", @@ -1987,10 +1685,7 @@ "cell_type": "code", "execution_count": 11, "id": "447d12ce", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -2034,9 +1729,7 @@ { "cell_type": "markdown", "id": "96fc590b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More preprocessing examples, two-dimensional example, the Franke function" ] @@ -2045,10 +1738,7 @@ "cell_type": "code", "execution_count": 12, "id": "8de8bbfc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Common imports\n", @@ -2148,9 +1838,7 @@ { "cell_type": "markdown", "id": "8dfe4a90", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## To think about, first part\n", "\n", @@ -2176,9 +1864,7 @@ { "cell_type": "markdown", "id": "c0edb6d6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More thinking\n", "\n", @@ -2211,9 +1897,7 @@ { "cell_type": "markdown", "id": "ed0f6bc0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Still thinking\n", "\n", @@ -2226,10 +1910,7 @@ "cell_type": "code", "execution_count": 13, "id": "99a1133f", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "#Model training, we compute the mean value of y and X\n", @@ -2251,9 +1932,7 @@ { "cell_type": "markdown", "id": "5ef9b25c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## What does centering (subtracting the mean values) mean mathematically?\n", "\n", @@ -2267,9 +1946,7 @@ { "cell_type": "markdown", "id": "80ca5485", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n", @@ -2279,9 +1956,7 @@ { "cell_type": "markdown", "id": "17ad9823", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.\n", "\n", @@ -2294,9 +1969,7 @@ { "cell_type": "markdown", "id": "7560f426", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C}{\\partial \\beta_j} = 0,\n", @@ -2306,9 +1979,7 @@ { "cell_type": "markdown", "id": "a7444f64", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for all $j$. For $\\beta_0$ we have" ] @@ -2316,9 +1987,7 @@ { "cell_type": "markdown", "id": "2aad0c8b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n", @@ -2328,9 +1997,7 @@ { "cell_type": "markdown", "id": "de870a72", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying away the constant $2/n$, we obtain" ] @@ -2338,9 +2005,7 @@ { "cell_type": "markdown", "id": "7a6c517e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n", @@ -2350,9 +2015,7 @@ { "cell_type": "markdown", "id": "753afefe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Further Manipulations\n", "\n", @@ -2363,9 +2026,7 @@ { "cell_type": "markdown", "id": "3005c150", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n", @@ -2375,9 +2036,7 @@ { "cell_type": "markdown", "id": "79eace73", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We obtain then" ] @@ -2385,9 +2044,7 @@ { "cell_type": "markdown", "id": "38d75fef", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", @@ -2397,9 +2054,7 @@ { "cell_type": "markdown", "id": "60f367fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we define" ] @@ -2407,9 +2062,7 @@ { "cell_type": "markdown", "id": "9131a799", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mu_1=\\frac{1}{n}\\sum_{i=0}^{n-1} (X_{i1},\n", @@ -2419,9 +2072,7 @@ { "cell_type": "markdown", "id": "8cbfafa2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and if we define the mean value of the outputs as" ] @@ -2429,9 +2080,7 @@ { "cell_type": "markdown", "id": "9d4a45df", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", @@ -2441,9 +2090,7 @@ { "cell_type": "markdown", "id": "19284bfb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have" ] @@ -2451,9 +2098,7 @@ { "cell_type": "markdown", "id": "e1ccefe0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_0 = \\mu_y - \\beta_1\\mu_{1}.\n", @@ -2463,9 +2108,7 @@ { "cell_type": "markdown", "id": "e6228ac9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the general case, that is we have more parameters than $\\beta_0$ and $\\beta_1$, we have" ] @@ -2473,9 +2116,7 @@ { "cell_type": "markdown", "id": "474d165a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n", @@ -2485,9 +2126,7 @@ { "cell_type": "markdown", "id": "01c6f359", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Replacing $y_i$ with $y_i - y_i - \\overline{\\boldsymbol{y}}$ and centering also our design matrix results in a cost function (in vector-matrix disguise)" ] @@ -2495,9 +2134,7 @@ { "cell_type": "markdown", "id": "104df9ea", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n", @@ -2507,9 +2144,7 @@ { "cell_type": "markdown", "id": "86f33ebf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Wrapping it up\n", "\n", @@ -2519,9 +2154,7 @@ { "cell_type": "markdown", "id": "b8893d78", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", @@ -2531,9 +2164,7 @@ { "cell_type": "markdown", "id": "cc8ec307", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", @@ -2544,9 +2175,7 @@ { "cell_type": "markdown", "id": "234e3f6e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", @@ -2556,9 +2185,7 @@ { "cell_type": "markdown", "id": "d1c0397f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "What does this mean? And why do we insist on all this? Let us look at some examples." ] @@ -2566,9 +2193,7 @@ { "cell_type": "markdown", "id": "799912a7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linear Regression code, Intercept handling first\n", "\n", @@ -2580,10 +2205,7 @@ "cell_type": "code", "execution_count": 14, "id": "5a3de91e", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -2677,9 +2299,7 @@ { "cell_type": "markdown", "id": "0f069973", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The intercept is the value of our output/target variable\n", "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", @@ -2698,9 +2318,7 @@ { "cell_type": "markdown", "id": "e76beb96", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\beta_j^2,\n", @@ -2710,9 +2328,7 @@ { "cell_type": "markdown", "id": "4fe81d64", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "but when we take out the intercept, this equation becomes" ] @@ -2720,9 +2336,7 @@ { "cell_type": "markdown", "id": "bb22d80f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\beta_j^2.\n", @@ -2732,9 +2346,7 @@ { "cell_type": "markdown", "id": "293ce41e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For Lasso regression we have" ] @@ -2742,9 +2354,7 @@ { "cell_type": "markdown", "id": "8a969677", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\beta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\beta_j\\vert.\n", @@ -2754,9 +2364,7 @@ { "cell_type": "markdown", "id": "d93685ad", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that, when scaling the design matrix and the outputs/targets,\n", "by subtracting the mean values, we have an optimization problem which\n", @@ -2770,9 +2378,7 @@ { "cell_type": "markdown", "id": "9abd5b44", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Boston housing data example\n", "\n", @@ -2814,9 +2420,7 @@ { "cell_type": "markdown", "id": "b8bd6825", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Housing data, the code\n", "We start by importing the libraries" @@ -2826,10 +2430,7 @@ "cell_type": "code", "execution_count": 15, "id": "2c2b3d87", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -2842,9 +2443,7 @@ { "cell_type": "markdown", "id": "fea1bc2e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and load the Boston Housing DataSet from **Scikit-Learn**" ] @@ -2853,10 +2452,7 @@ "cell_type": "code", "execution_count": 16, "id": "a80976dc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.datasets import load_boston\n", @@ -2871,9 +2467,7 @@ { "cell_type": "markdown", "id": "aa3b722c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Then we invoke Pandas" ] @@ -2882,10 +2476,7 @@ "cell_type": "code", "execution_count": 17, "id": "828172e1", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)\n", @@ -2896,9 +2487,7 @@ { "cell_type": "markdown", "id": "eee71c11", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and preprocess the data" ] @@ -2907,10 +2496,7 @@ "cell_type": "code", "execution_count": 18, "id": "7c8a5c54", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# check for missing values in all the columns\n", @@ -2920,9 +2506,7 @@ { "cell_type": "markdown", "id": "a0c893b5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can then visualize the data" ] @@ -2931,10 +2515,7 @@ "cell_type": "code", "execution_count": 19, "id": "58711c38", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# set the size of the figure\n", @@ -2948,9 +2529,7 @@ { "cell_type": "markdown", "id": "d899ec3f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It is now useful to look at the correlation matrix" ] @@ -2959,10 +2538,7 @@ "cell_type": "code", "execution_count": 20, "id": "59925df7", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# compute the pair wise correlation for all columns \n", @@ -2975,9 +2551,7 @@ { "cell_type": "markdown", "id": "083a679e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "From the above coorelation plot we can see that **MEDV** is strongly correlated to **LSTAT** and **RM**. We see also that **RAD** and **TAX** are stronly correlated, but we don't include this in our features together to avoid multi-colinearity" ] @@ -2986,10 +2560,7 @@ "cell_type": "code", "execution_count": 21, "id": "127a3dda", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(20, 5))\n", @@ -3010,9 +2581,7 @@ { "cell_type": "markdown", "id": "c72d1b7f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Now we start training our model" ] @@ -3021,10 +2590,7 @@ "cell_type": "code", "execution_count": 22, "id": "a1378976", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])\n", @@ -3034,9 +2600,7 @@ { "cell_type": "markdown", "id": "d85205c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We split the data into training and test sets" ] @@ -3045,10 +2609,7 @@ "cell_type": "code", "execution_count": 23, "id": "040b1e85", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -3065,9 +2626,7 @@ { "cell_type": "markdown", "id": "32d25d47", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Then we use the linear regression functionality from **Scikit-Learn**" ] @@ -3076,10 +2635,7 @@ "cell_type": "code", "execution_count": 24, "id": "1b0fac0d", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.linear_model import LinearRegression\n", @@ -3119,10 +2675,7 @@ "cell_type": "code", "execution_count": 25, "id": "02279b28", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# plotting the y_test vs y_pred\n", @@ -3134,9 +2687,7 @@ { "cell_type": "markdown", "id": "cd590eba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Material for lecture Thursday, August 31" ] @@ -3144,9 +2695,7 @@ { "cell_type": "markdown", "id": "9968e6df", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Mathematical Interpretation of Ordinary Least Squares\n", "\n", @@ -3158,9 +2707,7 @@ { "cell_type": "markdown", "id": "6e0b933c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -3170,9 +2717,7 @@ { "cell_type": "markdown", "id": "e3019867", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The **hat** over $\\boldsymbol{\\beta}$ means we have the optimal parameters after minimization of the cost function.\n", "\n", @@ -3182,9 +2727,7 @@ { "cell_type": "markdown", "id": "84b218f8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}=\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}} = \\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -3194,9 +2737,7 @@ { "cell_type": "markdown", "id": "19a7596b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We now define a matrix" ] @@ -3204,9 +2745,7 @@ { "cell_type": "markdown", "id": "bd703e67", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{A}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T.\n", @@ -3216,9 +2755,7 @@ { "cell_type": "markdown", "id": "123c9939", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can rewrite" ] @@ -3226,9 +2763,7 @@ { "cell_type": "markdown", "id": "e25cd9d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}=\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}} = \\boldsymbol{A}\\boldsymbol{y}.\n", @@ -3238,9 +2773,7 @@ { "cell_type": "markdown", "id": "7a90c314", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{A}$ has the important property that $\\boldsymbol{A}^2=\\boldsymbol{A}$. This is the definition of a projection matrix.\n", "We can then interpret our optimal model $\\tilde{\\boldsymbol{y}}$ as being represented by an orthogonal projection of $\\boldsymbol{y}$ onto a space defined by the column vectors of $\\boldsymbol{X}$. In our case here the matrix $\\boldsymbol{A}$ is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix." @@ -3249,9 +2782,7 @@ { "cell_type": "markdown", "id": "342cbb0e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Residual Error\n", "\n", @@ -3261,9 +2792,7 @@ { "cell_type": "markdown", "id": "8174f656", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon}=\\boldsymbol{y}-\\tilde{\\boldsymbol{y}}=\\left[\\boldsymbol{I}-\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\right]\\boldsymbol{y}.\n", @@ -3273,9 +2802,7 @@ { "cell_type": "markdown", "id": "9d4a6e02", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The residual errors are then the projections of $\\boldsymbol{y}$ onto the orthogonal component of the space defined by the column vectors of $\\boldsymbol{X}$." ] @@ -3283,9 +2810,7 @@ { "cell_type": "markdown", "id": "6165636b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple case\n", "\n", @@ -3295,9 +2820,7 @@ { "cell_type": "markdown", "id": "cd5ead70", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{X}\\boldsymbol{X}^T = \\boldsymbol{I}.\n", @@ -3307,9 +2830,7 @@ { "cell_type": "markdown", "id": "ea18f6fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the matrix $\\boldsymbol{A}$ becomes" ] @@ -3317,9 +2838,7 @@ { "cell_type": "markdown", "id": "9b42e1cf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{A}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T)=\\boldsymbol{I},\n", @@ -3329,9 +2848,7 @@ { "cell_type": "markdown", "id": "ddc58b22", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and we have the obvious case" ] @@ -3339,9 +2856,7 @@ { "cell_type": "markdown", "id": "288e8e6a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon}=\\boldsymbol{y}-\\tilde{\\boldsymbol{y}}=0.\n", @@ -3351,9 +2866,7 @@ { "cell_type": "markdown", "id": "4fd80694", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This serves also as a useful test of our codes." ] @@ -3361,9 +2874,7 @@ { "cell_type": "markdown", "id": "b006a8c2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The singular value decomposition\n", "\n", @@ -3401,9 +2912,7 @@ { "cell_type": "markdown", "id": "725d878b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linear Regression Problems\n", "\n", @@ -3418,9 +2927,7 @@ { "cell_type": "markdown", "id": "fd5e5178", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -3441,9 +2948,7 @@ { "cell_type": "markdown", "id": "153ac58e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The columns of $\\boldsymbol{X}$ are linearly dependent. We see this easily since the \n", "the first column is the row-wise sum of the other two columns. The rank (more correct,\n", @@ -3458,9 +2963,7 @@ { "cell_type": "markdown", "id": "3a4c4a00", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -3477,9 +2980,7 @@ { "cell_type": "markdown", "id": "c102eee7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see easily that $\\mbox{det}(\\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \\times (-1) - 1 \\times (-1) = 0$. Hence, $\\mathbf{X}$ is singular and its inverse is undefined.\n", "This is equivalent to saying that the matrix $\\boldsymbol{X}$ has at least an eigenvalue which is zero." @@ -3488,9 +2989,7 @@ { "cell_type": "markdown", "id": "7005d428", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Fixing the singularity\n", "\n", @@ -3500,9 +2999,7 @@ { "cell_type": "markdown", "id": "ebdbde59", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -3518,9 +3015,7 @@ { "cell_type": "markdown", "id": "8757437b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "has linearly dependent column vectors, we will not be able to compute the inverse\n", "of $\\boldsymbol{X}^T\\boldsymbol{X}$ and we cannot find the parameters (estimators) $\\beta_i$. \n", @@ -3534,9 +3029,7 @@ { "cell_type": "markdown", "id": "dba77cf1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^{T} \\boldsymbol{X} \\rightarrow \\boldsymbol{X}^{T} \\boldsymbol{X}+\\lambda \\boldsymbol{I},\n", @@ -3546,9 +3039,7 @@ { "cell_type": "markdown", "id": "2f275695", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{I}$ is the identity matrix. When we discuss **Ridge** regression this is actually what we end up evaluating. The parameter $\\lambda$ is called a hyperparameter. More about this later." ] @@ -3556,9 +3047,7 @@ { "cell_type": "markdown", "id": "32c17033", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Basic math of the SVD\n", "\n", @@ -3571,9 +3060,7 @@ { "cell_type": "markdown", "id": "13d50641", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\lambda_1,\\boldsymbol{u}_1),\\dots, (\\lambda_n,\\boldsymbol{u}_n),\n", @@ -3583,9 +3070,7 @@ { "cell_type": "markdown", "id": "e32d182b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the eigenvalues are given by the diagonal matrix" ] @@ -3593,9 +3078,7 @@ { "cell_type": "markdown", "id": "566185cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\\mathrm{Diag}(\\lambda_1, \\dots,\\lambda_n).\n", @@ -3605,9 +3088,7 @@ { "cell_type": "markdown", "id": "69cc07d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{X}$ can be written in terms of an orthogonal/unitary transformation $\\boldsymbol{U}$" ] @@ -3615,9 +3096,7 @@ { "cell_type": "markdown", "id": "5a62e55b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -3627,9 +3106,7 @@ { "cell_type": "markdown", "id": "185a7b6c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{I}$ or $\\boldsymbol{U}\\boldsymbol{U}^{\\dagger}=\\boldsymbol{I}$.\n", "\n", @@ -3639,9 +3116,7 @@ { "cell_type": "markdown", "id": "b63994c4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\begin{bmatrix} \n", @@ -3654,9 +3129,7 @@ { "cell_type": "markdown", "id": "1b5e5ea8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is not diagonalizable, it is a so-called [defective matrix](https://en.wikipedia.org/wiki/Defective_matrix). It is easy to see that the condition\n", "$\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{X}^T\\boldsymbol{X}$ is not fulfilled." @@ -3665,9 +3138,7 @@ { "cell_type": "markdown", "id": "c7cba8d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The SVD, a Fantastic Algorithm\n", "\n", @@ -3685,9 +3156,7 @@ { "cell_type": "markdown", "id": "34a321a1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\n", @@ -3697,9 +3166,7 @@ { "cell_type": "markdown", "id": "adeb4e2a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "As an example, the above defective matrix can be decomposed as" ] @@ -3707,9 +3174,7 @@ { "cell_type": "markdown", "id": "983c4d05", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& 1 \\\\ 1& -1\\\\ \\end{bmatrix} \\begin{bmatrix} 2& 0 \\\\ 0& 0\\\\ \\end{bmatrix} \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& -1 \\\\ 1& 1\\\\ \\end{bmatrix}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -3719,9 +3184,7 @@ { "cell_type": "markdown", "id": "30746d8f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with eigenvalues $\\sigma_1=2$ and $\\sigma_2=0$. \n", "The SVD exits always! \n", @@ -3748,9 +3211,7 @@ { "cell_type": "markdown", "id": "15b1904e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Economy-size SVD\n", "\n", @@ -3775,9 +3236,7 @@ { "cell_type": "markdown", "id": "825d56c0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Codes for the SVD" ] @@ -3786,10 +3245,7 @@ "cell_type": "code", "execution_count": 26, "id": "83389e7a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -3826,9 +3282,7 @@ { "cell_type": "markdown", "id": "6c1d5c64", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{X}$ has columns that are linearly dependent. The first\n", "column is the row-wise sum of the other two columns. The rank of a\n", @@ -3843,9 +3297,7 @@ { "cell_type": "markdown", "id": "092d08fb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Note about SVD Calculations\n", "\n", @@ -3866,9 +3318,7 @@ { "cell_type": "markdown", "id": "680ca861", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Mathematics of the SVD and implications\n", "\n", @@ -3880,9 +3330,7 @@ { "cell_type": "markdown", "id": "d9109bdb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -3899,9 +3347,7 @@ { "cell_type": "markdown", "id": "1bc782ad", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can SVD decompose our matrix as" ] @@ -3909,9 +3355,7 @@ { "cell_type": "markdown", "id": "fd98474c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -3921,9 +3365,7 @@ { "cell_type": "markdown", "id": "27505b56", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{U}$ is an orthogonal matrix of dimension $n\\times n$, meaning that $\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{U}^T\\boldsymbol{U}=\\boldsymbol{I}_n$. Here $\\boldsymbol{I}_n$ is the unit matrix of dimension $n \\times n$.\n", "\n", @@ -3935,9 +3377,7 @@ { "cell_type": "markdown", "id": "97458ec0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sigma_0 > \\sigma_1 > \\sigma_2 > \\dots > \\sigma_{p-1} > 0.\n", @@ -3947,9 +3387,7 @@ { "cell_type": "markdown", "id": "6a940c34", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "All values beyond $p-1$ are all zero." ] @@ -3957,9 +3395,7 @@ { "cell_type": "markdown", "id": "beb7c9c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example Matrix\n", "\n", @@ -3969,9 +3405,7 @@ { "cell_type": "markdown", "id": "892eca8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\n", @@ -3986,9 +3420,7 @@ { "cell_type": "markdown", "id": "d716176c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The singular values are $\\sigma_0=2$ and $\\sigma_1=1$. It is common to rewrite the matrix $\\boldsymbol{\\Sigma}$ as" ] @@ -3996,9 +3428,7 @@ { "cell_type": "markdown", "id": "d6f76ea3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\n", @@ -4012,9 +3442,7 @@ { "cell_type": "markdown", "id": "6cd96381", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where" ] @@ -4022,9 +3450,7 @@ { "cell_type": "markdown", "id": "3235bb2a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\tilde{\\Sigma}}=\n", @@ -4038,9 +3464,7 @@ { "cell_type": "markdown", "id": "cfb5e5fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "contains only the singular values. Note also (and we will use this below) that" ] @@ -4048,9 +3472,7 @@ { "cell_type": "markdown", "id": "30c811c8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}=\n", @@ -4064,9 +3486,7 @@ { "cell_type": "markdown", "id": "b64ff89b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is a $2\\times 2 $ matrix while" ] @@ -4074,9 +3494,7 @@ { "cell_type": "markdown", "id": "d2d214b1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T=\n", @@ -4091,9 +3509,7 @@ { "cell_type": "markdown", "id": "1734cbea", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is a $3\\times 3 $ matrix. The last row and column of this last matrix\n", "contain only zeros. This will have important consequences for our SVD\n", @@ -4103,9 +3519,7 @@ { "cell_type": "markdown", "id": "18b87d79", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the Matrix to be inverted\n", "\n", @@ -4115,9 +3529,7 @@ { "cell_type": "markdown", "id": "7424dbeb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -4127,9 +3539,7 @@ { "cell_type": "markdown", "id": "8bfde7d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using the orthogonality of the matrix $\\boldsymbol{U}$ we have" ] @@ -4137,9 +3547,7 @@ { "cell_type": "markdown", "id": "d7fae153", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -4149,9 +3557,7 @@ { "cell_type": "markdown", "id": "a5fe659e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We define $\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}=\\tilde{\\boldsymbol{\\Sigma}}^2$ which is a diagonal matrix containing only the singular values squared. It has dimensionality $p \\times p$.\n", "\n", @@ -4161,9 +3567,7 @@ { "cell_type": "markdown", "id": "0494a740", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -4173,9 +3577,7 @@ { "cell_type": "markdown", "id": "cd9e9a2d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using our SVD decomposition of $\\boldsymbol{X}$ we have" ] @@ -4183,9 +3585,7 @@ { "cell_type": "markdown", "id": "a36f09cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\left(\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^{2}(\\boldsymbol{V}^T\\right)^{-1}\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{y},\n", @@ -4195,9 +3595,7 @@ { "cell_type": "markdown", "id": "c595c4ab", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which gives us, using the orthogonality of the matrix $\\boldsymbol{V}$," ] @@ -4205,9 +3603,7 @@ { "cell_type": "markdown", "id": "54ea720c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_i\\boldsymbol{u}^T_i\\boldsymbol{y},\n", @@ -4217,9 +3613,7 @@ { "cell_type": "markdown", "id": "3b8cb8cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that the ordinary least square model (with the optimal\n", "parameters) $\\boldsymbol{\\tilde{y}}$, corresponds to an orthogonal\n", @@ -4233,9 +3627,7 @@ { "cell_type": "markdown", "id": "e825b356", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Further properties (important for our analyses later)\n", "\n", @@ -4245,9 +3637,7 @@ { "cell_type": "markdown", "id": "2cbc23f9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -4257,9 +3647,7 @@ { "cell_type": "markdown", "id": "8665bb8b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we now multiply from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" ] @@ -4267,9 +3655,7 @@ { "cell_type": "markdown", "id": "2aedf32c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}.\n", @@ -4279,9 +3665,7 @@ { "cell_type": "markdown", "id": "e64ed163", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This means the vectors $\\boldsymbol{v}_i$ of the orthogonal matrix $\\boldsymbol{V}$ are the eigenvectors of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$\n", "with eigenvalues given by the singular values squared, that is" @@ -4290,9 +3674,7 @@ { "cell_type": "markdown", "id": "b060bb08", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", @@ -4302,9 +3684,7 @@ { "cell_type": "markdown", "id": "0e9f682e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Similarly, if we use the SVD decomposition for the matrix $\\boldsymbol{X}\\boldsymbol{X}^T$, we have" ] @@ -4312,9 +3692,7 @@ { "cell_type": "markdown", "id": "084d10ae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T.\n", @@ -4324,9 +3702,7 @@ { "cell_type": "markdown", "id": "322ae2b5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we now multiply from the right with $\\boldsymbol{U}$ (using the orthogonality of $\\boldsymbol{U}$) we get" ] @@ -4334,9 +3710,7 @@ { "cell_type": "markdown", "id": "2b934378", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}\\boldsymbol{X}^T\\right)\\boldsymbol{U}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T.\n", @@ -4346,9 +3720,7 @@ { "cell_type": "markdown", "id": "76fa0775", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This means the vectors $\\boldsymbol{u}_i$ of the orthogonal matrix $\\boldsymbol{U}$ are the eigenvectors of the matrix $\\boldsymbol{X}\\boldsymbol{X}^T$\n", "with eigenvalues given by the singular values squared, that is" @@ -4357,9 +3729,7 @@ { "cell_type": "markdown", "id": "4ab1ab5e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}\\boldsymbol{X}^T\\right)\\boldsymbol{u}_i=\\boldsymbol{u}_i\\sigma_i^2.\n", @@ -4369,9 +3739,7 @@ { "cell_type": "markdown", "id": "5c3528cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "**Important note**: we have defined our design matrix $\\boldsymbol{X}$ to be an\n", "$n\\times p$ matrix. In most supervised learning cases we have that $n\n", @@ -4387,9 +3755,7 @@ { "cell_type": "markdown", "id": "99446c56", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Meet the Covariance Matrix\n", "\n", @@ -4403,9 +3769,7 @@ { "cell_type": "markdown", "id": "52b4ef86", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial^2 C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} =\\frac{2}{n}\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -4415,9 +3779,7 @@ { "cell_type": "markdown", "id": "14d03f17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).\n", "\n", @@ -4427,9 +3789,7 @@ { "cell_type": "markdown", "id": "08983b4e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{H}=\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -4439,9 +3799,7 @@ { "cell_type": "markdown", "id": "bf4aff1c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The Hessian matrix for ordinary least squares is also proportional to\n", "the covariance matrix. This means also that we can use the SVD to find\n", @@ -4452,9 +3810,7 @@ { "cell_type": "markdown", "id": "d1a70450", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Introducing the Covariance and Correlation functions\n", "\n", @@ -4468,9 +3824,7 @@ { "cell_type": "markdown", "id": "4441a82e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -4482,9 +3836,7 @@ { "cell_type": "markdown", "id": "985fe9f4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where for example" ] @@ -4492,9 +3844,7 @@ { "cell_type": "markdown", "id": "7d0b48ad", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", @@ -4504,9 +3854,7 @@ { "cell_type": "markdown", "id": "927a1cc9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "With this definition and recalling that the variance is defined as" ] @@ -4514,9 +3862,7 @@ { "cell_type": "markdown", "id": "28fe8612", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{var}[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", @@ -4526,9 +3872,7 @@ { "cell_type": "markdown", "id": "643c5c24", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we can rewrite the covariance matrix as" ] @@ -4536,9 +3880,7 @@ { "cell_type": "markdown", "id": "31eb5551", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -4550,9 +3892,7 @@ { "cell_type": "markdown", "id": "ee97772f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "**Note:** we have used $1/n$ in the above definitions of the *sample* variance and covariance. We assume then that we can calculate the exact mean value. \n", "What you will find in essentially all statistics texts are equations\n", @@ -4567,9 +3907,7 @@ { "cell_type": "markdown", "id": "fdef2a70", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Covariance and Correlation Matrix\n", "\n", @@ -4583,9 +3921,7 @@ { "cell_type": "markdown", "id": "68211a1f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{\\mathrm{var}[\\boldsymbol{x}] \\mathrm{var}[\\boldsymbol{y}]}}.\n", @@ -4595,9 +3931,7 @@ { "cell_type": "markdown", "id": "9a8a18e2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The correlation function is then given by values $\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]\n", "\\in [-1,1]$. This avoids eventual problems with too large values. We\n", @@ -4608,9 +3942,7 @@ { "cell_type": "markdown", "id": "e7d94e46", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & \\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -4622,9 +3954,7 @@ { "cell_type": "markdown", "id": "0c9c0c07", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the above example this is the function we constructed using **pandas**." ] @@ -4632,9 +3962,7 @@ { "cell_type": "markdown", "id": "3fc933af", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Function and Design/Feature Matrix\n", "\n", @@ -4645,9 +3973,7 @@ { "cell_type": "markdown", "id": "e97cce9c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -4664,9 +3990,7 @@ { "cell_type": "markdown", "id": "d1be42f8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors/features $p$ refering to the column numbers and the\n", "entries $n$ being the row elements.\n", @@ -4676,9 +4000,7 @@ { "cell_type": "markdown", "id": "e957492f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix} \\boldsymbol{x}_0 & \\boldsymbol{x}_1 & \\boldsymbol{x}_2 & \\dots & \\dots & \\boldsymbol{x}_{p-1}\\end{bmatrix},\n", @@ -4688,9 +4010,7 @@ { "cell_type": "markdown", "id": "6f4bd14c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with a given vector" ] @@ -4698,9 +4018,7 @@ { "cell_type": "markdown", "id": "ba1b4c89", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{x}_i^T = \\begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \\dots & \\dots x_{n-1,i}\\end{bmatrix}.\n", @@ -4710,9 +4028,7 @@ { "cell_type": "markdown", "id": "c9563317", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "With these definitions, we can now rewrite our $2\\times 2$\n", "correlation/covariance matrix in terms of a moe general design/feature\n", @@ -4723,9 +4039,7 @@ { "cell_type": "markdown", "id": "ed451710", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -4742,9 +4056,7 @@ { "cell_type": "markdown", "id": "4eabcccb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the correlation matrix" ] @@ -4752,9 +4064,7 @@ { "cell_type": "markdown", "id": "78678f6f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -4771,9 +4081,7 @@ { "cell_type": "markdown", "id": "5840e451", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Covariance Matrix Examples\n", "\n", @@ -4789,9 +4097,7 @@ { "cell_type": "markdown", "id": "75f212b9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{W} = \\begin{bmatrix} x_0 & x_1 & x_2 & \\dots & x_{n-2} & x_{n-1} \\\\\n", @@ -4803,9 +4109,7 @@ { "cell_type": "markdown", "id": "af4b97f1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which in turn is converted into into the $2\\times 2$ covariance matrix\n", "$\\boldsymbol{C}$ via the Numpy function **np.cov()**. We note that we can also calculate\n", @@ -4818,10 +4122,7 @@ "cell_type": "code", "execution_count": 27, "id": "bceeb8a6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Importing various packages\n", @@ -4839,9 +4140,7 @@ { "cell_type": "markdown", "id": "e438fa73", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Matrix\n", "\n", @@ -4856,10 +4155,7 @@ "cell_type": "code", "execution_count": 28, "id": "01ec3279", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -4888,9 +4184,7 @@ { "cell_type": "markdown", "id": "fb6ba409", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see that the matrix elements along the diagonal are one as they\n", "should be and that the matrix is symmetric. Furthermore, diagonalizing\n", @@ -4902,9 +4196,7 @@ { "cell_type": "markdown", "id": "6c5bfe67", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Matrix with Pandas\n", "\n", @@ -4915,10 +4207,7 @@ "cell_type": "code", "execution_count": 29, "id": "b2b07565", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -4940,9 +4229,7 @@ { "cell_type": "markdown", "id": "e26d9b42", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We expand this model to the Franke function discussed above." ] @@ -4950,22 +4237,26 @@ { "cell_type": "markdown", "id": "6200c4da", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Matrix with Pandas and the Franke function" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 4, "id": "1a0f5c54", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1. 1. 3. 1. 3. 9.]\n", + " [ 1. 2. 4. 4. 8. 16.]]\n" + ] + } + ], "source": [ "# Common imports\n", "import numpy as np\n", @@ -4998,26 +4289,24 @@ "\n", "\n", "# Making meshgrid of datapoints and compute Franke's function\n", - "n = 4\n", - "N = 100\n", - "x = np.sort(np.random.uniform(0, 1, N))\n", - "y = np.sort(np.random.uniform(0, 1, N))\n", + "n = 2\n", + "N = 2\n", + "x = np.array([1,2])#np.sort(np.random.uniform(0, 1, N))\n", + "y = np.array([3,4])#np.sort(np.random.uniform(0, 1, N))\n", "z = FrankeFunction(x, y)\n", "X = create_X(x, y, n=n) \n", - "\n", + "print(X)\n", "Xpd = pd.DataFrame(X)\n", "# subtract the mean values and set up the covariance matrix\n", "Xpd = Xpd - Xpd.mean()\n", "covariance_matrix = Xpd.cov()\n", - "print(covariance_matrix)" + "#print(covariance_matrix)" ] }, { "cell_type": "markdown", "id": "be666eb5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We note here that the covariance is zero for the first rows and\n", "columns since all matrix elements in the design matrix were set to one\n", @@ -5032,9 +4321,7 @@ { "cell_type": "markdown", "id": "cc79dbf1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Rewriting the Covariance and/or Correlation Matrix\n", "\n", @@ -5044,9 +4331,7 @@ { "cell_type": "markdown", "id": "7aaa9582", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}= \\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}].\n", @@ -5056,9 +4341,7 @@ { "cell_type": "markdown", "id": "3c9ca18c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "To see this let us simply look at a design matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{2\\times 2}$" ] @@ -5066,9 +4349,7 @@ { "cell_type": "markdown", "id": "7450e1de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -5083,9 +4364,7 @@ { "cell_type": "markdown", "id": "c190057c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we then compute the expectation value (note the $1/n$ factor instead of $1/(n-1)$)" ] @@ -5093,9 +4372,7 @@ { "cell_type": "markdown", "id": "20f4c172", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}=\\frac{1}{n}\\begin{bmatrix}\n", @@ -5108,9 +4385,7 @@ { "cell_type": "markdown", "id": "746607df", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is just" ] @@ -5118,9 +4393,7 @@ { "cell_type": "markdown", "id": "d2419f36", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]=\\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] \\\\\n", @@ -5132,9 +4405,7 @@ { "cell_type": "markdown", "id": "0c4d5713", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we wrote $$\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]$$ to indicate that this is the covariance of the vectors $\\boldsymbol{x}$ of the design/feature matrix $\\boldsymbol{X}$.\n", "\n", @@ -5144,9 +4415,7 @@ { "cell_type": "markdown", "id": "c51b1e85", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linking with the SVD\n", "\n", @@ -5156,9 +4425,7 @@ { "cell_type": "markdown", "id": "df3d6e17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -5168,9 +4435,7 @@ { "cell_type": "markdown", "id": "bda15ad2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $p\\times p$, with $p$ corresponding to the singular values, we defined earlier the matrix" ] @@ -5178,9 +4443,7 @@ { "cell_type": "markdown", "id": "40ad7cbc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma} = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix},\n", @@ -5190,9 +4453,7 @@ { "cell_type": "markdown", "id": "305bd308", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where the tilde-matrix $\\tilde{\\boldsymbol{\\Sigma}}$ is a matrix of dimension $p\\times p$ containing only the singular values $\\sigma_i$, that is" ] @@ -5200,9 +4461,7 @@ { "cell_type": "markdown", "id": "bb83e451", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{\\Sigma}}=\\begin{bmatrix} \\sigma_0 & 0 & 0 & \\dots & 0 & 0 \\\\\n", @@ -5217,9 +4476,7 @@ { "cell_type": "markdown", "id": "64d007a7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning we can write" ] @@ -5227,9 +4484,7 @@ { "cell_type": "markdown", "id": "c0fe2564", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2\\boldsymbol{V}^T.\n", @@ -5239,9 +4494,7 @@ { "cell_type": "markdown", "id": "630c43cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" ] @@ -5249,9 +4502,7 @@ { "cell_type": "markdown", "id": "1e412f4c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2.\n", @@ -5261,9 +4512,7 @@ { "cell_type": "markdown", "id": "2294efa0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## What does it mean?\n", "\n", @@ -5275,9 +4524,7 @@ { "cell_type": "markdown", "id": "ec2d9133", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", @@ -5287,9 +4534,7 @@ { "cell_type": "markdown", "id": "be85e557", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In other words, each non-zero singular value of $\\boldsymbol{X}$ is a positive\n", "square root of an eigenvalue of $\\boldsymbol{X}^T\\boldsymbol{X}$. It means also that\n", @@ -5309,9 +4554,7 @@ { "cell_type": "markdown", "id": "61531c33", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{X}]=\\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X},\n", @@ -5321,9 +4564,7 @@ { "cell_type": "markdown", "id": "549a9854", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that every squared non-singular value of $\\boldsymbol{X}$ divided by $n$ (\n", "the number of samples) are the eigenvalues of the covariance\n", @@ -5336,9 +4577,7 @@ { "cell_type": "markdown", "id": "ace8d1d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## And finally $\\boldsymbol{X}\\boldsymbol{X}^T$\n", "\n", @@ -5348,9 +4587,7 @@ { "cell_type": "markdown", "id": "4624e6fc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{U}^T.\n", @@ -5360,9 +4597,7 @@ { "cell_type": "markdown", "id": "4b8a8b55", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $n\\times n$, we have" ] @@ -5370,9 +4605,7 @@ { "cell_type": "markdown", "id": "e8a41480", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\boldsymbol{0}\\\\ \\end{bmatrix}=\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix},\n", @@ -5382,9 +4615,7 @@ { "cell_type": "markdown", "id": "6a36c786", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "leading to" ] @@ -5392,9 +4623,7 @@ { "cell_type": "markdown", "id": "462349df", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\boldsymbol{U}^T.\n", @@ -5404,9 +4633,7 @@ { "cell_type": "markdown", "id": "f3e432cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying with $\\boldsymbol{U}$ from the right gives us the eigenvalue problem" ] @@ -5414,9 +4641,7 @@ { "cell_type": "markdown", "id": "20a1bdac", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U}=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}.\n", @@ -5426,9 +4651,7 @@ { "cell_type": "markdown", "id": "43ac5bba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that the eigenvalues of $\\boldsymbol{X}\\boldsymbol{X}^T$ are again given by\n", "the non-zero singular values plus now a series of zeros. The column\n", @@ -5443,9 +4666,7 @@ { "cell_type": "markdown", "id": "ff1f6546", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ridge and LASSO Regression\n", "\n", @@ -5456,9 +4677,7 @@ { "cell_type": "markdown", "id": "cf2d5142", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", @@ -5468,9 +4687,7 @@ { "cell_type": "markdown", "id": "0e74ad82", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or we can state it as" ] @@ -5478,9 +4695,7 @@ { "cell_type": "markdown", "id": "f0e17a56", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -5491,9 +4706,7 @@ { "cell_type": "markdown", "id": "b11ffbe6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have used the definition of a norm-2 vector, that is" ] @@ -5501,9 +4714,7 @@ { "cell_type": "markdown", "id": "3f4e49db", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", @@ -5513,9 +4724,7 @@ { "cell_type": "markdown", "id": "e22c2482", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "By minimizing the above equation with respect to the parameters\n", "$\\boldsymbol{\\beta}$ we could then obtain an analytical expression for the\n", @@ -5526,9 +4735,7 @@ { "cell_type": "markdown", "id": "a7e7dfe5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -5539,9 +4746,7 @@ { "cell_type": "markdown", "id": "94414ad1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to the Ridge regression minimization problem where we\n", "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n", @@ -5551,9 +4756,7 @@ { "cell_type": "markdown", "id": "cbb571a8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -5563,9 +4766,7 @@ { "cell_type": "markdown", "id": "08d69de8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have a new optimization equation" ] @@ -5573,9 +4774,7 @@ { "cell_type": "markdown", "id": "17827d0b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -5586,9 +4785,7 @@ { "cell_type": "markdown", "id": "a2606f59", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", "\n", @@ -5598,9 +4795,7 @@ { "cell_type": "markdown", "id": "1a261469", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", @@ -5610,9 +4805,7 @@ { "cell_type": "markdown", "id": "2c25d0ff", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Ridge Regression Equations\n", "\n", @@ -5622,9 +4815,7 @@ { "cell_type": "markdown", "id": "1cbbda83", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n", @@ -5634,9 +4825,7 @@ { "cell_type": "markdown", "id": "10e5cd79", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and \n", "taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n", @@ -5648,9 +4837,7 @@ { "cell_type": "markdown", "id": "8149c527", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -5660,9 +4847,7 @@ { "cell_type": "markdown", "id": "86370c66", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that" ] @@ -5670,9 +4855,7 @@ { "cell_type": "markdown", "id": "ca970c01", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n", @@ -5682,9 +4865,7 @@ { "cell_type": "markdown", "id": "0a6651d6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $t$ a finite positive number. \n", "\n", @@ -5694,9 +4875,7 @@ { "cell_type": "markdown", "id": "feb46caa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+n\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -5706,9 +4885,7 @@ { "cell_type": "markdown", "id": "ba2ccc4c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In many textbooks the $1/n$ term is often omitted. Note that a library like **Scikit-Learn** does not include the $1/n$ factor in the setup of the cost function.\n", "\n", @@ -5718,9 +4895,7 @@ { "cell_type": "markdown", "id": "99bd5e5f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -5730,9 +4905,7 @@ { "cell_type": "markdown", "id": "122ac3fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n", "\n", @@ -5749,9 +4922,7 @@ { "cell_type": "markdown", "id": "2ed2f1af", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\beta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n", @@ -5761,9 +4932,7 @@ { "cell_type": "markdown", "id": "ab6d373f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For Ridge regression this becomes" ] @@ -5771,9 +4940,7 @@ { "cell_type": "markdown", "id": "12e89aa0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\beta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n", @@ -5783,9 +4950,7 @@ { "cell_type": "markdown", "id": "f5ad1a22", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$." ] @@ -5793,9 +4958,7 @@ { "cell_type": "markdown", "id": "972a591b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpreting the Ridge results\n", "\n", @@ -5805,9 +4968,7 @@ { "cell_type": "markdown", "id": "29b0a40c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n", @@ -5817,9 +4978,7 @@ { "cell_type": "markdown", "id": "974b89b6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n", "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n", @@ -5833,9 +4992,7 @@ { "cell_type": "markdown", "id": "d32b5a75", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More interpretations\n", "\n", @@ -5845,9 +5002,7 @@ { "cell_type": "markdown", "id": "b7b0e4a9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n", @@ -5857,9 +5012,7 @@ { "cell_type": "markdown", "id": "00480506", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the standard OLS results in" ] @@ -5867,9 +5020,7 @@ { "cell_type": "markdown", "id": "e478d349", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{n-1}\\boldsymbol{u}_i\\boldsymbol{u}_i^T\\boldsymbol{y},\n", @@ -5879,9 +5030,7 @@ { "cell_type": "markdown", "id": "9941badc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -5889,9 +5038,7 @@ { "cell_type": "markdown", "id": "d78d129f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n", @@ -5901,9 +5048,7 @@ { "cell_type": "markdown", "id": "81e43d35", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n", "the Ridge estimator converges to zero when the hyperparameter goes to\n", @@ -5918,9 +5063,7 @@ { "cell_type": "markdown", "id": "ab030d9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Lasso Regression Equations\n", "\n", @@ -5930,9 +5073,7 @@ { "cell_type": "markdown", "id": "5c1a039a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -5942,9 +5083,7 @@ { "cell_type": "markdown", "id": "64005abb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Taking the derivative with respect to $\\boldsymbol{\\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)" ] @@ -5952,9 +5091,7 @@ { "cell_type": "markdown", "id": "dd03458c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{d \\vert \\beta\\vert}{d \\beta}=\\mathrm{sgn}(\\beta)=\\left\\{\\begin{array}{cc} 1 & \\beta > 0 \\\\-1 & \\beta < 0, \\end{array}\\right.\n", @@ -5964,9 +5101,7 @@ { "cell_type": "markdown", "id": "c1845e9a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have that the derivative of the cost function is" ] @@ -5974,9 +5109,7 @@ { "cell_type": "markdown", "id": "9daa3df6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-\\frac{2}{n}\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=0,\n", @@ -5986,9 +5119,7 @@ { "cell_type": "markdown", "id": "add636f0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and reordering we have" ] @@ -5996,9 +5127,7 @@ { "cell_type": "markdown", "id": "79c36fde", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta}+\\frac{n}{2}\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -6008,9 +5137,7 @@ { "cell_type": "markdown", "id": "d4b4abb2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can redefine $\\lambda$ to absorb the constant $n/2$ and we rewrite the last equation as" ] @@ -6018,9 +5145,7 @@ { "cell_type": "markdown", "id": "93a6f35c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta}+\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -6030,15 +5155,31 @@ { "cell_type": "markdown", "id": "62647fc5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later." ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/pub/week38/ipynb/week38.ipynb b/doc/pub/week38/ipynb/week38.ipynb index b4915cba6..92233816e 100644 --- a/doc/pub/week38/ipynb/week38.ipynb +++ b/doc/pub/week38/ipynb/week38.ipynb @@ -169,13 +169,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "id": "7487b649", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAG0CAYAAADacZikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABYLUlEQVR4nO3dd3gUZb/G8e+mF1JIgBQSejP03pQiHaWoCDaKvQCCKCIqihWxwavYC+W1gIoURVqUKiA1SCcgkFBiqGmQunP+4HWPgSSQkGR2s/fnuuYcZuaZ2XuW5d2fzzz7jMUwDAMRERERJ+JidgARERGR0qYCSERERJyOCiARERFxOiqARERExOmoABIRERGnowJIREREnI4KIBEREXE6KoBERETE6agAEhEREaejAkhEREScjqkF0KRJk2jZsiV+fn5UqlSJ/v37s2/fvlxtDMNg4sSJhIeH4+3tTadOndi1a9cVzz137lyioqLw9PQkKiqKefPmldRliIiIiIOxmPkssJ49e3LHHXfQsmVLsrOzee6559ixYwe7d+/G19cXgMmTJ/Paa68xY8YM6tSpw6uvvsrq1avZt28ffn5+eZ53/fr13HDDDbzyyivccsstzJs3jxdeeIG1a9fSunXrK+ayWq0cP34cPz8/LBZLsV6ziIiIlAzDMEhJSSE8PBwXlyv08Rh2JDEx0QCMVatWGYZhGFar1QgNDTXeeOMNW5v09HQjICDA+Pjjj/M9z8CBA42ePXvm2tajRw/jjjvuuKoc8fHxBqBFixYtWrRoccAlPj7+it/1btiRpKQkAIKCggA4dOgQCQkJdO/e3dbG09OTjh07sm7dOh5++OE8z7N+/XqeeOKJXNt69OjB1KlT82yfkZFBRkaGbd34X6dYfHw8/v7+Rb4eERERKT3JyclERkbme4fo3+ymADIMgzFjxnD99dfToEEDABISEgAICQnJ1TYkJIQjR47ke66EhIQ8j/nnfJeaNGkSL7300mXb/f39VQCJiIg4mKsZvmI3vwIbMWIEf/75J99+++1l+y69EMMwrnhxhTlm/PjxJCUl2Zb4+PhCphcRERFHYhc9QCNHjmThwoWsXr2aiIgI2/bQ0FDgYo9OWFiYbXtiYuJlPTz/FhoaellvT0HHeHp64unpeS2XICIiIg7E1B4gwzAYMWIEP/74I7/99hvVq1fPtb969eqEhoayfPly27bMzExWrVpFu3bt8j1v27Ztcx0DsGzZsgKPEREREedhag/Q8OHD+eabb1iwYAF+fn62XpuAgAC8vb2xWCyMHj2a119/ndq1a1O7dm1ef/11fHx8uOuuu2znGTJkCJUrV2bSpEkAjBo1ig4dOjB58mT69evHggULiI6OZu3atcWaPycnh6ysrGI9p8il3N3dcXV1NTuGiEiZYmoB9NFHHwHQqVOnXNunT5/OsGHDAHj66ae5cOECjz32GGfPnqV169YsW7Ys1wjvuLi4XL/3b9euHbNnz+b5559nwoQJ1KxZkzlz5lzVHEBXwzAMEhISOHfuXLGcT+RKAgMDCQ0N1bxUIiLFxNSJEO1VcnIyAQEBJCUl5fkrsBMnTnDu3DkqVaqEj4+PvpSkxBiGwfnz50lMTCQwMDDXWDgREcntSt/f/2YXg6AdSU5Ojq34CQ4ONjuOOAFvb2/g4kD+SpUq6XaYiEgxsJufwTuKf8b8+Pj4mJxEnMk/nzeNORMRKR4qgIpIt72kNOnzJiJSvFQAiYiIiNNRASQiIiJORwWQlEkWi4X58+ebHUNEROyUfgUmIiIipcKanUPamXOknU/HGhhEeKC3aVlUAEmesrKycHd3NztGmZOZmYmHh4fZMUREiswwDFLPJJF85Dhpx05w4fjfZCYkkpOYiHHqFK6nT/PNXWM4gzsp6VkMmPcxXTctxSc9Dd+MC/hhsKlGCz55+j3mPNzWtOvQLbBiYBgG5zOzTVkKM4+l1Wpl8uTJ1KpVC09PT6pUqcJrr73G4cOHsVgsfPfdd3Tq1AkvLy+++uorrFYrL7/8MhEREXh6etKkSROWLFliO19mZiYjRowgLCwMLy8vqlWrZnscCcDEiROpUqUKnp6ehIeH8/jjj18x4/jx42nTps1l2xs1asSLL74IwKZNm+jWrRsVKlQgICCAjh07snXr1qt+H/7tStdw7tw5HnroIUJCQvDy8qJBgwb8/PPPtv1z586lfv36eHp6Uq1aNd55551c569WrRqvvvoqw4YNIyAggAcffBCAdevW0aFDB7y9vYmMjOTxxx8nLS2tSNcgIlJcDKuVUwfj2Dt/GVvf+YwNj09gw633sqVdT+74YDU3vPkbdScsYVnHW6ncvD51+nal8SN303LiE7T5cBJtv/uMVr/+yLotB1m9/yTb4s6RcTaZkKST+GWcx4WL31m+2RmY/eNW9QAVgwtZOUS9sNSU1979cg98PK7ur3H8+PF89tlnTJkyheuvv54TJ06wd+9e2/5x48bxzjvvMH36dDw9PfnPf/7DO++8wyeffELTpk358ssv6du3L7t27aJ27dq89957LFy4kO+++44qVaoQHx9PfHw8AD/88ANTpkxh9uzZ1K9fn4SEBLZv337FjHfffTdvvPEGBw8epGbNmgDs2rWLHTt28MMPPwCQkpLC0KFDee+99wB455136N27N7GxsbkekXI1CroGq9VKr169SElJ4auvvqJmzZrs3r3bNhHhli1bGDhwIBMnTmTQoEGsW7eOxx57jODgYNujXADeeustJkyYwPPPPw/Ajh076NGjB6+88gpffPEFJ0+eZMSIEYwYMYLp06cXKr+ISFGkJJ7m2OqN/BkZxeGzFzh8Oo0bv3ibXr8voELmBSrkccyRJndwwr8iAGe9/Tnv7kmybwCp5QI5H1CezMAgsoOCMYKDeaJPYzwqBlPOy40KPV8g9vwYvIID8akUhG+FIFr5ejPbxdw+GD0KIw8FTaWdnp7OoUOHqF69Ol5eXgCcz8y2+wIoJSWFihUrMm3aNB544IFc+w4fPkz16tWZOnUqo0aNsm2vXLkyw4cP59lnn7Vta9WqFS1btuSDDz7g8ccfZ9euXURHR182T827777LJ598ws6dOwt9K61x48YMGDCACRMmAPDss88SHR3Nxo0b82yfk5ND+fLl+eabb7j55puBi4Og582bR//+/Qt8rYKuYdmyZfTq1Ys9e/ZQp06dy469++67OXnyJMuWLbNte/rpp1m0aBG7du0CLvYANW3alHnz5tnaDBkyBG9vbz755BPbtrVr19KxY0fS0tJsn6t/y+tzJyJyJYbVSsLO/RxftprMLdvw3ruLSkdiCT978eHjbR6dQYL/xXJn7KqZDN/wPVYsJAZW5GxwKOcrhJAVGgYRESQNvJPgKuGE+HtRwdsNb2/7u52vR2GUMm93V3a/3MO0174ae/bsISMjgy5duuTbpkWLFrY/Jycnc/z4cdq3b5+rTfv27W09OcOGDaNbt27UrVuXnj17cvPNN9O9e3cAbr/9dqZOnUqNGjXo2bMnvXv3pk+fPri5Xfkjd/fdd/Pll18yYcIEDMPg22+/ZfTo0bb9iYmJvPDCC/z222/8/fff5OTkcP78eeLi4q7qvfi3gq4hJiaGiIiIPIsfuPie9uvX77L3Z+rUqeTk5Nh6iv79vsLFnqMDBw7w9ddf27YZhoHVauXQoUNcd911hb4OERGAjLTz7DiRytbjKWw9co62n73F0DVzyOspgn8HVKRLJVdcm1alarAvdbs/wxHPcYQ2uY5QXx9CSz196VIBVAwsFstV34Yyyz/PkyqIr6/vZdsu7RUxDMO2rVmzZhw6dIjFixcTHR3NwIED6dq1Kz/88AORkZHs27eP5cuXEx0dzWOPPcZbb73FqlWrrtgjdNddd/HMM8+wdetWLly4QHx8PHfccYdt/7Bhwzh58iRTp06latWqeHp60rZtWzIzM6/mrciloGu40nv27/fi39suden7arVaefjhh/McE1WlSpVCX4OIOC/DauXQig0k/vgzPqtXUHvvNiYNeoUtEVEAeAZEcJeLK4cr1+Js3QYYDRtSrmVTIm5oRUhEKK/lOlt1My7BNPb9rS3Fpnbt2nh7e/Prr79edgssL/7+/oSHh7N27Vo6dOhg275u3TpatWqVq92gQYMYNGgQAwYMoGfPnpw5c4agoCC8vb3p27cvffv2Zfjw4dSrV48dO3bQrFmzAl87IiKCDh068PXXX3PhwgW6du1KSEiIbf+aNWv48MMP6d27NwDx8fGcOnWqsG/JFa+hUaNGHD16lP379+fZCxQVFcXatWtzbVu3bh116tQp8IGlzZo1Y9euXdSqVavImUXEeWWlZ7D3mwWcn/MDNdf9So3UM9T41/62Zw8R3L0zzaqWp/nQJmT/90VqBxZufKQzUAHkJLy8vBg3bhxPP/00Hh4etG/fnpMnT7Jr1658b4uNHTuWF198kZo1a9KkSROmT59OTEyM7dbNlClTCAsLo0mTJri4uPD9998TGhpKYGAgM2bMICcnh9atW+Pj48N///tfvL29qVq16lXlvfvuu5k4cSKZmZlMmTIl175atWrx3//+lxYtWpCcnMzYsWOvqocrLwVdQ8eOHenQoQO33XYb7777LrVq1WLv3r1YLBZ69uzJk08+ScuWLXnllVcYNGgQ69evZ9q0aXz44YcFvua4ceNo06YNw4cP58EHH8TX15c9e/awfPly3n///SJdh4iUbdk5VtYcOMXP20+Q9MtSPp85zrbvvLsnsfWacaHjjYTc1ocnO7TEYvIAY4dgyGWSkpIMwEhKSrps34ULF4zdu3cbFy5cMCHZtcnJyTFeffVVo2rVqoa7u7tRpUoV4/XXXzcOHTpkAMa2bdsua//SSy8ZlStXNtzd3Y3GjRsbixcvtu3/9NNPjSZNmhi+vr6Gv7+/0aVLF2Pr1q2GYRjGvHnzjNatWxv+/v6Gr6+v0aZNGyM6Ovqqs549e9bw9PQ0fHx8jJSUlFz7tm7darRo0cLw9PQ0ateubXz//fdG1apVjSlTptjaAMa8efOu+DoFXYNhGMbp06eNe++91wgODja8vLyMBg0aGD///LNt/w8//GBERUXZ3s+33nor1/kvzfWPjRs3Gt26dTPKlStn+Pr6Go0aNTJee+21fHM68udORIru8NrNxrrbHzDe6/mgUXXcz0bVcT8bNcYuMA5UrGps6DbA2P75bCM9Nc3smHajoO/vS+lXYHko7K/AREqaPnciziMnK5sdn3yD+7T3qL9vCwCJvuXp/eTX3NQskt4Nw2hRtTyururluZR+BSYiIuJg0pNTiXnxbSL++xlNTh8HIMfiwo5G7TCGDWPdI13w8LK/n547KpWPUqrWrFlDuXLl8l2K2+uvv57va/Xq1avYX09EpLDSs3L4cu0hfu52F22mvkTE6eMke5Vj/YD7Obl9D01i1tB09P0qfoqZeoCkVLVo0YKYmJhSe71HHnmEgQMH5rmvqAOnRUSKQ3ZGJnPX7uedDQkkpmRQs1Fv2sZu5uh9j9LwucdpWz7A7IhlmgogKVXe3t6l+vPvoKAggoKCSu31RESuxs5ZP+I77ilcK9Qg8aYnqBzozf239KbiBw9Q2c7nlSsr9C6LiIiUkoSdsZwY+hBNt64EIDD5NK93DGdAt8Z4uGlUSmnSuy0iIlLCDKuVjePfoFzzJjTdupJsiwt/3HQXLvv2cVevpip+TKAeIBERkRJ0fOcBzgy8k1Z7NgOwt3oDvKZ/QeuOra5wpJQklZwiIiIl5JcdJxg4cxshRw5wwc2TDSOeo/a+bVRT8WM69QCJiIgUs4yMTF5fsp+Z64+Aqw/THnyZB+7sQJvWjc2OJv+jHiApNsOGDaN///4FtunUqROjR48ulTwiImY48edejtRpwqkvvwLgkY41mfDOcCJV/NgVFUBOJK8C5YcffsDLy4s333yTiRMnYrFYLluio6PNCSwi4mD2zl+GR7u21Inbw7OrZzD97sY806sebnpshd3R34gT+/zzz7n77ruZNm0aTz/9NAD169fnxIkTuZYOHTqYnFRExP5tnvwR1QfcTHDaOQ5Wro3LqpV0bhhhdizJhwqg4pSWlv+Snn71bS9cuLq21+DNN99kxIgRfPPNNzzwwAO27W5uboSGhuZaPDwuTr++Y8cObrzxRry9vQkODuahhx4iNTW1gLcjjSFDhlCuXDnCwsJ45513rimziIi9Wj9yAi2eeQzPnCximtxA6PaNhDWqZ3YsKYAKoOJUrlz+y2235W5bqVL+bS99RlW1anm3K6JnnnmGV155hZ9//pnbLs2Vj/Pnz9OzZ0/Kly/Ppk2b+P7774mOjmbEiBH5HjN27FhWrFjBvHnzWLZsGStXrmTLli1Fzi0iYm8Mq5X1g0fSdtqrAGzocw8NN/6Gb3CgucHkivQrMCezePFiFixYwK+//sqNN9542f4dO3bkeihpVFQUGzdu5Ouvv+bChQvMmjULX19fAKZNm0afPn2YPHkyISEhuc6TmprKF198waxZs+jWrRsAM2fOJCJC3cEiUjYYhsHLP++myp4jtAXW3/cEbT57G4uL+hYcgQqg4lTA7SBcXXOvJybm3/bSfzyHDxc50qUaNWrEqVOneOGFF2jZsiV+fn659tetW5eFCxfa1j09PQHYs2cPjRs3thU/AO3bt8dqtbJv377LCqCDBw+SmZlJ27ZtbduCgoKoW7dusV2LiIhZDMPg1UV7mL7uCJYuDxIx+Ha6jRpsdiwpBBVAxelfxYFpba+gcuXKzJ07l86dO9OzZ0+WLFmSqwjy8PDI82GlhmFgsVjyPGde2w3DKLbMIiL2xLBa+enZKczMqQWubky6rTHdWlUxO5YUkvrpnFCVKlVYtWoViYmJdO/eneTk5CseExUVRUxMDGn/Gnz9+++/4+LiQp06dS5rX6tWLdzd3dmwYYNt29mzZ9m/f3/xXISIiEn+eGgsfSc/xQcL3uDlPtdxh4ofh2RqAbR69Wr69OlDeHg4FouF+fPn59qf15w0FouFt956K99zzpgxI89j0i/9FZaTi4iIYOXKlZw+fZru3buTlJRUYPu7774bLy8vhg4dys6dO1mxYgUjR45k8ODBl93+AihXrhz3338/Y8eO5ddff2Xnzp0MGzYMF90bFxEHtumlKbT54l0AArt1Zkj7GiYnkqIy9dsoLS2Nxo0bM23atDz3XzofzZdffonFYrniL5f8/f0vO9bLy6skLsGhVa5cmVWrVnHu3Dm6devGuXPn8m3r4+PD0qVLOXPmDC1btmTAgAF06dIl3787gLfeeosOHTrQt29funbtyvXXX0/z5s1L4EpERErejhnf0+TlsQCsv/0BWn/wusmJ5FpYDDsZrGGxWJg3b16Bj1Lo378/KSkp/Prrr/m2mTFjBqNHjy7wy/xKkpOTCQgIICkpCX9//1z70tPTOXToENWrV1dRJaVGnzsRcx1asZ6KPbtSLvM8m9v1pNmqn3Fxc73ygVKqCvr+vpTD3I/4+++/WbRoEffff/8V26amplK1alUiIiK4+eab2bZtW4HtMzIySE5OzrWIiIgAJB1NwHPAbZTLPM+u2k1puGyuip8ywGEKoJkzZ+Ln58ett95aYLt69eoxY8YMFi5cyLfffouXlxft27cnNjY232MmTZpEQECAbYmMjCzu+CIi4oByrAbT/vMjASlnOR4URsRvv+Dp62N2LCkGDlMAffnll7aBuAVp06YN99xzD40bN+aGG27gu+++o06dOrz//vv5HjN+/HiSkpJsS3x8fHHHFxERB/Tm0r185lqVO++dQvp3PxAQEWp2JCkmDjEP0Jo1a9i3bx9z5swp9LEuLi60bNmywB4gT09P24R/V8tOhk6Jk9DnTaT0Ld+VwCer/gLgoeF9qdEo3OREUpwcogfoiy++oHnz5jRu3LjQxxqGQUxMDGFhYcWSxd3dHbj4bCyR0vLP5+2fz5+IlKy/d8dSpWt7Wsft4P7rq3Ozip8yx9QeoNTUVA4cOGBbP3ToEDExMQQFBVGlysWJpZKTk/n+++/zfZL4kCFDqFy5MpMmTQLgpZdeok2bNtSuXZvk5GTee+89YmJi+OCDD4ols6urK4GBgST+71EWPj4++c6QLHKtDMPg/PnzJCYmEhgYiOulj1QRkWKXk5XNqf6DqJ/wFy+vm0X1mWPNjiQlwNQCaPPmzXTu3Nm2PmbMGACGDh3KjBkzAJg9ezaGYXDnnXfmeY64uLhck+udO3eOhx56iISEBAICAmjatCmrV6+mVatWxZY7NPTiPeDEgp7nJVKMAgMDbZ87ESlZGx8aS9vYbaR5eOMz51s83PUfHmWR3cwDZE+udh6BnJwcsrKySjGZOCN3d3f1/IiUkgNLV1O19424W3PY9PJUWk4YZXYkKYTCzAPkEIOg7ZWrq6u+mEREyojM8+m43Hcf7tYctrbqouKnjHOIQdAiIiIlbcvDT1Hj+EHO+gRQdfZ0s+NICVMBJCIiTm/n0XOc3bwdgEMTJxNcXRPilnW6BSYiIk4tx2rw7Pyd/Nl3HI8PGMKYsQ+aHUlKgQogERFxat/8cYQ/jybh5+3OPU8ONjuOlBLdAhMREad16mAc7o+PJOh8EmN71KWSX8GPW5KyQz1AIiLitA7d+xh3bF5E3ZQEGk3Ne745KZvUAyQiIk5pz/e/0HLNIqxY8H3rDVxdNKu/M1EBJCIiTseanYPbU08CsLnrrdTp08XkRFLaVACJiIjT2Tr5Q2rH7SXNw5saH79rdhwxgQogERFxKunJqUS89QoAOwY/SoWaVUxOJGZQASQiIk5l69iXCU06SUJARZq8+5LZccQk+hWYiIg4jTNpmYz1b8n9zfvScEAPQv3LmR1JTKICSEREnMYnqw5yzNWHuUOeYtiI682OIybSLTAREXEKiWdSmLn+MABPdq+Di3727tRUAImIiFP4a+ijfP7f8dxiOUnnupXMjiMm0y0wEREp8xJ2xtJs8Rw8crIJvM4fi0W9P85OPUAiIlLmHXnyOTxystlVpxn17+lvdhyxAyqARESkTDu59yBNf50HgOXll7G46KtPVACJiEgZd3D8K3jkZLOnZiOiBt1kdhyxEyqARESkzDoXd4JGi+YAkDnuGZPTiD1RASQiImXWzpffwScrnYOVa9Po/kFmxxE7ol+BiYhImZSakc2osM7c2CuLAX1aUVNjf+Rf9GkQEZEyac6meE5nweYbb6HFw3eZHUfsjAogEREpc3Kyc5i19gAAD9xQHVfN+iyXUAEkIiJlzp/TZjBz8lCG7FvBrU0jzI4jdkgFkIiIlDle096n2rkT9PJKxdvD1ew4YodUAImISJlyYNFKrju4nUwXN2pNfNrsOGKnVACJiEiZcm7SmwBsb9edinVrmJxG7JUKIBERKTNO7T9E4/XLAAgcr94fyZ8KIBERKTNiJ72HuzWHvTUaUrt3R7PjiB1TASQiImVCTlY21ed9DUDqvQ+YnEbsnQogEREpE37bf4pRvccwv2l3Goy63+w4YudUAImISJnw9cY4/qjSkN2vTsXLz9fsOGLnVACJiIjDiz9znlX7TwJwV6sqJqcRR2BqAbR69Wr69OlDeHg4FouF+fPn59o/bNgwLBZLrqVNmzZXPO/cuXOJiorC09OTqKgo5s2bV0JXICIi9uCvJ59nwvJPuS0gnWoV1PsjV2ZqAZSWlkbjxo2ZNm1avm169uzJiRMnbMsvv/xS4DnXr1/PoEGDGDx4MNu3b2fw4MEMHDiQP/74o7jji4iIHchKz6DB99O5b8tC7vE+Z3YccRAWwzAMs0MAWCwW5s2bR//+/W3bhg0bxrlz5y7rGSrIoEGDSE5OZvHixbZtPXv2pHz58nz77bdXdY7k5GQCAgJISkrC39//ql9bRERKX8wH/6XJiCGc9g3E/1QC7l6eZkcSkxTm+9vuxwCtXLmSSpUqUadOHR588EESExMLbL9+/Xq6d++ea1uPHj1Yt25dvsdkZGSQnJycaxEREcdgnTEdgNhu/VT8yFWz6wKoV69efP311/z222+88847bNq0iRtvvJGMjIx8j0lISCAkJCTXtpCQEBISEvI9ZtKkSQQEBNiWyMjIYrsGEREpOWePHKfB1jUAhDz+sMlpxJHYdQE0aNAgbrrpJho0aECfPn1YvHgx+/fvZ9GiRQUeZ7FYcq0bhnHZtn8bP348SUlJtiU+Pr5Y8ouISMnaN+UTPKzZxEbWpXrntmbHEQfiZnaAwggLC6Nq1arExsbm2yY0NPSy3p7ExMTLeoX+zdPTE09PdZuKiDiaij9cHNt5ZsCdJicRR2PXPUCXOn36NPHx8YSFheXbpm3btixfvjzXtmXLltGuXbuSjiciIqVo9+GT/BZen+P+Fak7+iGz44iDMbUHKDU1lQMHDtjWDx06RExMDEFBQQQFBTFx4kRuu+02wsLCOHz4MM8++ywVKlTglltusR0zZMgQKleuzKRJkwAYNWoUHTp0YPLkyfTr148FCxYQHR3N2rVrS/36RESk5Hy/I5HpNz5AzMhn+aBK/v9hLJIXUwugzZs307lzZ9v6mDFjABg6dCgfffQRO3bsYNasWZw7d46wsDA6d+7MnDlz8PPzsx0TFxeHi8v/d2S1a9eO2bNn8/zzzzNhwgRq1qzJnDlzaN26deldmIiIlKjMbCsLYo4DMKBlVZPTiCOym3mA7InmARIRsW/rZy9h2vwtxNZvybpnu+Lm6lAjOqSEFOb726EGQYuIiAB4Tnmbrzf+yuo7H8PNtfuVDxC5hEpmERFxKKmnzhL1v7l/wocMNDmNOCoVQCIi4lD2fvxfvLIzia8YSc3uN5gdRxyUCiAREXEo7j98B8DRHn2xuOhrTIpGnxwREXEYSfEniNqxAYDwh4eZG0YcmgogERFxGPs+moW7NYeD4bWoen0Ls+OIA1MBJCIiDiPztxUAJN7U39wg4vD0M3gREXEIicnpDO40ggZ1e/LxE33NjiMOTj1AIiLiEBbtOIGBBbfWLal8XQ2z44iDUwEkIiIOYdHWOAD6Ng43OYmUBboFJiIidi9hx34+f/pmltRtx43PzDM7jpQB6gESERG7d/jzrwhMT6XR+UQqBfqYHUfKABVAIiJi9/wX/wRAcs+bTU4iZYUKIBERsWunD8VTL3Y7AFUfuNvkNFJWqAASERG7dvDzb3DBIDayLmGN6pkdR8oIFUAiImLXPH9aAMDp7jeZnETKEhVAIiJit5L/PsV1uzYBEHbvXSankbJEP4MXERG7tXZPArFtBtAiKZ727ZubHUfKEBVAIiJit34+lsEvN9zDiM61aG92GClTdAtMRETsUnpWDiv2ngSgZ4NQk9NIWaMCSERE7FLMj8u5Yddaqvu6UD/c3+w4UsboFpiIiNgl148+5NNVP7HeOhSLpZfZcaSMUQ+QiIjYnZysbGptWg2A/6BbTU4jZZEKIBERsTuxP0VT/nwSyV7lqHtrT7PjSBmkAkhEROzOuTk/ArC/2fW4eXqYnEbKIhVAIiJid0JWR1/8w82a/VlKhgogERGxK8dj9lA94RDZFhdqDx5gdhwpo1QAiYiIXflr7i8A7K/ViIAIzf8jJUMFkIiI2JVPq99Ax4c+Zc+TL5odRcowzQMkIiJ2Iy0jmw0HT5NZPpzGt3Q0O46UYeoBEhERu7Em9hSZOVaqBvtQs6Kv2XGkDFMPkIiI2I1yT4/h44OHOfbASCwWi9lxpAxTASQiInbBmp1DvTVLqJB6lp0hT5kdR8o43QITERG7cGDxSiqkniXF04c6A3qbHUfKOBVAIiJiF07/sBCA2EZt8PDxMjmNlHUqgERExC6UX/MbANnde5icRJyBqQXQ6tWr6dOnD+Hh4VgsFubPn2/bl5WVxbhx42jYsCG+vr6Eh4czZMgQjh8/XuA5Z8yYgcViuWxJT08v4asREZGiSjqaQO3DuwGoetctJqcRZ2BqAZSWlkbjxo2ZNm3aZfvOnz/P1q1bmTBhAlu3buXHH39k//799O3b94rn9ff358SJE7kWLy91p4qI2KsD38zD1bByOLQ6IVG1zY4jTsDUX4H16tWLXr165bkvICCA5cuX59r2/vvv06pVK+Li4qhSpUq+57VYLISGXv306RkZGWRkZNjWk5OTr/pYERG5dttPZ2GtHEV2m7ZUMzuMOAWHGgOUlJSExWIhMDCwwHapqalUrVqViIgIbr75ZrZt21Zg+0mTJhEQEGBbIiMjizG1iIgUxDAMPvaP4vZ73sQ6aZLZccRJOEwBlJ6ezjPPPMNdd92Fv79/vu3q1avHjBkzWLhwId9++y1eXl60b9+e2NjYfI8ZP348SUlJtiU+Pr4kLkFERPKwNyGFxJQMvN1daVE9yOw44iQcYiLErKws7rjjDqxWKx9++GGBbdu0aUObNm1s6+3bt6dZs2a8//77vPfee3ke4+npiaenZ7FmFhGRqxMT/Qf+6am0qFcDTzdXs+OIk7D7HqCsrCwGDhzIoUOHWL58eYG9P3lxcXGhZcuWBfYAiYiIeRq9+gzb3ruLIfF/mB1FnIhdF0D/FD+xsbFER0cTHBxc6HMYhkFMTAxhYWElkFBERK5F6qmz1D7wJ66GlVo9OpgdR5yIqbfAUlNTOXDggG390KFDxMTEEBQURHh4OAMGDGDr1q38/PPP5OTkkJCQAEBQUBAeHh4ADBkyhMqVKzPpfwPnXnrpJdq0aUPt2rVJTk7mvffeIyYmhg8++KD0L1BERAp0YPZPNLFmczQ4nIhWjcyOI07E1AJo8+bNdO7c2bY+ZswYAIYOHcrEiRNZuPDitOhNmjTJddyKFSvo1KkTAHFxcbi4/H9H1rlz53jooYdISEggICCApk2bsnr1alq1alWyFyMiIoWWsWgRAMdadyDC5CziXCyGYRhmh7A3ycnJBAQEkJSUVOgxRyIicvWOB4cTfuYEMR/Oosmjg82OIw6uMN/fdj0GSEREyq7jm3cSfuYEmS5u1B7Yx+w44mRUAImIiCni//f09wM1G+AbHGhuGHE6DjEPkIiIlD3zKzflx54j6dC6DlFmhxGnowJIRERKXY7VYPFZV8417sHAe9uZHUeckG6BiYhIqdt1PIlz57Pw83SjcUSA2XHECakAEhGRUnf8i68YtnkhN/ln4OaqryIpfboFJiIipS5i9gwm7t3CH43Cgb5mxxEnpLJbRERKVXpSCnVitwMQetvNJqcRZ6UCSERESlXsvCV45GST6F+BKm2bmh1HnJQKIBERKVWpi5YCcKRZOywu+hoSc+iTJyIiparihjUAWLp2NTmJODMVQCIiUmrOHjlOjaOxAFQbpMdfiHlUAImISKnZu2wt2S6uHAqrQYVa1cyOI05MBZCIiJSahcH1aDLqWxZP+I/ZUcTJaR4gEREpNWsPnOK8hzfXdW5pdhRxcuoBEhGRUhF3+jzxZy7g7mqhVfUgs+OIk1MBJCIipeLEO++zcOZonjqyGl9P3YAQc+kTKCIipcJtxW80SjhAmut5s6OIqAdIRERKnmG1UnXnJgACenUzOY2ICiARESkFcWu3UCH1LOluHtTs08XsOCIqgEREpOQlLFwMwIHajfD09TE5jYgKIBERKQXuq1cBkNL2BpOTiFykAkhEREqUNTuH6rs2A1D+pu4mpxG5SAWQiIiUqP0HjrOmahOOBIVTs3dns+OIAPoZvIiIlLDfT2XzSt+n6VCnIrO8PM2OIwKoB0hERErY+oOnAWhbI9jkJCL/r8gFUHZ2NtHR0XzyySekpKQAcPz4cVJTU4stnIiIOLacrGxO/bEFDIO2NVUAif0o0i2wI0eO0LNnT+Li4sjIyKBbt274+fnx5ptvkp6ezscff1zcOUVExAH9Ff078z96hMNBlYl47YjZcURsitQDNGrUKFq0aMHZs2fx9va2bb/lllv49ddfiy2ciIg4tlM/LwEgKbIabm6uJqcR+X9F6gFau3Ytv//+Ox4eHrm2V61alWPHjhVLMBERcXzea9cAcKF9R5OTiORWpB4gq9VKTk7OZduPHj2Kn5/fNYcSERHHl52RSa292wCo0LeHyWlEcitSAdStWzemTp1qW7dYLKSmpvLiiy/Su3fv4somIiIO7OCSVZTLPE+SVzmqd2lvdhyRXIp0C2zKlCl07tyZqKgo0tPTueuuu4iNjaVChQp8++23xZ1RREQc0JlFywD4K6oFTTX+R+xMkQqg8PBwYmJimD17Nlu2bMFqtXL//fdz99135xoULSIizst33cXxPxk3dDA5icjlijwTtLe3N/feey/33ntvceYREZEyIDPbyrtN+tOmXATdbu9ndhyRyxRpDNDMmTNZtGiRbf3pp58mMDCQdu3aceTI1c/zsHr1avr06UN4eDgWi4X58+fn2m8YBhMnTiQ8PBxvb286derErl27rnjeuXPnEhUVhaenJ1FRUcybN++qM4mIyLXbfvQcKyMa8ulND1O9bVOz44hcpkgF0Ouvv2671bV+/XqmTZvGm2++SYUKFXjiiSeu+jxpaWk0btyYadOm5bn/zTff5N1332XatGls2rSJ0NBQunXrZpt5Oi/r169n0KBBDB48mO3btzN48GAGDhzIH3/8UbiLFBGRIvvn8RdtagTh4mIxOY3I5SyGYRiFPcjHx4e9e/dSpUoVxo0bx4kTJ5g1axa7du2iU6dOnDx5svBBLBbmzZtH//79gYu9P+Hh4YwePZpx48YBkJGRQUhICJMnT+bhhx/O8zyDBg0iOTmZxYsX27b17NmT8uXLX/UA7eTkZAICAkhKSsLf37/Q1yIi4uy+uPtpVmeVo8djA7mrUz2z44iTKMz3d5F6gMqVK8fp0xer+2XLltG1a1cAvLy8uHDhQlFOeZlDhw6RkJBA9+7dbds8PT3p2LEj69aty/e49evX5zoGoEePHgUek5GRQXJycq5FRESKJj0ljXvmTGXm9y/S3k3PhxT7VOR5gB544AEeeOAB9u/fz0033QTArl27qFq1arEES0hIACAkJCTX9pCQENu+/I4r7DGTJk0iICDAtkRGRl5DchER53ZwYTSeOVmcKhdElXbNzI4jkqciFUAffPABbdu25eTJk8ydO5fg4ItP+N2yZQt33XVXsQa0WHLfOzYM47Jt13rM+PHjSUpKsi3x8fFFDywi4uSSlywH4EjDllhcivQ1I1LiivQz+MDAQN5++23+/PNPEhMTWbhwIQDNmzcvtmChoaHAxR6dsLAw2/bExMTLenguPe7S3p4rHePp6Ymnp+c1JhYREYCAP34HILtjJ3ODiBSgSAXQkiVLGDJkCKdPn+bSMdQWiyXP54QVVvXq1QkNDWX58uU0bXrxJ5SZmZmsWrWKyZMn53tc27ZtWb58ea5foy1btox27dpdcyYRESnYhaRUav21E4Dw/r1MTiOSvyL1TY4YMYLbb7+d48ePY7Vacy2FKX5SU1OJiYkhJiYGuDjwOSYmhri4OCwWC6NHj+b1119n3rx57Ny5k2HDhuHj45PrNtuQIUMYP368bX3UqFEsW7aMyZMns3fvXiZPnkx0dDSjR48uyqWKiEghHPwpGo+cbBL9golo2dDsOCL5KlIPUGJiImPGjCnwttLV2Lx5M507d7atjxkzBoChQ4cyY8YMnn76aS5cuMBjjz3G2bNnad26NcuWLcv1xPm4uDhc/nWPuV27dsyePZvnn3+eCRMmULNmTebMmUPr1q2vKauIiFzZuehVAMQ1akUljf8RO1akeYDuu+8+2rdvz/33318SmUyneYBERIpm4EfrOLNlO090rc1Nd3Q1O444mcJ8fxepB2jatGncfvvtrFmzhoYNG+Lu7p5r/+OPP16U04qIiAO7kJlDzNEkMitUIaprW7PjiBSoSAXQN998w9KlS/H29mblypW5fmJusVhUAImIOKFtcWfJzLES4u9JtWAfs+OIFKhIBdDzzz/Pyy+/zDPPPJNr/I2IiDivjLff5b3VvxN/651YLLr9JfatSNVLZmYmgwYNUvEjIiI2Ib8tpu+e1bS0JpkdReSKilTBDB06lDlz5hR3FhERcVD/nv8nrF9Pk9OIXFmRboHl5OTw5ptvsnTpUho1anTZIOh33323WMKJiIhjOPhTNA00/484kCIVQDt27LDNzrxz585c+670nC4RESl7Uv73/C/N/yOOokgF0IoVK4o7h4iIOLCAjesAyOnY0eQkIldHZbqIiFyTXM//6qfnf4ljKFIPkIiIyD92b9mLW8XqVEpPIqJFA7PjiFwVFUAiInJNVmX78d7QKdxavwLvavyPOAh9UkVE5Jps+OsMAC3rhpmcROTqqQASEZEiu5CWzr6DCQC0qRFschqRq6cCSEREiuzgvCVsfvd2vvx5sp7/JQ5FBZCIiBRZypLluFtz8A8sp3ngxKGoABIRkSLT/D/iqFQAiYhIkWj+H3FkKoBERKRIDv4UjUdONon+FTT/jzgcFUAiIlIktud/NWyJRfP/iIPRJ1ZERIrkn/E/Vo3/EQekmaBFRKTQLmTm8G3N62ln8aFh/95mxxEpNBVAIiJSaNvizvLfxj1ZfkN/1mv8jzgg3QITEZFC2/DXaQDa1AjS/D/ikFQAiYhIoWXNX0iVsydoUz3I7CgiRaJbYCIiUigXzqXwxKfPMi4nm6MP/Gl2HJEiUQ+QiIgUyr/n/6ncvL7ZcUSKRAWQiIgUSuqSaEDz/4hj0ydXREQKxX/j74Dm/xHHpgJIRESu2oVzKdQ6tAvQ87/EsakAEhGRq5Zr/I/m/xEHpgJIRESumsb/SFmhn8GLiMhV+6RZX75ID2Jg72ZmRxG5JirfRUTkqpzPzGbtmRyW125D7X7dzY4jck1UAImIyFXZfPgsWTkGlQO9qRrsY3YckWuiAkhERK5K2rQPeWLNV/T3PKfnf4nDs/sCqFq1algslsuW4cOH59l+5cqVebbfu3dvKScXESlbai2Yzah1s+mcdtTsKCLXzO4HQW/atImcnBzb+s6dO+nWrRu33357gcft27cPf39/23rFihVLLKOISFmXdCyRGvH7Aahy200mpxG5dnZfAF1auLzxxhvUrFmTjleYgbRSpUoEBgaWYDIREefx14+/0NSwElcxkirX1TQ7jsg1s/tbYP+WmZnJV199xX333XfF+89NmzYlLCyMLl26sGLFigLbZmRkkJycnGsREZH/l7Hs4vw/x5u3MzmJSPFwqAJo/vz5nDt3jmHDhuXbJiwsjE8//ZS5c+fy448/UrduXbp06cLq1avzPWbSpEkEBATYlsjIyBJILyLiuEI2rwPAo1tXk5OIFA+LYRiG2SGuVo8ePfDw8OCnn34q1HF9+vTBYrGwcOHCPPdnZGSQkZFhW09OTiYyMpKkpKRc44hERJzRqYNxVKhVFYBzR44TWCXM5EQieUtOTiYgIOCqvr/tfgzQP44cOUJ0dDQ//vhjoY9t06YNX331Vb77PT098fT0vJZ4IiJl1t7VW2jo6cupCuHUVPEjZYTD3AKbPn06lSpV4qabCv/rg23bthEWpn+0IiJF8bN/TZo+/g0/vfap2VFEio1D9ABZrVamT5/O0KFDcXPLHXn8+PEcO3aMWbNmATB16lSqVatG/fr1bYOm586dy9y5c82ILiLi8NYdPI3VxZVGra8zO4pIsXGIAig6Opq4uDjuu+++y/adOHGCuLg423pmZiZPPfUUx44dw9vbm/r167No0SJ69+5dmpFFRMqE+JMpxJ1Ow9XVhVbVg82OI1JsHGoQdGkpzCAqEZGybOML7xD+n8ms7HQr9yz42Ow4IgUqzPe3w4wBEhGR0ufy269EJJ+kpo+e/SVliwogERHJk2G1UvXPPwDwu6mHyWlEipcKIBERyVPc+m1UTDlDhqs7tfp1MzuOSLFSASQiInlKmPszALG1G+Hl52tyGpHipQJIRETy5LniVwBSbuhschKR4qcCSERELpOVnkGt3ZsBqHjrzSanESl+DjEPkIiIlK4/9x1ne8NuND15kMZdrzc7jkixUwEkIiKXWfV3Ju91fYibG4Uxzc3V7DgixU63wERE5DKrY08B0KF2RZOTiJQMFUAiIpJLUsJpfH5fjUd2FtfXrmB2HJESoVtgIiKSy1+z5/PNt88SW7k24W/3NzuOSIlQD5CIiOSSuWQpAKebtjI5iUjJUQEkIiK5RGxaC4BXbz3+QsouFUAiImJzbPMOKp85QZaLK7Vu1/w/UnapABIREZuj3y0EILZmQ8pVKG9yGpGSowJIRERs3P/3+Iuk9p3MDSJSwlQAiYgIANnpGdTasRGAoP69TU4jUrL0M3gREQFg6/FUXrxrEt2P/snjvfUAVCnbVACJiAgAK/afZE+lGtTpfj2u7vp6kLJNt8BERASAFXsTAehct5LJSURKngogEREhcc9BHvp8IjfvXU2HOnr+l5R96uMUEREOf/UDt+5aQf20vwnynWx2HJESpx4gERHBfekSAM506GpyEpHSoQJIRMTJZZ5Pp86OPwCoMLC/uWFESokKIBERJ7f/xyX4Zl7gtG8gNXt0MDuOSKlQASQi4uRS5118/MVfza/Hxc3V5DQipUMFkIiIkwv9fQUALjdp9mdxHiqARESc2LH4RM4bFrItLtS6+zaz44iUGv0MXkTEia04doHn732fzsEWplfWBIjiPNQDJCLixH7d8zcALVrUMTmJSOlSASQi4qRSk9OI2RUPQPeoEJPTiJQuFUAiIk5q/2ffsGHqHfxn1SfUqlTO7DgipUoFkIiIk8qZPx/PnGwqhVfAYrGYHUekVKkAEhFxQlnpGdTdvBqAwDsGmJxGpPSpABIRcUL7vv8F//RUzvgGUKd/d7PjiJQ6uy6AJk6ciMViybWEhoYWeMyqVato3rw5Xl5e1KhRg48//riU0oqIOI60OT8AcKBVZ1zdNSOKOB+7/9TXr1+f6Oho27qra/7TtB86dIjevXvz4IMP8tVXX/H777/z2GOPUbFiRW67TRN8iYgAGFYrVX+/+L+rHrfeYnIaEXPYfQHk5uZ2xV6ff3z88cdUqVKFqVOnAnDdddexefNm3n77bRVAIiL/81f0WmqeS+SCmyd171EBJM7Jrm+BAcTGxhIeHk716tW54447+Ouvv/Jtu379erp3z30vu0ePHmzevJmsrKx8j8vIyCA5OTnXIiJSVi1P8mBil4eIvmkw3oF+ZscRMYVdF0CtW7dm1qxZLF26lM8++4yEhATatWvH6dOn82yfkJBASEjuybxCQkLIzs7m1KlT+b7OpEmTCAgIsC2RkZHFeh0iIvZk4fEsZrToS/rzL5gdRcQ0dl0A9erVi9tuu42GDRvStWtXFi1aBMDMmTPzPebSuSwMw8hz+7+NHz+epKQk2xIfH18M6UVE7E/8mfPsPpGMiwW6XKfZn8V52f0YoH/z9fWlYcOGxMbG5rk/NDSUhISEXNsSExNxc3MjODg43/N6enri6elZrFlFROzR3g+mc0fMXk53702Qr4fZcURMY9c9QJfKyMhgz549hIWF5bm/bdu2LF++PNe2ZcuW0aJFC9zd3UsjooiIXav6+Qe8sXQaDyRsMTuKiKnsugB66qmnWLVqFYcOHeKPP/5gwIABJCcnM3ToUODirashQ4bY2j/yyCMcOXKEMWPGsGfPHr788ku++OILnnrqKbMuQUTEbhyP2UOduD3kWFyo+fCQKx8gUobZ9S2wo0ePcuedd3Lq1CkqVqxImzZt2LBhA1WrVgXgxIkTxMXF2dpXr16dX375hSeeeIIPPviA8PBw3nvvPf0EXkQEOPLJTMKBPXWa0qBmFbPjiJjKYvwzSlhskpOTCQgIICkpCX9/f7PjiIgUi9iq11E7bi9/jH2V1m8+Z3YckWJXmO9vu74FJiIixePYll3UjttLjsWFWo/q9peICiAREScQ9+nF6UP21GtGcHXNdSaiAkhExAkk7jtEjsWFtL569IUI2PkgaBERuXYHElMY1WYYkxr145fHe5gdR8QuqAdIRKSMm7ftGAD1m9UlKLyiyWlE7IMKIBGRMsyancPvK7cDcEuzyianEbEfugUmIlKG7f3hF358405W1G1L+1dWmx1HxG6oB0hEpAxL/WImLhj4hVfCy93V7DgidkMFkIhIGZWekka9NUsA8LlvqMlpROyLCiARkTJq92ff4p+Rxt8BFYm6o4/ZcUTsigogEZEyynXWDAD+6t4PFzfd/hL5NxVAIiJl0N+7Y2nw53oAIsY8ZnIaEfujAkhEpAw6+PZHuBpWdtduQmSbpmbHEbE7+hm8iEgZY7UaTKjWhWa9rPS/uaXZcUTsknqARETKmLUHTnHwPCxt1YvmD95hdhwRu6QCSESkjJm9KQ6AW5pW1tw/IvlQASQiUoacPnSU4U8O4r5NC7ijRYTZcUTslsYAiYiUIbGvTaHN3wdx9/KgTuVAs+OI2C31AImIlBFZ6RnU+H4WAEn3P2xyGhH7pgJIRKSM+PO9L6mUfIpT5crTaMyDZscRsWsqgEREygjfTz4CILb/3Xj6+picRsS+qQASESkDYhetoN5fO8h0caP2i0+aHUfE7qkAEhEpA85NfgeAP9v1oEKtauaGEXEA+hWYiIiDO5F0gSlVOzDk7zPUGDfG7DgiDkEFkIiIg/ts9SHWVa5P9pgb+O7mtmbHEXEIugUmIuLATqdm8O3GizM/j+hcy+Q0Io5DBZCIiAPbPepZnlryMR19M7mhdgWz44g4DN0CExFxUCmJp2n8zafckJ5Ks7v7YLFYzI4k4jDUAyQi4qB2TpiMf3oqRypVofGIYWbHEXEoKoBERBxQ8t+nqPfVJwAkPvYELm566rtIYagAEhFxQLvHvED588kcCalK02ceMzuOiMNRASQi4mBOHYyj0fdfAnB6/Au4eXqYnEjE8agAEhFxMLFPPI9PVgb7q15H05HDzI4j4pD0KzAREQcSf+Y8o2v05OHmSbQafS8WF/13rEhR6F+OiIgDeW3RHv72CuC3R56lwT39zY4j4rBUAImIOIj1G/awZOcJXF0sTLg5yuw4Ig7NrgugSZMm0bJlS/z8/KhUqRL9+/dn3759BR6zcuVKLBbLZcvevXtLKbWISPHLSDtPRN8ezPruBUbU9qJuqJ/ZkUQcml2PAVq1ahXDhw+nZcuWZGdn89xzz9G9e3d2796Nr69vgcfu27cPf39/23rFihVLOq6ISInZOmYibU/G430hlSa9G5kdR8Th2XUBtGTJklzr06dPp1KlSmzZsoUOHToUeGylSpUIDAy8qtfJyMggIyPDtp6cnFzorCIiJeV4zB4aT38fgENPv0jLED3zS+Ra2fUtsEslJSUBEBQUdMW2TZs2JSwsjC5durBixYoC206aNImAgADbEhkZWSx5RUSulTU7h7OD7sEnK53dtZvQ4rmRZkcSKRMshmEYZoe4GoZh0K9fP86ePcuaNWvybbdv3z5Wr15N8+bNycjI4L///S8ff/wxK1euzLfXKK8eoMjISJKSknLdRhMRKW1/jHmJ1lMmct7dk7PrNlG5RUOzI4nYreTkZAICAq7q+9thCqDhw4ezaNEi1q5dS0RERKGO7dPn4lOSFy5ceFXtC/MGioiUlKMb/yTo+lb4ZGXwx5Mv0frtF8yOJGLXCvP97RC3wEaOHMnChQtZsWJFoYsfgDZt2hAbG1sCyURESkZWjpV352/luF9FdtZtTss3njM7kkiZYtcFkGEYjBgxgh9//JHffvuN6tWrF+k827ZtIywsrJjTiYiUnDcW7+VHa0XuePgDys//Xk97Fylmdv0rsOHDh/PNN9+wYMEC/Pz8SEhIACAgIABvb28Axo8fz7Fjx5g1axYAU6dOpVq1atSvX5/MzEy++uor5s6dy9y5c027DhGRwli+bh9frD0EwGt3t6JyvVCTE4mUPXZdAH300UcAdOrUKdf26dOnM2zYMABOnDhBXFycbV9mZiZPPfUUx44dw9vbm/r167No0SJ69+5dWrFFRIosfsM2WnbtyANtbsf1ySfpUV/Fj0hJcJhB0KVJg6BFxAxnDh3lfMs2RJw+xu5ajamzcyNunh5mxxJxGGVuELSISFl34VwKJ7v0JOL0MY6XD6XS4vkqfkRKkAogERGT5WRls7dLX+oe2kWSVzmyfvqZCrWqmR1LpExTASQiYqLsjEy2depD060ryXB15/isOVRt39zsWCJlngogERGTZOVY+WL8NFqsW0KWiyu73/2E627XDzZESoNd/wpMRKSsSs/KYdTsbSz1qEtyp6F0u60zTUcMNTuWiNNQASQiUspOHYzjye//ZNU5Cx6uLjT7cDJNrgsxO5aIU9EtMBGRUvTXr+vIatGKx98fSyV3g1n3t6KLih+RUqcCSESkFBhWK388/SrhPTsTdu5vKmWm8sNttWhTI9jsaCJOSbfARERK2JlDRzk8cAitN68AYHvDtlT9eS6BVfSMQhGzqAdIRKSEWLNz2Pj8m7hGXUezzSvIdHFjw/Bnabh1jYofEZOpABIRKQFbjpxh0CfrcZv+JQHpqRwMr8WRn6JpM+01PdldxA7oFpiISDE6sHQ17x/IYEF8JgAv3TyKZ93iaf7uRD3aQsSOqAASEblGOVnZ7Pj8W9z/8x/q79tCVKtb+bnL/QxsEcHjXW4kLMDb7IgicgkVQCIiRRT/x3aOTv2Y6ovn0iTpJADZFhca+VpZPvoGalTyMzmhiORHBZCIyFWyWg3+PJbE8t0JdHjiXlrv2UDk//Yle5Vjd+/bqfbSeNo2qG1qThG5MhVAIiL5yEg7z5EVGziz9DfcN6znkV5PcDLTAkAFn4o0t7iwu34rsgYPof6jQ2jj52tyYhG5WiqARMTppWflcPTsBU5v3IaxaBFuO/4k+OBeIv8+Qh3DamtXtW4PztdoRKe6lQi9cSJp9T+nUeVK5gUXkSJTASQiZVJ2jpXUjGxSTpwkI/YAFxISST9yjJwTJ+DECdxPJuJ9MoHJPR9hdblIDAOGbPmJl6M/yXWec95+HK7TmPS27Zk4tBd1W9bH3VUziIg4OhVApSgjO4eTp1NwTThx2T7DuPj/rf7+GIHlL65kZeF6/Fi+5zP8/Mgp/79p9LOzcYuP+/99GLnaWsuVw1rx4n+pGjlW3A4fuvzFAQMwfMuRExpq2+f214HL2tnO6+NLdli4bd39wP7c5/vXIYa3D9kRkf/f9uB+yPn//7rOdV4PT7KrVvtX21gsWdlcngAMTw+yqte0vZb7wVgsmZmXtrr4f909yKz1/+Mz3P86iOXChbwzuLqRWaeebd3jr4NYzqflOp/t7C4uZEQ1uPhnAzwOH8IlNflfDXK3T2/Y5P/Pe/gQlqSkPDMAXGjQEFwufuF6HDmM69kzl1zVv9rWbwRuF/9Zu8fH4Xb65GXnM/6X5cJ1DTA8PS+2PRqPe2JCnu8vwIWoBlg9L/6Syf3YUTwS8v9cXqgXhdW3HABux4/hHheHNSsLIzsHIycbIysLI9uKNSebU01bkx5QnhyrFd/YfZT/cyvWnBzIzsbIzoacHMjJwcjMZNcNvUisEE5GVg6Vt66n2a/zcMlIxzUjHbfMTNwy0/G8kIbP+VSe6fskv1VuCMDA7ct4c8l7+eYNPHwAo34kvh6upEU1YmtyVzIaNMS7RVPCOrahUr2aNHFRwSNS1qgAKkW7jicz/qVvWPrliHzbfNR6AJM7DQOgytkTrP70wXzbzmh2MxO7PQJAxdSzbPpgcL5t5zTsxrjeowDwzTjPrqkD8237U70bGNlvHAAu1hz+eqtfvm2ja7bkgQEv2tb3vX0LnjlZebb9vWoj7r7jddv6tv/cSfn0lDzbbguryy1D3rGtr/twGOEpp/Jsu7dCVXre/4Ft/dfPHqHGmaN5tj0SGErPhz+3rf80YxQN/z6YZ9uTvoG0HPGVbf27r5+m1dHdebZN9fCmwRPf29ZnfvcCHQ9tzbNtjsWFmk8vtK1/NO91eu1fl2dbgLpP/kiG28X5Y979+R1u3bUi37ZNHv+Gc97+ALy2dBp3xyzJt23bR6dzwr8iAM/99jkPbpqfb9suD3zEweCLxeuY1f/l8fVz8m1789Cp7AytBcCjG75n3KqZ+ba9/a432BR5sXC8d/MCXvz1s3zbfpTkz+oazQEYtH0HD/2+ON+23snnoPLFP5/3DyTRL5g0X39SgyqSHlyR7JBQLKGhuEVU5oGO1/NCgzoE+XpgsViAx/I9r4iUHSqASpGLxYKHmwsX3Dzzb+Tujrf7xVliPdzdSHP3AsCSV1sPD3w8XLEA3h6upHnknmvE+NdRVk9Pynle/Ov2wY0UT598I+R4eeHndbGti9VCsle5fNtmefng7+X2vy8OSPEqR/qlBdD/YmR6+RDo427bnOLjBy55XhkXfMoR5Pv/k8al+Ppzxpp3YZXm60/w/9paLJDq68/pjMA826b6BlCh3P+//2nlAjiVFpRn23O+AVTy+/+2F/wCOemXu+0/73G6hxch/v/fNt0/kET/Cnme12qxEBbgZVvPCAgkIaBinm0Bwvy9yPK4eO7sgPIkBOY/5iQ00Adf74ufg+zAII6XD823bUiQD27+F9saQUEcC8r/0QwVg8qRFeSDxQIuwUEcDQ6/rM0/f5MVg/2oFnzx8+UWHERcxUisLq4YLi7kuLpiuLhidXHB6upG9WohuFcLxtXFgv/5umw/1R6rqxuGiwuGqxuGqyu4uGL18KB1+/rUq10DTzcXwutY2FDLH4u3FxZvH1x8vHH19sI9MADPCsGMu642r1SqgJ+XG+6uNwEv5XttIuKcLIaRx30NJ5ecnExAQABJSUn4+/ubHUdERESuQmG+v3VjW0RERJyOCiARERFxOiqARERExOmoABIRERGnowJIREREnI4KIBEREXE6KoBERETE6agAEhEREaejAkhEREScjgogERERcToqgERERMTpqAASERERp6MCSERERJyOCiARERFxOm5mB7BHhmEAkJycbHISERERuVr/fG//8z1eEBVAeUhJSQEgMjLS5CQiIiJSWCkpKQQEBBTYxmJcTZnkZKxWK8ePH8fPzw+LxVKs505OTiYyMpL4+Hj8/f2L9dz2oKxfH5T9a9T1Ob6yfo26PsdXUtdoGAYpKSmEh4fj4lLwKB/1AOXBxcWFiIiIEn0Nf3//MvvBhrJ/fVD2r1HX5/jK+jXq+hxfSVzjlXp+/qFB0CIiIuJ0VACJiIiI01EBVMo8PT158cUX8fT0NDtKiSjr1wdl/xp1fY6vrF+jrs/x2cM1ahC0iIiIOB31AImIiIjTUQEkIiIiTkcFkIiIiDgdFUAiIiLidFQAmahv375UqVIFLy8vwsLCGDx4MMePHzc7VrE4fPgw999/P9WrV8fb25uaNWvy4osvkpmZaXa0YvXaa6/Rrl07fHx8CAwMNDvONfvwww+pXr06Xl5eNG/enDVr1pgdqdisXr2aPn36EB4ejsViYf78+WZHKlaTJk2iZcuW+Pn5UalSJfr378++ffvMjlWsPvroIxo1amSbPK9t27YsXrzY7FglZtKkSVgsFkaPHm12lGIxceJELBZLriU0NNS0PCqATNS5c2e+++479u3bx9y5czl48CADBgwwO1ax2Lt3L1arlU8++YRdu3YxZcoUPv74Y5599lmzoxWrzMxMbr/9dh599FGzo1yzOXPmMHr0aJ577jm2bdvGDTfcQK9evYiLizM7WrFIS0ujcePGTJs2zewoJWLVqlUMHz6cDRs2sHz5crKzs+nevTtpaWlmRys2ERERvPHGG2zevJnNmzdz44030q9fP3bt2mV2tGK3adMmPv30Uxo1amR2lGJVv359Tpw4YVt27NhhXhhD7MaCBQsMi8ViZGZmmh2lRLz55ptG9erVzY5RIqZPn24EBASYHeOatGrVynjkkUdybatXr57xzDPPmJSo5ADGvHnzzI5RohITEw3AWLVqldlRSlT58uWNzz//3OwYxSolJcWoXbu2sXz5cqNjx47GqFGjzI5ULF588UWjcePGZsewUQ+QnThz5gxff/017dq1w93d3ew4JSIpKYmgoCCzY0geMjMz2bJlC927d8+1vXv37qxbt86kVHItkpKSAMrsv7mcnBxmz55NWloabdu2NTtOsRo+fDg33XQTXbt2NTtKsYuNjSU8PJzq1atzxx138Ndff5mWRQWQycaNG4evry/BwcHExcWxYMECsyOViIMHD/L+++/zyCOPmB1F8nDq1ClycnIICQnJtT0kJISEhASTUklRGYbBmDFjuP7662nQoIHZcYrVjh07KFeuHJ6enjzyyCPMmzePqKgos2MVm9mzZ7N161YmTZpkdpRi17p1a2bNmsXSpUv57LPPSEhIoF27dpw+fdqUPCqAilleg7wuXTZv3mxrP3bsWLZt28ayZctwdXVlyJAhGHY8OXdhrw/g+PHj9OzZk9tvv50HHnjApORXryjXWFZYLJZc64ZhXLZN7N+IESP4888/+fbbb82OUuzq1q1LTEwMGzZs4NFHH2Xo0KHs3r3b7FjFIj4+nlGjRvHVV1/h5eVldpxi16tXL2677TYaNmxI165dWbRoEQAzZ840JY+bKa9aho0YMYI77rijwDbVqlWz/blChQpUqFCBOnXqcN111xEZGcmGDRvstku3sNd3/PhxOnfuTNu2bfn0009LOF3xKOw1lgUVKlTA1dX1st6exMTEy3qFxL6NHDmShQsXsnr1aiIiIsyOU+w8PDyoVasWAC1atGDTpk385z//4ZNPPjE52bXbsmULiYmJNG/e3LYtJyeH1atXM23aNDIyMnB1dTUxYfHy9fWlYcOGxMbGmvL6KoCK2T8FTVH80/OTkZFRnJGKVWGu79ixY3Tu3JnmzZszffp0XFwco8PxWv4OHZWHhwfNmzdn+fLl3HLLLbbty5cvp1+/fiYmk6tlGAYjR45k3rx5rFy5kurVq5sdqVQYhmHX/5tZGF26dLnsV1H33nsv9erVY9y4cWWq+IGL33V79uzhhhtuMOX1VQCZZOPGjWzcuJHrr7+e8uXL89dff/HCCy9Qs2ZNu+39KYzjx4/TqVMnqlSpwttvv83Jkydt+8yc96G4xcXFcebMGeLi4sjJySEmJgaAWrVqUa5cOXPDFdKYMWMYPHgwLVq0sPXYxcXFlZlxW6mpqRw4cMC2fujQIWJiYggKCqJKlSomJisew4cP55tvvmHBggX4+fnZevMCAgLw9vY2OV3xePbZZ+nVqxeRkZGkpKQwe/ZsVq5cyZIlS8yOViz8/PwuG7P1zxjRsjCW66mnnqJPnz5UqVKFxMREXn31VZKTkxk6dKg5gcz8CZoz+/PPP43OnTsbQUFBhqenp1GtWjXjkUceMY4ePWp2tGIxffp0A8hzKUuGDh2a5zWuWLHC7GhF8sEHHxhVq1Y1PDw8jGbNmpWpn1CvWLEiz7+roUOHmh2tWOT372369OlmRys29913n+3zWbFiRaNLly7GsmXLzI5VosrSz+AHDRpkhIWFGe7u7kZ4eLhx6623Grt27TItj8Uw7HjErYiIiEgJcIxBGSIiIiLFSAWQiIiIOB0VQCIiIuJ0VACJiIiI01EBJCIiIk5HBZCIiIg4HRVAIiIi4nRUAImIiIjTUQEkIletU6dOjB492uwYeTp9+jSVKlXi8OHDAKxcuRKLxcK5c+dK9HWL+jozZswgMDCwUMe0bNmSH3/8sVDHiEjeVACJiGlOnDjBXXfdRd26dXFxccm3uJo7dy5RUVF4enoSFRXFvHnzLmszadIk+vTpQ7Vq1Uo2tIkmTJjAM888g9VqNTuKiMNTASQipsnIyKBixYo899xzNG7cOM8269evZ9CgQQwePJjt27czePBgBg4cyB9//GFrc+HCBb744gseeOCB0opuiptuuomkpCSWLl1qdhQRh6cCSESK5OzZswwZMoTy5cvj4+NDr169iI2NzdXms88+IzIyEh8fH2655RbefffdXLd9qlWrxn/+8x+GDBlCQEBAnq8zdepUunXrxvjx46lXrx7jx4+nS5cuTJ061dZm8eLFuLm50bZt23zznj59mjvvvJOIiAh8fHxo2LAh3377ba42nTp1YuTIkYwePZry5csTEhLCp59+SlpaGvfeey9+fn7UrFmTxYsXX3b+33//ncaNG+Pl5UXr1q3ZsWNHrv0zZsygSpUqtvfi9OnTufYfPHiQfv36ERISQrly5WjZsiXR0dG52ri6utK7d+/LcotI4akAEpEiGTZsGJs3b2bhwoWsX78ewzDo3bs3WVlZwMWC4JFHHmHUqFHExMTQrVs3XnvttUK/zvr16+nevXuubT169GDdunW29dWrV9OiRYsCz5Oenk7z5s35+eef2blzJw899BCDBw/O1ZMEMHPmTCpUqMDGjRsZOXIkjz76KLfffjvt2rVj69at9OjRg8GDB3P+/Plcx40dO5a3336bTZs2UalSJfr27Wt7L/744w/uu+8+HnvsMWJiYujcuTOvvvpqruNTU1Pp3bs30dHRbNu2jR49etCnTx/i4uJytWvVqhVr1qy5ujdPRPJn2nPoRcThdOzY0Rg1apSxf/9+AzB+//13275Tp04Z3t7exnfffWcYhmEMGjTIuOmmm3Idf/fddxsBAQEFnvtS7u7uxtdff51r29dff214eHjY1vv162fcd999udqsWLHCAIyzZ8/mez29e/c2nnzyyVwZrr/+ett6dna24evrawwePNi27cSJEwZgrF+/PtfrzJ4929bm9OnThre3tzFnzhzDMAzjzjvvNHr27JnrtQcNGpTve/GPqKgo4/3338+1bcGCBYaLi4uRk5NT4LEiUjD1AIlIoe3Zswc3Nzdat25t2xYcHEzdunXZs2cPAPv27aNVq1a5jrt0/WpZLJZc64Zh5Np24cIFvLy8CjxHTk4Or732Go0aNSI4OJhy5cqxbNmyy3pYGjVqZPuzq6srwcHBNGzY0LYtJCQEgMTExFzH/fv2W1BQUK73Ys+ePZfdnrt0PS0tjaeffpqoqCgCAwMpV64ce/fuvSyft7c3VquVjIyMAq9XRArmZnYAEXE8hmHku/2fwuTSIqWg4woSGhpKQkJCrm2JiYm2QgSgQoUKnD17tsDzvPPOO0yZMoWpU6fSsGFDfH19GT16NJmZmbnaubu751q3WCy5tv1zTVfzS6x/vxdXMnbsWJYuXcrbb79NrVq18Pb2ZsCAAZflO3PmDD4+Pnh7e1/xnCKSP/UAiUihRUVFkZ2dnWv8zOnTp9m/fz/XXXcdAPXq1WPjxo25jtu8eXOhX6tt27YsX74817Zly5bRrl0723rTpk3ZvXt3gedZs2YN/fr145577qFx48bUqFHjskHb12LDhg22P589e5b9+/dTr1494OL79e/9l7b/J9+wYcO45ZZbaNiwIaGhobY5jf5t586dNGvWrNhyizgrFUAiUmi1a9emX79+PPjgg6xdu5bt27dzzz33ULlyZfr16wfAyJEj+eWXX3j33XeJjY3lk08+YfHixZf1CsXExBATE0NqaionT54kJiYmVzEzatQoli1bxuTJk9m7dy+TJ08mOjo615xBPXr0YNeuXQX2AtWqVYvly5ezbt069uzZw8MPP3xZz9K1ePnll/n111/ZuXMnw4YNo0KFCvTv3x+Axx9/nCVLlvDmm2+yf/9+pk2bxpIlSy7L9+OPPxITE8P27du566678uxlWrNmzWWDwkWk8FQAiUiRTJ8+nebNm3PzzTfTtm1bDMPgl19+sd0uat++PR9//DHvvvsujRs3ZsmSJTzxxBOXjdVp2rQpTZs2ZcuWLXzzzTc0bdqU3r172/a3a9eO2bNnM336dBo1asSMGTOYM2dOrvFHDRs2pEWLFnz33Xf55p0wYQLNmjWjR48edOrUidDQUFuBUhzeeOMNRo0aRfPmzTlx4gQLFy7Ew8MDgDZt2vD555/z/vvv06RJE5YtW8bzzz+f6/gpU6ZQvnx52rVrR58+fejRo8dlPT3Hjh1j3bp13HvvvcWWW8RZWYyi3JQXESmCBx98kL1795bIz7h/+eUXnnrqKXbu3ImLS9n8b7uxY8eSlJTEp59+anYUEYenQdAiUmLefvttunXrhq+vL4sXL2bmzJl8+OGHJfJavXv3JjY2lmPHjhEZGVkir2G2SpUq8dRTT5kdQ6RMUA+QiJSYgQMHsnLlSlJSUqhRowYjR47kkUceMTuWiIgKIBEREXE+ZfNGuYiIiEgBVACJiIiI01EBJCIiIk5HBZCIiIg4HRVAIiIi4nRUAImIiIjTUQEkIiIiTkcFkIiIiDid/wMJyAelV+9DGgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABX8klEQVR4nO3deVgU9R8H8PdyLfctl5zeooC34G2Kind5leaRZlmemUf2q7QTtUwrtbTMI+/Cq7wxFc37QBFRUVFQQBR1l3OB3fn9QW6uHAIuzLL7fj3PPI8z853Z9ywL+/E735mRCIIggIiIiMiAGIkdgIiIiKiqsQAiIiIig8MCiIiIiAwOCyAiIiIyOCyAiIiIyOCwACIiIiKDwwKIiIiIDI6J2AF0kUqlQnJyMmxsbCCRSMSOQ0RERGUgCAIyMjLg4eEBI6PS+3hYABUjOTkZXl5eYscgIiKiCkhKSoKnp2epbVgAFcPGxgZA4Rtoa2srchoiIiIqC7lcDi8vL/X3eGlYABXjyWkvW1tbFkBERETVTFmGr3AQNBERERkcFkBERERkcFgAERERkcHhGKAXoFQqkZ+fL3YM0nOmpqYwNjYWOwYRkV5hAVQBgiAgNTUVjx8/FjsKGQh7e3u4ubnxvlRERFrCAqgCnhQ/Li4usLS05JcSVRpBEJCdnY20tDQAgLu7u8iJiIj0AwugclIqlerix8nJSew4ZAAsLCwAAGlpaXBxceHpMCIiLeAg6HJ6MubH0tJS5CRkSJ583jjmjIhIO1gAVRBPe1FV4ueNiEi7WAARERGRwWEBRERERAaHBRDpJYlEgm3btokdg4iIdBSvAiMiIqJKJ6hUUGTnIveRHDn5BRAcneBhbyFaHhZAVKz8/HyYmpqKHUPv5OXlwczMTOwYREQVUqBU4VF2Ph4npyHvzDnkptxDQVoaVOkPAbkcRhlyGGdm4kjLbjhevzUyFAXwvnYB81Z+CCtFNsxVSpgDiPZrjqUzfsDmcSGiHQtPgWmBIAjIzisQZRIEocw5VSoV5s2bhzp16kAqlcLb2xtffvklbt26BYlEgs2bN6NTp04wNzfH2rVroVKp8Nlnn8HT0xNSqRRNmjTBnj171PvLy8vDhAkT4O7uDnNzc/j6+iI8PFy9fs6cOfD29oZUKoWHhwcmTZr03IyzZs1CcHBwkeWBgYGYPXs2AOD06dMIDQ2Fs7Mz7Ozs0LFjR5w7d67M78PTnncMjx8/xltvvQVXV1eYm5ujcePG+Ouvv9TrIyIi0KhRI0ilUvj6+mLBggUa+/f19cUXX3yBUaNGwc7ODmPHjgUAHDt2DB06dICFhQW8vLwwadIkZGVlVegYiIheVIFShdQ7abiybR/OLfgZJyZ/ghMDRuNsmx6IqxOEpBpe+N8rM1Dnf7vR8stIzJm9Bo1e74/m099G668/RsivCxHyxwq03vs7WvyzG4rzF3Dq1kPEpchxJ0sJ+5wMmKqU6teTqvJhJHIFwh4gLcjJV8L/k72ivPblz7rD0qxsP8ZZs2bh559/xsKFC9GuXTukpKTgypUr6vUzZ87EggULsHLlSkilUnz33XdYsGABli1bhqZNm+LXX39F3759ERsbi7p16+L777/Hjh07sHnzZnh7eyMpKQlJSUkAgD/++AMLFy7Exo0b0ahRI6SmpuLChQvPzThs2DDMnTsXN27cQO3atQEAsbGxiImJwR9//AEAyMjIwMiRI/H9998DABYsWICePXsiPj4eNjY25Xr/SjsGlUqFsLAwZGRkYO3atahduzYuX76svhHh2bNnMXjwYMyZMwdDhgzBsWPH8O6778LJyQmjRo1Sv8bXX3+Njz/+GB999BEAICYmBt27d8fnn3+OFStW4P79+5gwYQImTJiAlStXlis/EVFZKfMLkHL+Mh5evIzs2CuQxMfjYKN22OPcAHce5SD4xjms3fxxidvbpd8DAEgkQK5zDdx28Ua2jT1y7R2Rb+cAla0tBFtbSGxt0axFMJY0bQYbcxPYoCluj24Dc2d7WDjaw8LeFiHmUojX91NIIpSnC8FAyOVy2NnZQSaTwdbWVmNdbm4uEhIS4OfnB3NzcwBAdl6BzhdAGRkZqFGjBhYvXow333xTY92tW7fg5+eHRYsWYfLkyerlNWvWxPjx4/Hhhx+ql7Vq1QotW7bEkiVLMGnSJMTGxiIyMrLIfWq+/fZbLFu2DJcuXSr3qbSgoCAMHDgQH39c+Iv44YcfIjIyEqdOnSq2vVKphIODA9avX4/evXsDKBwEvXXrVvTv37/U1yrtGPbt24ewsDDExcWhXr16RbYdNmwY7t+/j3379qmXzZgxAzt37kRsbCyAwh6gpk2bYuvWreo2I0aMgIWFBZYtW6ZedvToUXTs2BFZWVnqz9XTivvcERGVRJ6bj8vJctw9fRHeP/8Au/gr8Eq+CYsChUa7hW2H4rt2QwEAdR/dwW+bPsFjRxdk13BDnrsH4OkJUx8vWHjVhEXjhrDz84adhSmMjXTz3mSlfX8/iz1AWmBhaozLn3UX7bXLIi4uDgqFAl26dCmxTYsWLdT/lsvlSE5ORtu2bTXatG3bVt2TM2rUKISGhqJ+/fro0aMHevfujW7dugEABg0ahEWLFqFWrVro0aMHevbsiT59+sDE5PkfuWHDhuHXX3/Fxx9/DEEQsGHDBkyZMkW9Pi0tDZ988gn+/vtv3Lt3D0qlEtnZ2UhMTCzTe/G00o4hOjoanp6exRY/QOF72q9fvyLvz6JFi6BUKtU9RU+/r0Bhz9H169exbt069TJBEKBSqZCQkICGDRuW+ziIyHAJKhUSj5/Hvb2HgGPHEOneCD97toYgAH4P7+Lggf/+A5ZrYoYU55p47OEDhV8tBLwUivXdW8PXyQqutuYwXvY23MQ7lCrFAkgLJBJJmU9DieXJ86RKY2VlVWTZs70igiColzVr1gwJCQnYvXs3IiMjMXjwYHTt2hV//PEHvLy8cPXqVezfvx+RkZF499138fXXX+Pw4cPP7REaOnQoPvjgA5w7dw45OTlISkrCq6++ql4/atQo3L9/H4sWLYKPjw+kUilCQkKQl5dXlrdCQ2nH8Lz37On34ullz3r2fVWpVHj77beLHRPl7e1d7mMgIsOTcvEKkjZuh8mB/agVcwo+ORnw+XfdvQb3IdRsjZr2FqhTPxDHs96FWdMg1GjbAjVbBMLPVLe/r6oK3wUDUbduXVhYWODAgQNFToEVx9bWFh4eHjh69Cg6dOigXn7s2DG0atVKo92QIUMwZMgQDBw4ED169MDDhw/h6OgICwsL9O3bF3379sX48ePRoEEDxMTEoFmzZqW+tqenJzp06IB169YhJycHXbt2haurq3r9kSNHsHTpUvTs2RMAkJSUhAcPHpT3LXnuMQQGBuLOnTu4du1asb1A/v7+OHr0qMayY8eOoV69eqU+sLRZs2aIjY1FnTp1KpyZiAyLSiXgbOIj7IpJwekLt7Dtk75wF1Tq9bkmZrjp2xCyJi3g3r0bTg/piho20sKVbxS9sIRYABkMc3NzzJw5EzNmzICZmRnatm2L+/fvIzY2tsTTYtOnT8fs2bNRu3ZtNGnSBCtXrkR0dLT61M3ChQvh7u6OJk2awMjICL///jvc3Nxgb2+PVatWQalUonXr1rC0tMRvv/0GCwsL+Pj4FPtazxo2bBjmzJmDvLw8LFy4UGNdnTp18Ntvv6FFixaQy+WYPn16mXq4ilPaMXTs2BEdOnTAgAED8O2336JOnTq4cuUKJBIJevTogffffx8tW7bE559/jiFDhuD48eNYvHgxli5dWuprzpw5E8HBwRg/fjzGjh0LKysrxMXFYf/+/fjhhx8qdBxEpH+U+QWI2/QXstZthOzuPbzVc5p63VlPf9iaGeFx205w6NcTtXp0hL8lxweWi0BFyGQyAYAgk8mKrMvJyREuX74s5OTkiJDsxSiVSuGLL74QfHx8BFNTU8Hb21v46quvhISEBAGAcP78+SLtP/30U6FmzZqCqampEBQUJOzevVu9fvny5UKTJk0EKysrwdbWVujSpYtw7tw5QRAEYevWrULr1q0FW1tbwcrKSggODhYiIyPLnPXRo0eCVCoVLC0thYyMDI11586dE1q0aCFIpVKhbt26wu+//y74+PgICxcuVLcBIGzduvW5r1PaMQiCIKSnpwtvvPGG4OTkJJibmwuNGzcW/vrrL/X6P/74Q/D391e/n19//bXG/p/N9cSpU6eE0NBQwdraWrCyshICAwOFL7/8ssSc1flzR0Tlc+fMJeH40HeFFLsaggAIAiAUSIyEju9vEN7bdF7YHZMiyDL5t6A4pX1/P4tXgRWjvFeBEVU2fu6I9JtSJeD86i0wnx+OxlfOqpfLza1xpU0opK8OQsNhL8OMvTyl4lVgRERE1UCWogC/n0nCymO30ObAIYRfOQsVJLjUqBXyR4xEo3dGoJVN0QtU6MWxAKIqdeTIEYSFhZW4PjMzU6uv99VXX+Grr74qdl379u2xe/durb4eEVFZZD+S4cKHcxHxyBR/+LYuXNY8FL0dClBr1mQEBjYQOaH+YwFEVapFixaIjo6ustcbN24cBg8eXOy6ig6cJiKqqNyMLER/GI66K5cgJOsx3BzccW5GB7zRsQ4GNPeEpVlfsSMaDBZAVKUsLCyq9PJvR0dHODo6VtnrEREVR1CpEL1kDVzmfIjghykAgGRHdzwY/z72vdcBJlI+JLmqsQAiIiKqRLePnoH87fFoernwcT73bRxxa+IMNPl4CjzMpSKnM1wsgIiIiCpBgVKFZVE3cfHHv7Ds8inkGZvg7MDRCFwyDy2d7MWOZ/BYABEREWnZtZupeH/ndcTclQG1WmL9a1PRceoohLQIEDsa/ctI7ABERET6QlCpcGLC/+AQ2BD3rtyErbkJvh0chNfWfYOaLH50CnuAiIiItECWch83+wxG8NlDAICZaSfRPnwBXGx580JdxB4g0ppRo0ahf//+pbbp1KkTpkyZUiV5iIiqyvW9UchsHISmZw8hz9gEJ6d/jlc2/cDiR4exADIgxRUof/zxB8zNzTF//nzMmTMHEomkyBQZGSlOYCKiauDs/B/h1asraj5MQbKDG25v34fW8z+CxIhfsbqMPx0D9ssvv2DYsGFYvHgxZsyYAQBo1KgRUlJSNKYOHTqInJSISPcIgoA9s39A85nvQqrMx4WgtrCKuYC6vTqLHY3KgAWQNmVllTzl5pa9bU5O2dq+gPnz52PChAlYv3493nzzTfVyExMTuLm5aUxmZoU36IqJicFLL70ECwsLODk54a233ir10RVZWVkYMWIErK2t4e7ujgULFrxQZiIiXVGgVGFmxEVMzXBHrEstnOg/Eo1PH4JdTRexo1EZsQDSJmvrkqcBAzTburiU3PbZZ2X5+hbfroI++OADfP755/jrr78w4NlcJcjOzkaPHj3g4OCA06dP4/fff0dkZCQmTJhQ4jbTp0/HwYMHsXXrVuzbtw+HDh3C2bNnS2xPRFQdKBR5mLjhPDafuQOFuSUubvwLwVtXwdiU1xVVJ6IWQFFRUejTpw88PDwgkUiwbds2jfXFjUeRSCT4+uuvS9znqlWrit0m99keGAO1e/duzJs3D9u3b0fXrl2LrI+JiYG1tbV6atWqFQBg3bp1yMnJwZo1a9C4cWO89NJLWLx4MX777Tfcu3evyH4yMzOxYsUKfPPNNwgNDUVAQABWr14NpVJZ6cdIRFRZch5n4GqLjvD5ZTHMjI3w47BmeK1zQ7FjUQWIWq5mZWUhKCgIb7zxRrE9ESkpKRrzu3fvxpgxY57ba2Fra4urV69qLDM3r4KR+KU9ydzYWHM+La3kts8OnLt1q8KRnhUYGIgHDx7gk08+QcuWLWFjY6Oxvn79+tixY4d6XiotvE17XFwcgoKCYGVlpV7Xtm1bqFQqXL16Fa6urhr7uXHjBvLy8hASEqJe5ujoiPr162vtWIiIqlKuPBM3Qjoj8MpZ1DaNxkufT0WrRm5ix6IKErUACgsLQ9izp3ue4uam+cHavn07OnfujFq1apW6X4lEUmTbKvFUcSBa2+eoWbMmIiIi0LlzZ/To0QN79uzRKILMzMyKfVipIAiQSCTF7rO45YIgaC0zEZHYFFnZuNquG4KunEWWmQWS1kWgVXve2LA6qzZjgO7du4edO3dizJgxz22bmZkJHx8feHp6onfv3jh//nyp7RUKBeRyucakz7y9vXH48GGkpaWhW7duZTpef39/REdHI+upwdf//PMPjIyMUK9evSLt69SpA1NTU5w4cUK97NGjR7h27Zp2DoKIqIoUKPIQ26k3gmKOI8dEiturN6HBwJL/807VQ7UpgFavXg0bGxu88sorpbZr0KABVq1ahR07dmDDhg0wNzdH27ZtER8fX+I24eHhsLOzU09eXl7ajq9zPD09cejQIaSnp6Nbt26QyWSlth82bBjMzc0xcuRIXLp0CQcPHsTEiRMxfPjwIqe/AMDa2hpjxozB9OnTceDAAVy6dAmjRo2CEe+LQUTViKBS4VzvoWh25iDyjE1w/eff4P9qH7FjkRZUm2+jX3/9Vf0lXJrg4GC8/vrrCAoKQvv27bF582bUq1cPP/zwQ4nbzJo1CzKZTD0lJSVpO75OqlmzJg4fPozHjx8jNDQUjx8/LrGtpaUl9u7di4cPH6Jly5YYOHAgunTpgsWLF5e4zddff40OHTqgb9++6Nq1K9q1a4fmzZtXwpEQEVWO7YvWo1VkBFSQIPbb5QgYNUjsSKQlEkFHBmtIJBJs3bq12EcpHDlyBB06dEB0dDSCgoLKve+xY8fizp072L17d5nay+Vy2NnZQSaTwdbWVmNdbm4uEhIS4OfnVzUDq4nAzx2RGLZH38XkjdF4/fwu9G7kguBvZ4sdiZ6jtO/vZ1WLmxasWLECzZs3r1DxIwgCoqOjERDAwWpERFQ20UmPMf33iwAA84njEdzbX+REpG2ingLLzMxEdHQ0oqOjAQAJCQmIjo5GYmKiuo1cLsfvv/+ucbfip40YMQKzZs1Sz3/66afYu3cvbt68iejoaIwZMwbR0dEYN25cpR4LERHphwc3EnG332BYZj5GqL8rPuzJ+/zoI1F7gM6cOYPOnf97ZsrUqVMBACNHjsSqVasAABs3boQgCHjttdeK3UdiYqLGwNrHjx/jrbfeQmpqKuzs7NC0aVNERUWpb+hHRERUkvxcBdLC+qFXfDRqZMvQcN5RGBkVfwsQqt50ZgyQLinLGCBfX19YWFiIlJAMTU5ODm7dusUxQESV7ET/kQjevgaZZpZ4eDAK3m144UZ1Up4xQNXmKjBdYWpqCqDw2VhEVeXJ5+3J54+ItC/6x98QvH0NACD+myUsfvRctRgErUuMjY1hb2+PtH8fZWFpaVniHZKJXpQgCMjOzkZaWhrs7e1h/OwjVYhIKx5cS4DPtIkAgBN9hyN44ihxA1GlYwFUAU8es5FW2vO8iLTI3t5enMe7EBkAVYESqf0Ho3G2DDdq1kXT9T+JHYmqAAugCpBIJHB3d4eLiwvy8/PFjkN6ztTUlD0/RJVoU8Q/aH/3NrJNpTD9fSOkVpZiR6IqwALoBRgbG/OLiYioGrtxPxNzYrJgMnoJvmsAdA1pJnYkqiIsgIiIyCApVQJm/HERigIVWjX2RpfRvF2KIeFVYEREZJBOzPgS9bevh7WZMeYOCOQFLQaGPUBERGRw7p6NRbPvvkDbAgW6d2mCmva8r5uhYQ8QEREZFEGlwoMxb8OiQIFLDZqjw/ujxY5EImABREREBiV6yRoEXfgHeUYmsP11OSRG/Co0RPypExGRwch+JIPbJx8AAM4OGg1vXvVlsFgAERGRwbjwzgy4P76HVHsXBP04X+w4JCIWQEREZBDuxiei2R8rAQApn86FpYOdyIlITLwKjIiIDEL46QdIGjoXY9POodeEkWLHIZGxACIiIr13LvER/rqYAknN+qg1900OfCaeAiMiIv0mqFT4ce1hAMDAZp7w97AVORHpAhZARESk184vXIHFnwzCtGMbMK17fbHjkI5gAURERHqrQJEH53mfQ6osQHMfe7jamosdiXQECyAiItJb5+cuhff9JDyytEXjBZ+KHYd0CAsgIiLSS/m5Cnh8X3ivn6vDx8HGxUnkRKRLWAAREZFeOv/ZItR8mIJ0K3sEhn8odhzSMSyAiIhI7yiysuG9dAEAIH7MRN70kIpgAURERHrn0NpdsM+SIc3WGU2+mCl2HNJBLICIiEivFChV+ELmiHbjVuD03KUwt7ESOxLpIBZARESkV3bGpCDpYQ5ULq54acwrYschHcUCiIiI9IaqQIm9v+0GAIxu6wsLM2ORE5Gu4rPAiIhIb1xcth5Lvx2LyAZt0PKTg2LHIR3GHiAiItILgkoFiwXzAABWjRvCztJM5ESky1gAERGRXri6bR/qJ8RCYWyKuuEfiR2HdBwLICIi0gvZ8wrv+3OhQy841/EVNwzpPBZARERU7aVcvIKg038DAJw/miFyGqoOWAAREVG1d/uzr2EsqHCpQQvUeilE7DhUDbAAIiKiai0rNx/OUQcAAAWTJomchqoLUQugqKgo9OnTBx4eHpBIJNi2bZvG+lGjRkEikWhMwcHBz91vREQE/P39IZVK4e/vj61bt1bSERARkdi2nL+LsJGL8MnwOQgcO1TsOFRNiFoAZWVlISgoCIsXLy6xTY8ePZCSkqKedu3aVeo+jx8/jiFDhmD48OG4cOEChg8fjsGDB+PkyZPajk9ERCITBAGrjt1CvrEpao8bCSMT3viQykbUGyGGhYUhLCys1DZSqRRubm5l3ueiRYsQGhqKWbNmAQBmzZqFw4cPY9GiRdiwYcML5SUiIt1y5twNJNyTw8rcDAOae4odh6oRnR8DdOjQIbi4uKBevXoYO3Ys0tLSSm1//PhxdOvWTWNZ9+7dcezYsRK3USgUkMvlGhMREek+o/HvImrZWEwTEmAt5cMNqOx0ugAKCwvDunXr8Pfff2PBggU4ffo0XnrpJSgUihK3SU1Nhaurq8YyV1dXpKamlrhNeHg47Ozs1JOXl5fWjoGIiCrHg+u3EXj6b3jK09ChQ6DYcaia0ekCaMiQIejVqxcaN26MPn36YPfu3bh27Rp27txZ6nYSiURjXhCEIsueNmvWLMhkMvWUlJSklfxERFR5rs/9HqYqJa74NUbt0LZix6Fqplr1F7q7u8PHxwfx8fEltnFzcyvS25OWllakV+hpUqkUUqlUazmJiKhyKfML4BOxDgCQMXKMyGmoOtLpHqBnpaenIykpCe7u7iW2CQkJwf79+zWW7du3D23atKnseEREVEUu/boZ7o/v4bGFDQLee1PsOFQNidoDlJmZievXr6vnExISEB0dDUdHRzg6OmLOnDkYMGAA3N3dcevWLXz44YdwdnbGyy+/rN5mxIgRqFmzJsLDwwEAkydPRocOHTBv3jz069cP27dvR2RkJI4ePVrlx0dERJVD+csvAIArof0RbGstchqqjkQtgM6cOYPOnTur56dOnQoAGDlyJH788UfExMRgzZo1ePz4Mdzd3dG5c2ds2rQJNjY26m0SExNhZPRfR1abNm2wceNGfPTRR/j4449Ru3ZtbNq0Ca1bt666AyMiokrzKDEFjc8dAQC4Th4nchqqriSCIAhih9A1crkcdnZ2kMlksLW1FTsOERE9ZeXRm/hz2RYMSI/DsF0rxI5DOqQ839/VagwQERHRH+fu4lzNhij46GOxo1A1xgKIiIiqjcvJcsQmy2FmbIS+QR5ix6FqrFpdBk9ERIbt/vuz8MWVBNx8bTQcrMzEjkPVGHuAiIioWsjLzkXgjnV4PXo3+tnmih2HqjkWQEREVC3ErtgIh2wZHlg7otEbg8WOQ9UcCyAiIqoeVq0CAFzv3h8mUp7+ohfDAoiIiHTeo9vJaBxdeENbt4lvi5yG9AELICIi0nnXlqyEqUqJ65514duxldhxSA+wACIiIp1nvS0CAPCg9ysiJyF9wcvgiYhIp6U+zsEh57qwSUuG77ujxY5DeoI9QEREpNP+iknB1x1GYOpXEXALqCd2HNITLICIiEin/XkhGQDQt2lNkZOQPmEBREREOivp4jU4Ho6EmaoAYY3dxY5DeoQFEBER6aw73/2ElX98inWRC1HDRip2HNIjLICIiEhnue/aWviPXr3FDUJ6hwUQERHppIRDJ+Gbegt5Riao9+5IseOQnmEBREREOil12UoAQGxQG9i51xA5DekbFkBERKRzBJUKnpE7AQDKQYNETkP6iAUQERHpnNtHzsDrwR3kGZug/pjXxI5DeogFEBER6Zzbm3cAAOIat4aNi5PIaUgf8VEYRESkc+bV7YYvR7tjWudaYkchPcUCiIiIdEpiejYup2bAyMUXLV7uInYc0lM8BUZERDplb2wqAKC1nxMcrcxETkP6ij1ARESkU5q/OwzfqaRQzpkjdhTSYyyAiIhIZzy4loBmsSfQDMC9AE+x45Ae4ykwIiLSGTd+WQcAuOrrD9eGdUROQ/qMBRAREekMy78KL39/1J3P/qLKxQKIiIh0guxOKhpeOQsAqPnGUJHTkL5jAURERDrh2spNMBFUSHDzg1frILHjkJ5jAURERDrBaGfhs79SO3UXOQkZAl4FRkREossrUOFvW1+YutWBw6uviB2HDAALICIiEt3pWw+xpFk/bOowCKf6dBU7DhkAngIjIiLRHYhLAwB0ru8CIyOJyGnIELAAIiIiUQkqFbK3bIOVIhtdGrqIHYcMhKgFUFRUFPr06QMPDw9IJBJs27ZNvS4/Px8zZ85EQEAArKys4OHhgREjRiA5ObnUfa5atQoSiaTIlJubW8lHQ0REFZF4/DzmrvofTi4difbetmLHIQMhagGUlZWFoKAgLF68uMi67OxsnDt3Dh9//DHOnTuHLVu24Nq1a+jbt+9z92tra4uUlBSNydzcvDIOgYiIXlDK2t8BADfrBcHK1krkNGQoRB0EHRYWhrCwsGLX2dnZYf/+/RrLfvjhB7Rq1QqJiYnw9vYucb8SiQRubm5lzqFQKKBQKNTzcrm8zNsSEdGLsT2wFwCQE9pD5CRkSKrVGCCZTAaJRAJ7e/tS22VmZsLHxweenp7o3bs3zp8/X2r78PBw2NnZqScvLy8tpiYiopLI7qSi3vWLAACvEYNFTkOGpNoUQLm5ufjggw8wdOhQ2NqWfI64QYMGWLVqFXbs2IENGzbA3Nwcbdu2RXx8fInbzJo1CzKZTD0lJSVVxiEQEdEz4lf/rr77s0eThmLHIQNSLe4DlJ+fj1dffRUqlQpLly4ttW1wcDCCg4PV823btkWzZs3www8/4Pvvvy92G6lUCqlUqtXMRET0fJInd3/uGAo/kbOQYdH5Aig/Px+DBw9GQkIC/v7771J7f4pjZGSEli1bltoDREREVa8gLx91z/8DAHAY/LLIacjQ6PQpsCfFT3x8PCIjI+Hk5FTufQiCgOjoaLi7u1dCQiIiqqjo5Az0GrkIX/SeiLq8+zNVMVF7gDIzM3H9+nX1fEJCAqKjo+Ho6AgPDw8MHDgQ586dw19//QWlUonU1FQAgKOjI8zMzAAAI0aMQM2aNREeHg4A+PTTTxEcHIy6detCLpfj+++/R3R0NJYsWVL1B0hERCU6fO0+kuzdcK9jMxib6vwJCdIzon7izpw5g86dO6vnp06dCgAYOXIk5syZgx07dgAAmjRporHdwYMH0alTJwBAYmIijIz+68h6/Pgx3nrrLaSmpsLOzg5NmzZFVFQUWrVqVbkHQ0RE5XLo6n0AQMd6NUROQoZIIgiCIHYIXSOXy2FnZweZTFbuMUdERPR86QlJONt9MA7Vao4pm7+Gi62F2JFID5Tn+5t9jkREVOUS1m1Ft/gTqJWTDhfbok8DIKpsOj0ImoiI9NSePQCA+207P6chUeVgAURERFVKmV+A2v9e/m73ch+R05ChYgFERERV6vreKDhky5EhtUTdfqFixyEDxQKIiIiq1MM/Cq/wjQ9oDVNz3oWfxMECiIiIqpRD1AEAQEFod5GTkCFjAURERFXmsTwbj5QS5BsZw3voK2LHIQPGy+CJiKjKHEl4jImvhSPIzgjbG9cVOw4ZMPYAERFRlYm6Vnj355YBPiInIUPHAoiIiKqEoFLhQswtAEB7Pv6CRMZTYEREVCWSTl3A7i9ewTnPhmg857zYccjAsQeIiIiqRPLvf8JYUMHS0hwW5qZixyEDxwKIiIiqhPTQQQCAvH0ncYMQgQUQERFVgQJFHurEngIAOPXrKXIaIhZARERUBa7vPAgbRTZk5tao3b2D2HGIWAAREVHle7RjNwDgRmBrGJvy+hsSHwsgIiKqdHb/HAIA5Hd+SdwgRP9iGU5ERJUqU1GAVfU7o4OJLZoO7i92HCIALICIiKiSnbyZjs2Nu+JEh76IauYvdhwiADwFRkRElezo9QcAgLZ1nEVOQvQfFkBERFSpLNavRe0HSWhfx0nsKERqEkEQBLFD6Bq5XA47OzvIZDLY2tqKHYeIqNpKu3ITLg1rQwUJMpKSYefpJnYk0mPl+f5mDxAREVWa25t2AABueNdn8UM6hQUQERFVGklkJADgQQhvfki6hQUQERFVCkGlgm/0cQCATe8wkdMQaWIBRERElSLx6Fk4Zz5ErokZ6vQLFTsOkQYWQEREVClS/338xfW6gTC3sRI5DZEmFkBERFQpTKKiAAAZIe1FTkJUFO8ETUREWqdSCZgUOgF1fDtg2qjeYschKoIFEBERad3Vexm4qzTFw/qt0KBNE7HjEBXBU2BERKR1x2+kAwBa+DrA1JhfNaR72ANERERa5zb3U3zwIBPWAe+IHYWoWCzLiYhIq1QFSrQ9EIFxJyPQ0loldhyiYolaAEVFRaFPnz7w8PCARCLBtm3bNNYLgoA5c+bAw8MDFhYW6NSpE2JjY5+734iICPj7+0MqlcLf3x9bt26tpCMgIqJn3TzwD+xyM5FpZonaPTqKHYeoWBUugAoKChAZGYlly5YhIyMDAJCcnIzMzMwy7yMrKwtBQUFYvHhxsevnz5+Pb7/9FosXL8bp06fh5uaG0NBQ9esV5/jx4xgyZAiGDx+OCxcuYPjw4Rg8eDBOnjxZvgMkIqIKefDnXgDAjQZNYCI1EzkNUfEq9DT427dvo0ePHkhMTIRCocC1a9dQq1YtTJkyBbm5ufjpp5/KH0QiwdatW9G/f38Ahb0/Hh4emDJlCmbOnAkAUCgUcHV1xbx58/D2228Xu58hQ4ZALpdj9+7d6mU9evSAg4MDNmzYUOw2CoUCCoVCPS+Xy+Hl5cWnwRMRVUB0k/ZocuEoTrzzAYKXhosdhwxIpT8NfvLkyWjRogUePXoECwsL9fKXX34ZBw4cqMgui0hISEBqaiq6deumXiaVStGxY0ccO3asxO2OHz+usQ0AdO/evdRtwsPDYWdnp568vLxe/ACIiAxQgSIPta+cAwA49e4hchqiklWoADp69Cg++ugjmJlpdm36+Pjg7t27WgmWmpoKAHB1ddVY7urqql5X0nbl3WbWrFmQyWTqKSkp6QWSExEZrpv7jsBGkQ25uTVqhbYVOw5RiSp0GbxKpYJSqSyy/M6dO7CxsXnhUE+TSCQa84IgFFn2ottIpVJIpdKKhyQiIgDAzYvxcLC0x536gWhqyjutkO6qUA9QaGgoFi1apJ6XSCTIzMzE7Nmz0bNnT60Ec3NzA4AiPTdpaWlFenie3a682xARkXZscG+GlhN+Q8xX34sdhahUFSqAFi5ciMOHD8Pf3x+5ubkYOnQofH19cffuXcybN08rwfz8/ODm5ob9+/erl+Xl5eHw4cNo06ZNiduFhIRobAMA+/btK3UbIiJ6cflKFU7feghIJGgR6Cd2HKJSVah/0sPDA9HR0di4cSPOnj0LlUqFMWPGYNiwYRqDop8nMzMT169fV88nJCQgOjoajo6O8Pb2xpQpU/DVV1+hbt26qFu3Lr766itYWlpi6NCh6m1GjBiBmjVrIjy88EqDyZMno0OHDpg3bx769euH7du3IzIyEkePHq3IoRIRURnF3EhDtqIADlZmaOCm3eEQRFoniOjgwYMCgCLTyJEjBUEQBJVKJcyePVtwc3MTpFKp0KFDByEmJkZjHx07dlS3f+L3338X6tevL5iamgoNGjQQIiIiypVLJpMJAASZTPYih0dEZFCOj5kq3LNyEDYPfU/sKGSgyvP9XaH7AK1evRrOzs7o1asXAGDGjBlYvnw5/P39sWHDBvj4+GivQhNBee4jQEREhWIatUbA5VM4MWU2ghfOETsOGaBKvw/QV199pT7Vdfz4cSxevBjz58+Hs7Mz3nvvvYrskoiIqjFFdg5qx18AALj2CxM5DdHzVWgMUFJSEurUqQMA2LZtGwYOHIi33noLbdu2RadOnbSZj4iIqoEbfx2Ef74CD63s4NuhpdhxiJ6rQj1A1tbWSE9PB1B4hVXXrl0BAObm5sjJydFeOiIiqhZku/cBAG41agmJkajP2SYqkwr1AIWGhuLNN99E06ZNce3aNfVYoNjY2Go//oeIiMrP9njhlbb5Hfn0d6oeKlSmL1myBCEhIbh//z4iIiLg5OQEADh79qzGJepERKT/cjOyUOf6RQCAG8f/UDVRoavAACA3NxcXL15EWloaVCqVxrq+fftqJZxYeBUYEVHZnTp3A+ffmYHAR4kIvnKSp8BINOX5/q7QKbA9e/ZgxIgRSE9Px7P1k0QiKfY5YUREpJ+OpivxfefR6BvkgRAWP1RNVOiTOmHCBAwaNAjJyclQqVQaE4sfIiLDcuJG4UUxIbWdRE5CVHYVKoDS0tIwdepUPmCUiMjA5cgyYXX4ACzychFSiwUQVR8VKoAGDhyIQ4cOaTkKERFVN9e37sXKjR9j/+qJ8HGyFDsOUZlVaAzQ4sWLMWjQIBw5cgQBAQEwNTXVWD9p0iSthCMiIt2WuXc/ACClUTN4SiQipyEquwoVQOvXr8fevXthYWGBQ4cOQfLUh14ikbAAIiIyEA4n/yn8B58CQNVMhQqgjz76CJ999hk++OADGHHEPxGRQcpKf4zat+MAADVf7ilyGqLyqVD1kpeXhyFDhrD4ISIyYNe37oWpSokUe1d4NPUXOw5RuVSoghk5ciQ2bdqk7SxERFSNZO+LBADcadJa5CRE5VehU2BKpRLz58/H3r17ERgYWGQQ9LfffquVcEREpLucTj0Z/9NZ3CBEFVChAigmJgZNmzYFAFy6dEljnYRXARAR6b2M3HxM6joerW/H4J0h1fvxR2SYKlQAHTx4UNs5iIioGjl96yGuOPsip74/Pm1QS+w4ROXGUcxERFRux588/oJ3f6ZqigUQERGVW53v52LIhb1o72L6/MZEOqhCp8CIiMhwye6mYdC+tRgCAQ++e0/sOEQVwh4gIiIql5tbdsEIAhJreMG5np/YcYgqhAUQERGVi2L/AQBASrNgkZMQVRwLICIiKheXs8cBACZdXhI5CVHFsQAiIqIye3Q7GbWSbwAAfAf2EjkNUcWxACIiojJLiNgFALjl6gsnPy+R0xBVHAsgIiIqs/Tzl6CCBPeac/wPVW+8DJ6IiMpsftOXMc0xGAv71hc7CtELYQFERERlcj9Dgfi0TMDCBk2CG4kdh+iF8BQYERGVyYmbhY+/aOBmA0crM5HTEL0YFkBERFQmjv+bgc3rZmD448tiRyF6YTwFRkREZeJ9Kgpe95Ng6mQudhSiF8YeICIieq60uBvwup8EpcQItXj/H9IDOl8A+fr6QiKRFJnGjx9fbPtDhw4V2/7KlStVnJyISH8kRuwEANz0qgc79xoipyF6cTp/Cuz06dNQKpXq+UuXLiE0NBSDBg0qdburV6/C1tZWPV+jBn9hiYgqSvX3QQDAg5ZtUVfkLETaoPMF0LOFy9y5c1G7dm107Nix1O1cXFxgb29ficmIiAyHZ/QJAIBVj64iJyHSDp0/Bfa0vLw8rF27FqNHj4ZEIim1bdOmTeHu7o4uXbrg4MGDpbZVKBSQy+UaExERFUo+fxkej1KRb2SM2i/3EDsOkVZUqwJo27ZtePz4MUaNGlViG3d3dyxfvhwRERHYsmUL6tevjy5duiAqKqrEbcLDw2FnZ6eevLz4fBsioicuXkvBvrrBuODfGlZO9mLHIdIKiSAIgtghyqp79+4wMzPDn3/+Wa7t+vTpA4lEgh07dhS7XqFQQKFQqOflcjm8vLwgk8k0xhERERmiKRvPY1t0Mia+VAfvd+MjMEh3yeVy2NnZlen7W+fHAD1x+/ZtREZGYsuWLeXeNjg4GGvXri1xvVQqhVQqfZF4RER6SRAEHLtReAfokNpOIqch0p5qcwps5cqVcHFxQa9e5b//xPnz5+Hu7l4JqYiI9Nut+CSYJd2GmYkRmnk7iB2HSGuqRQ+QSqXCypUrMXLkSJiYaEaeNWsW7t69izVr1gAAFi1aBF9fXzRq1Eg9aDoiIgIRERFiRCciqtbu/fIbji77CMdahsL8izCx4xBpTbUogCIjI5GYmIjRo0cXWZeSkoLExET1fF5eHqZNm4a7d+/CwsICjRo1ws6dO9GzZ8+qjExEpBdMDx8CAEga8envpF+q1SDoqlKeQVRERPpKVaCEzM4JDtkyXPl9FxoMZA8Q6bbyfH9XmzFARERUtW5FnYJDtgzZpuao1bOz2HGItIoFEBERFStt+24AwPX6TWBmySfAk35hAURERMWSHjkMAMhu20HkJETaxwKIiIiKUOYXoPblswAAp97dRU5DpH3V4iowIiKqWrF3HmFJz8lolxyHod3aix2HSOtYABERURHHEuXYW68NlP1exnAzU7HjEGkdT4EREVERTx5/0YaPvyA9xR4gIiLSkJ+rQMs1P0Dh0Qghfm3FjkNUKVgAERGRhvjtkZh4eC1et7SD3ZrpYschqhQ8BUZERBpkfxXe/+dmYGsYmRiLnIaocrAAIiIiDQ7/FN7/R9mli8hJiCoPCyAiIlKT33uAOrcuAwC8h/QXNwxRJWIBREREatc3/wUTQYUkZ0+4BdQTOw5RpWEBREREavl79gIAklvw6i/SbyyAiIhIzTYmGgBgFsbHX5B+42XwREQEALj7OAe9Xp2HwLSbWDO0v9hxiCoVe4CIiAgA8E/8A6iMjGHUqiVsnR3EjkNUqVgAERERAODI9QcAgHZ1nEVOQlT5eAqMiIigKlDinWlD0M7JF3UHLxU7DlGlYwFERES4eeAf+CfHw+fBHZj5e4kdh6jS8RQYERHhwdadAID4hs1hai4VOQ1R5WMBREREsDpyCACQ27GzqDmIqgoLICIiA5ebkYW616IBAO4D+ogbhqiKsAAiIjJw17fuhXlBHtJsnODdrrnYcYiqBAsgIiIDl/HXbgDA7aYhkBjxa4EMA68CIyIycBfzzVHD0ROSbt3EjkJUZVgAEREZsHvyXITX74G5DXrg7OSuYschqjLs6yQiMmCHr94HAAR62sPRmpe/k+FgAUREZMCuHTgGs4J8dKpXQ+woRFWKp8CIiAxUgSIPk+aMwXsqJW73jhI7DlGVYg8QEZGBurZtP2wVWcgzNUP9tk3FjkNUpVgAEREZKNnWHQCAm03awNiUJwTIsLAAIiIyUC5HDwIAhB49RE5CVPV0ugCaM2cOJBKJxuTm5lbqNocPH0bz5s1hbm6OWrVq4aeffqqitERE1ceD+ATUvhsPAPB7/RWR0xBVPZ0ugACgUaNGSElJUU8xMTEltk1ISEDPnj3Rvn17nD9/Hh9++CEmTZqEiIiIKkxMRKT7En7bAgCI924AJz8vkdMQVT2dP+lrYmLy3F6fJ3766Sd4e3tj0aJFAICGDRvizJkz+OabbzBgwIBKTElEVL0Y79sDAEhv9xLqipyFSAw63wMUHx8PDw8P+Pn54dVXX8XNmzdLbHv8+HF0e+ZW7t27d8eZM2eQn59f4nYKhQJyuVxjIiLSVwVKFb5oNhDzOo6EzYihYschEoVOF0CtW7fGmjVrsHfvXvz8889ITU1FmzZtkJ6eXmz71NRUuLq6aixzdXVFQUEBHjx4UOLrhIeHw87OTj15ebE7mIj01/mkxzhn64kNLw1Fg9A2YschEoVOF0BhYWEYMGAAAgIC0LVrV+zcuRMAsHr16hK3kUgkGvOCIBS7/GmzZs2CTCZTT0lJSVpIT0SkmyLj7gEAOtarAWOjkv82EukznR8D9DQrKysEBAQgPj6+2PVubm5ITU3VWJaWlgYTExM4OTmVuF+pVAqplM/AISLDUDf8E/S190P3V94WOwqRaHS6B+hZCoUCcXFxcHd3L3Z9SEgI9u/fr7Fs3759aNGiBUxNTasiIhGRTks6fg4Dj/yBb3YuRHsfO7HjEIlGpwugadOm4fDhw0hISMDJkycxcOBAyOVyjBw5EkDhqasRI0ao248bNw63b9/G1KlTERcXh19//RUrVqzAtGnTxDoEIiKdcve33wEAVxs0g42rs8hpiMSj06fA7ty5g9deew0PHjxAjRo1EBwcjBMnTsDHxwcAkJKSgsTERHV7Pz8/7Nq1C++99x6WLFkCDw8PfP/997wEnojoX7b7dwMAssN6iZyESFwS4ckoYVKTy+Wws7ODTCaDra2t2HGIiLTi0e1k2Pp5wVhQIfXiVbgF1BM7EpFWlef7W6dPgRERkfZcX7UJxoIKN2rWZfFDBo8FEBGRgTD5608AQFrnbs9pSaT/WAARERmA3LwC5N0vvIms82uDRE5DJD6dHgRNRETacTzhId549SsECJnY0aO92HGIRMceICIiAxB5ufDuz0Eh/pAY8U8/EX8LiIj0nKpAiRPnbgAAujR0fU5rIsPAAoiISM9d27Efu78ahKU7v0Gb2iU/FojIkLAAIiLSc4/XbICZqgAu9laQmhiLHYdIJ7AAIiLSY4JSCd9DhXd/Nh48UOQ0RLqDBRARkR6L33kQbrL7yDKzQMORLICInmABRESkxx6uXg8AuNK8A8xtrUVOQ6Q7WAAREekpQaWC98HC01+Sgez9IXoaCyAiIj11Y+8ReDxKRbapFA1HDxE7DpFO4Z2giYj01M4MM6R3fRutbAX0trcROw6RTmEBRESkh1QqAZtv5uBu8z5oNbSp2HGIdA5PgRER6aFziY9w93EOrMyM0ZV3fyYqgj1ARER66N4XX2PonQxIBg+GuSlvfkj0LBZARER6Jj9XgTYbfkSvbBliBvHJ70TF4SkwIiI9c3l1BByyZUi3skfD118WOw6RTmIBRESkZ/J/WwsAuN6pJ0ykZiKnIdJNLICIiPRI9iMZ/E8dBADYjx0lbhgiHcYCiIhIj1xeugaW+bm46+iOen26iB2HSGexACIi0iPma1cDABJ7D4LEiH/iiUrC3w4iIj2RdE+G1Hwj5BsZw/f9d8SOQ6TTeBk8EZGe+OPiPXw3cDZ6uJnip8AGYsch0mnsASIi0gMqlYA/zt4BAIR1aiRyGiLdxwKIiEgPnPv7NITE27AxN0H3Rm5ixyHSeSyAiIj0gGT2bBz9cQzmJv3NR18QlQELICKiau5xYgoanzwAIwio90oPseMQVQssgIiIqrkrc7+HVJmPGzXrok5YR7HjEFULLICIiKoxVYESXpsK7/2TPnIM7/1DVEb8TSEiqsZift2Emg9TIJdaIWD6u2LHIao2WAAREVVnS5cCAC53fwUW9jYihyGqPnS6AAoPD0fLli1hY2MDFxcX9O/fH1evXi11m0OHDkEikRSZrly5UkWpiYiqxp1bKfC9Eg0AqPnh++KGIapmdPpO0IcPH8b48ePRsmVLFBQU4H//+x+6deuGy5cvw8rKqtRtr169CltbW/V8jRo1KjsuEVGV+i3uMX57dxXezL+Fqa2DxI5DVK3odAG0Z88ejfmVK1fCxcUFZ8+eRYcOHUrd1sXFBfb29pWYjohIPJmKAqw/mYhsMwsEvDlc7DhE1Y5OnwJ7lkwmAwA4Ojo+t23Tpk3h7u6OLl264ODBg6W2VSgUkMvlGhMRkS77888TyMjJR60aVujSwEXsOETVTrUpgARBwNSpU9GuXTs0bty4xHbu7u5Yvnw5IiIisGXLFtSvXx9dunRBVFRUiduEh4fDzs5OPXl5eVXGIRARaUV+rgKd33wFu1dOxFRfIxgZScSORFTtSARBEMQOURbjx4/Hzp07cfToUXh6epZr2z59+kAikWDHjh3FrlcoFFAoFOp5uVwOLy8vyGQyjXFERES64MwX36PFx5PxwNoB1slJMLcpfUwkkaGQy+Wws7Mr0/d3tegBmjhxInbs2IGDBw+Wu/gBgODgYMTHx5e4XiqVwtbWVmMiItJFgkoFp6XfAQDiXx3N4oeognS6ABIEARMmTMCWLVvw999/w8/Pr0L7OX/+PNzd3bWcjoio6sX8uhl+KTeRbWoO/zkzxI5DVG3p9FVg48ePx/r167F9+3bY2NggNTUVAGBnZwcLCwsAwKxZs3D37l2sWbMGALBo0SL4+vqiUaNGyMvLw9q1axEREYGIiAjRjoOISBsElQrSLz8HAFzsNQTBNTn4maiidLoA+vHHHwEAnTp10li+cuVKjBo1CgCQkpKCxMRE9bq8vDxMmzYNd+/ehYWFBRo1aoSdO3eiZ8+eVRWbiKhSXPx5I4JuXUaOiRR1vvlM7DhE1Vq1GQRdlcoziIqIqCoIgoDIkN4IPbkLJwaMRvAfK8SORKRzyvP9rdM9QEREVGj/5Xt4q+M76OXTEp/PfVPsOETVnk4PgiYiIkClErAwMh6QSOA7+lU4+pX/algi0sQCiIhIxx1avwt3E5JhLTXB2Pa1xI5DpBd4CoyISIdlP5Kh8fhROJifhwOL1sDe0kzsSER6gT1AREQ67MKUj+EifwCFuSX6vtZV7DhEeoMFEBGRjrp3OR5N1i8HAKR8+Cnv+kykRSyAiIh0VNLb78GiQIG42oFoNpVXfhFpEwsgIiIddGnVH2hxdCdUkMDku0WQGPHPNZE28TeKiEjH5DySw/H9SQCA071eQ91enUVORKR/eBUYEZGOWXLgKmp6BqCzSolGqxaLHYdIL7EAIiLSITF3ZFh6Ng2qsElYPaAeOjo7iB2JSC/xFBgRkY7IlmdiyoazUAlA70B3dGxZV+xIRHqLBRARkY6I7T8cny6ZikbIwGf9Gosdh0iv8RQYEZEOODv/R7Q8uA0qSDAvwAKOVrzjM1FlYg8QEZHI7p6JQd1PpgMATr76NhoP6ydyIiL9xwKIiEhE8nsPUNC7D2wVWbji1xgtVi4SOxKRQWABREQkEmV+ARK69YPPvdtIs3WG854dMDWXih2LyCCwACIiEsk/o6Yg6OIx5JhIIdvwB5zr+YkdichgsAAiIhLB0kPX8bF1EO7YuuDylwtRt2dHsSMRGRReBUZEVMVWH7uF+XuuAg4e2LtxP8aEBYodicjgsAAiIqpCJz+aj4NXs4HaLTHppToY062+2JGIDBILICKiKiCoVDjx9gyE/LIAzYyMsXLBJowNrSd2LCKDxQKIiKiSqQqUOPXKKIT8uRYAcGbgaIyd9DIkEonIyYgMFwdBExFVItndNMQ074Dgf4ufExM/Qsim5ZAY8c8vkZj4G0hEVElu7P8HmYFNEHTxGHJNzHDmqx8Q/P3nYsciIvAUGBGR1ilVAn45chMP56/CrIcpSHZ0R/b6jWjRvYPY0YjoXyyAiIi06HrSA8z48yrOJT6GUdM+CJAWoP1P4fDwdBM7GhE9hafAiIi0QHYnFSf6vA6ToCDE3kyDjdQEcwc1Ra8/f4Udix8incMeICKiF5CRlo5LH81Fw7XLEJyTAQCYmh2H3h+9h5r2FiKnI6KSsAAiIqqAtLgbuPnFt/DfsgYhuZkAgAQ3P2SGz8PbowaJnI6InocFEBFRGeUVqHD0+n3sPHAR4ZN6IlhVAAC47eKNtAlT0XTGOzCRmomckojKggUQEVEpsh/JcG3jDtyNOoUP6vdGRm5h0dPHJwguJkrkvv0umkweDR8TY5GTElF5sAAiInpKVvpj3N5/BLLdkbA7HoU6Ny6hiaoAgZBgzvhWsHB3R69Ad3i+sxt1fGqIHZeIKqhaFEBLly7F119/jZSUFDRq1AiLFi1C+/btS2x/+PBhTJ06FbGxsfDw8MCMGTMwbty4KkxMRLquQJGHlOg4XDezQ3yGEldTMxG4YhFe378G/oJKo22qXQ3cbheK5UObICi4MYyM+AgLoupO5wugTZs2YcqUKVi6dCnatm2LZcuWISwsDJcvX4a3t3eR9gkJCejZsyfGjh2LtWvX4p9//sG7776LGjVqYMCAASIcARFVpbwcBWS5BZDlC5Dl5CM3JhYmUYegSroDk5RkWKSlwPbBPbilp8BLVYD3hs3DGc9GAAAjY1sYCyqk2TrjToMmyO/cGR4v94JnywC48dEVRHpFIgiCIHaI0rRu3RrNmjXDjz/+qF7WsGFD9O/fH+Hh4UXaz5w5Ezt27EBcXJx62bhx43DhwgUcP368TK8pl8thZ2cHmUwGW1vbFz+IfykKlLifngHj1JQi6578FFS2thDsHQpn8vNhnHy3xP0JNjZQOjgVzhQUwCQpsWgbFO5YZW0NVQ2Xf19EBZNbCSjpJy9YWaPA1VUdzOTm9eLDAlBZWkHp7qGeN7kRj5J2rDK3REFNT/W86Y14QKksPoNUigIfX/XLmd68Dkl+QTHHBwhSM+T71vpvvzevQ6LIQ3EpBFNT5Neuq35fTBNuwCg3V+PYnsQXTEyQV6/Bf/tNuAFJVnaxeWFkhNyG/v+1vZUA40z507vVkNs46L+2t2/BWP64+P0CyPEPBP798jVNvAWTR4/+2+8zR5nTKAAwKfx/jUliIkwePih2nwKA3AaNIEilhfu9kwjT+2n/Zi36zuU0bATB3AICANPkOzAt5jP8JEpO/YZQWVkX7jc1GWZ37wAQoMpXQpWfD6EgH6r8Agh5BUhv0gK5NvYoUKlgef0q7C6eBwoKIBTkQ8gvKPx3Xh6QnY3oTn2R4uyBnDwlfM8eRZt9m2GiyIWJIhfSnCxYZslhm50By/xcDB/8GY74NQMAvBa9B+F7Fxf7PuSamOGb4R8htWsv1HGxRhNbCRrbm8C5nl+x7YlIt5Xn+1une4Dy8vJw9uxZfPDBBxrLu3XrhmPHjhW7zfHjx9GtWzeNZd27d8eKFSuQn58PU1PTItsoFAooFAr1vFwuL9JGG2KT5Zj16Xrs/XVCiW1+bD0Q8zqNAgB4P0pB1PKxJbZd1aw35oQWntqrkfkIp5cML7HtpoBQzOw5GQBgpchG7KLBJbb9s0F7TOw3EwBgpFLi5tf9SmwbWbsl3hw4Wz1/9ZuXIVXmF9v2H59ADHv1K/X8+e9eg0NuRrFtz7vXx8sjFqjnjy0dBY+M4r/Mrzj7oMeYJer5Az+PQ+2Hd4pte9veDS+9/Yt6/s9VkxFw70axbe9b2aPlhLXq+c3rZqDVncvFts00s0Dj935Xz6/e/Ak6Jpwrtq1SYoTaM3ao53/c+hXCrhX/eQaA+u9vgcKk8Mqib/9agFdiD5bYtsmk9XhsUfhL/+XexRgWvafEtiHvrESKbeEYlv/9/QvGnt5WYtsub/6IG05eAICpUb9h0vFNJbbtPXIRLrnVAQC8c+J3zDy8usS2A5/qfXnjzHbMPvBziW1/VLggqlZzAMCQC1cx/nxUiW3tcjNhY24Ce0tTKBs2xPn7HaFwdYdQsyZMvL1g4esNp6CGcPWvi484eJnIIOl0AfTgwQMolUq4PumN+JerqytSU1OL3SY1NbXY9gUFBXjw4AHc3d2LbBMeHo5PP/1Ue8FLYCSRwMzECDkm0pIbmZrCwrTwD7KZqQmyTM0BAMWOODAzg6WZMSQALMyMkWVW8k3XBKkU1tLCH7clTJAhtSyxrVJqDhupCSABjFQSyKVWJbbNN7eErfl/H6MMc2vkllAA5Zlbwt7yvwI0w9IakqcOTHhqJsfSGg5Ptc20tMWjEvabbWUDJ6v/Lj3OtLJFusK+2LaZVnZwtn7SVoJsazs8yHIo0k4CQGZlhxo2//2ssm3scd/Gsdj95piZw8VGqj6eHFt7pNk4FdtWMDKCm625el5ha497diUPpnWzNUe+aWHmfDsHpBZp+9/75mZvCSuLws+B0t4BKfauKImLgyWMbf/9zNg7INmh5LsVOztaI8+xsK3EyRF3HYv+Hj1Rw9EaPk6Fny8TRwfccSrsIVQZGUNlbAylkQlUxoX/9vOqAWNfR5gYS2CTXR8X7reBytgEgrExBGMTCCYmEExNoTK3QLuOQWhStw6kpsZwC5DiZBMPGFtZwsjSEiZ2NjB3rQFL1xqwca+B71ydYWz65HP5EoB3S8xLRIZJp0+BJScno2bNmjh27BhCQkLUy7/88kv89ttvuHLlSpFt6tWrhzfeeAOzZs1SL/vnn3/Qrl07pKSkwM2t6B/54nqAvLy8tH4KjIiIiCqP3pwCc3Z2hrGxcZHenrS0tCK9PE+4ubkV297ExAROTsX/j1wqlUIqLaVXhoiIiPSKTl/WYGZmhubNm2P//v0ay/fv3482bdoUu01ISEiR9vv27UOLFi2KHf9DREREhkenCyAAmDp1Kn755Rf8+uuviIuLw3vvvYfExET1fX1mzZqFESNGqNuPGzcOt2/fxtSpUxEXF4dff/0VK1aswLRp08Q6BCIiItIxOn0KDACGDBmC9PR0fPbZZ0hJSUHjxo2xa9cu+Pj4AABSUlKQmPjf5d9+fn7YtWsX3nvvPSxZsgQeHh74/vvveQ8gIiIiUtPpQdBiqaz7ABEREVHlKc/3t86fAiMiIiLSNhZAREREZHBYABEREZHBYQFEREREBocFEBERERkcFkBERERkcFgAERERkcFhAUREREQGhwUQERERGRydfxSGGJ7cHFsul4uchIiIiMrqyfd2WR5ywQKoGBkZGQAALy8vkZMQERFReWVkZMDOzq7UNnwWWDFUKhWSk5NhY2MDiUSi1X3L5XJ4eXkhKSlJL58zpu/HB+j/MfL4qj99P0YeX/VXWccoCAIyMjLg4eEBI6PSR/mwB6gYRkZG8PT0rNTXsLW11dsPNqD/xwfo/zHy+Ko/fT9GHl/1VxnH+Lyenyc4CJqIiIgMDgsgIiIiMjgsgKqYVCrF7NmzIZVKxY5SKfT9+AD9P0YeX/Wn78fI46v+dOEYOQiaiIiIDA57gIiIiMjgsAAiIiIig8MCiIiIiAwOCyAiIiIyOCyARNS3b194e3vD3Nwc7u7uGD58OJKTk8WOpRW3bt3CmDFj4OfnBwsLC9SuXRuzZ89GXl6e2NG06ssvv0SbNm1gaWkJe3t7seO8sKVLl8LPzw/m5uZo3rw5jhw5InYkrYmKikKfPn3g4eEBiUSCbdu2iR1Jq8LDw9GyZUvY2NjAxcUF/fv3x9WrV8WOpVU//vgjAgMD1TfPCwkJwe7du8WOVWnCw8MhkUgwZcoUsaNoxZw5cyCRSDQmNzc30fKwABJR586dsXnzZly9ehURERG4ceMGBg4cKHYsrbhy5QpUKhWWLVuG2NhYLFy4ED/99BM+/PBDsaNpVV5eHgYNGoR33nlH7CgvbNOmTZgyZQr+97//4fz582jfvj3CwsKQmJgodjStyMrKQlBQEBYvXix2lEpx+PBhjB8/HidOnMD+/ftRUFCAbt26ISsrS+xoWuPp6Ym5c+fizJkzOHPmDF566SX069cPsbGxYkfTutOnT2P58uUIDAwUO4pWNWrUCCkpKeopJiZGvDAC6Yzt27cLEolEyMvLEztKpZg/f77g5+cndoxKsXLlSsHOzk7sGC+kVatWwrhx4zSWNWjQQPjggw9ESlR5AAhbt24VO0alSktLEwAIhw8fFjtKpXJwcBB++eUXsWNoVUZGhlC3bl1h//79QseOHYXJkyeLHUkrZs+eLQQFBYkdQ409QDri4cOHWLduHdq0aQNTU1Ox41QKmUwGR0dHsWNQMfLy8nD27Fl069ZNY3m3bt1w7NgxkVLRi5DJZACgt79zSqUSGzduRFZWFkJCQsSOo1Xjx49Hr1690LVrV7GjaF18fDw8PDzg5+eHV199FTdv3hQtCwsgkc2cORNWVlZwcnJCYmIitm/fLnakSnHjxg388MMPGDdunNhRqBgPHjyAUqmEq6urxnJXV1ekpqaKlIoqShAETJ06Fe3atUPjxo3FjqNVMTExsLa2hlQqxbhx47B161b4+/uLHUtrNm7ciHPnziE8PFzsKFrXunVrrFmzBnv37sXPP/+M1NRUtGnTBunp6aLkYQGkZcUN8np2OnPmjLr99OnTcf78eezbtw/GxsYYMWIEBB2+OXd5jw8AkpOT0aNHDwwaNAhvvvmmSMnLriLHqC8kEonGvCAIRZaR7pswYQIuXryIDRs2iB1F6+rXr4/o6GicOHEC77zzDkaOHInLly+LHUsrkpKSMHnyZKxduxbm5uZix9G6sLAwDBgwAAEBAejatSt27twJAFi9erUoeUxEeVU9NmHCBLz66qultvH19VX/29nZGc7OzqhXrx4aNmwILy8vnDhxQme7dMt7fMnJyejcuTNCQkKwfPnySk6nHeU9Rn3g7OwMY2PjIr09aWlpRXqFSLdNnDgRO3bsQFRUFDw9PcWOo3VmZmaoU6cOAKBFixY4ffo0vvvuOyxbtkzkZC/u7NmzSEtLQ/PmzdXLlEoloqKisHjxYigUChgbG4uYULusrKwQEBCA+Ph4UV6fBZCWPSloKuJJz49CodBmJK0qz/HdvXsXnTt3RvPmzbFy5UoYGVWPDscX+RlWV2ZmZmjevDn279+Pl19+Wb18//796Nevn4jJqKwEQcDEiROxdetWHDp0CH5+fmJHqhKCIOj038zy6NKlS5Grot544w00aNAAM2fO1KviByj8rouLi0P79u1FeX0WQCI5deoUTp06hXbt2sHBwQE3b97EJ598gtq1a+ts7095JCcno1OnTvD29sY333yD+/fvq9eJed8HbUtMTMTDhw+RmJgIpVKJ6OhoAECdOnVgbW0tbrhymjp1KoYPH44WLVqoe+wSExP1ZtxWZmYmrl+/rp5PSEhAdHQ0HB0d4e3tLWIy7Rg/fjzWr1+P7du3w8bGRt2bZ2dnBwsLC5HTaceHH36IsLAweHl5ISMjAxs3bsShQ4ewZ88esaNphY2NTZExW0/GiOrDWK5p06ahT58+8Pb2RlpaGr744gvI5XKMHDlSnEBiXoJmyC5evCh07txZcHR0FKRSqeDr6yuMGzdOuHPnjtjRtGLlypUCgGInfTJy5Mhij/HgwYNiR6uQJUuWCD4+PoKZmZnQrFkzvbqE+uDBg8X+rEaOHCl2NK0o6fdt5cqVYkfTmtGjR6s/nzVq1BC6dOki7Nu3T+xYlUqfLoMfMmSI4O7uLpiamgoeHh7CK6+8IsTGxoqWRyIIOjziloiIiKgSVI9BGURERERaxAKIiIiIDA4LICIiIjI4LICIiIjI4LAAIiIiIoPDAoiIiIgMDgsgIiIiMjgsgIiIiMjgsAAiojLr1KkTpkyZInaMYqWnp8PFxQW3bt0CABw6dAgSiQSPHz+u1Net6OusWrUK9vb25dqmZcuW2LJlS7m2IaLisQAiItGkpKRg6NChqF+/PoyMjEosriIiIuDv7w+pVAp/f39s3bq1SJvw8HD06dMHvr6+lRtaRB9//DE++OADqFQqsaMQVXssgIhINAqFAjVq1MD//vc/BAUFFdvm+PHjGDJkCIYPH44LFy5g+PDhGDx4ME6ePKluk5OTgxUrVuDNN9+squii6NWrF2QyGfbu3St2FKJqjwUQEVXIo0ePMGLECDg4OMDS0hJhYWGIj4/XaPPzzz/Dy8sLlpaWePnll/Htt99qnPbx9fXFd999hxEjRsDOzq7Y11m0aBFCQ0Mxa9YsNGjQALNmzUKXLl2waNEidZvdu3fDxMQEISEhJeZNT0/Ha6+9Bk9PT1haWiIgIAAbNmzQaNOpUydMnDgRU6ZMgYODA1xdXbF8+XJkZWXhjTfegI2NDWrXro3du3cX2f8///yDoKAgmJubo3Xr1oiJidFYv2rVKnh7e6vfi/T0dI31N27cQL9+/eDq6gpra2u0bNkSkZGRGm2MjY3Rs2fPIrmJqPxYABFRhYwaNQpnzpzBjh07cPz4cQiCgJ49eyI/Px9AYUEwbtw4TJ48GdHR0QgNDcWXX35Z7tc5fvw4unXrprGse/fuOHbsmHo+KioKLVq0KHU/ubm5aN68Of766y9cunQJb731FoYPH67RkwQAq1evhrOzM06dOoWJEyfinXfewaBBg9CmTRucO3cO3bt3x/Dhw5Gdna2x3fTp0/HNN9/g9OnTcHFxQd++fdXvxcmTJzF69Gi8++67iI6ORufOnfHFF19obJ+ZmYmePXsiMjIS58+fR/fu3dGnTx8kJiZqtGvVqhWOHDlStjePiEom2nPoiaja6dixozB58mTh2rVrAgDhn3/+Ua978OCBYGFhIWzevFkQBEEYMmSI0KtXL43thw0bJtjZ2ZW672eZmpoK69at01i2bt06wczMTD3fr18/YfTo0RptDh48KAAQHj16VOLx9OzZU3j//fc1MrRr1049X1BQIFhZWQnDhw9XL0tJSREACMePH9d4nY0bN6rbpKenCxYWFsKmTZsEQRCE1157TejRo4fGaw8ZMqTE9+IJf39/4YcfftBYtn37dsHIyEhQKpWlbktEpWMPEBGVW1xcHExMTNC6dWv1MicnJ9SvXx9xcXEAgKtXr6JVq1Ya2z07X1YSiURjXhAEjWU5OTkwNzcvdR9KpRJffvklAgMD4eTkBGtra+zbt69ID0tgYKD638bGxnByckJAQIB6maurKwAgLS1NY7unT785OjpqvBdxcXFFTs89O5+VlYUZM2bA398f9vb2sLa2xpUrV4rks7CwgEqlgkKhKPV4iah0JmIHIKLqRxCEEpc/KUyeLVJK2640bm5uSE1N1ViWlpamLkQAwNnZGY8ePSp1PwsWLMDChQuxaNEiBAQEwMrKClOmTEFeXp5GO1NTU415iUSisezJMZXlSqyn34vnmT59Ovbu3YtvvvkGderUgYWFBQYOHFgk38OHD2FpaQkLC4vn7pOISsYeICIqN39/fxQUFGiMn0lPT8e1a9fQsGFDAECDBg1w6tQpje3OnDlT7tcKCQnB/v37NZbt27cPbdq0Uc83bdoUly9fLnU/R44cQb9+/fD6668jKCgItWrVKjJo+0WcOHFC/e9Hjx7h2rVraNCgAYDC9+vp9c+2f5Jv1KhRePnllxEQEAA3Nzf1PY2edunSJTRr1kxruYkMFQsgIiq3unXrol+/fhg7diyOHj2KCxcu4PXXX0fNmjXRr18/AMDEiROxa9cufPvtt4iPj8eyZcuwe/fuIr1C0dHRiI6ORmZmJu7fv4/o6GiNYmby5MnYt28f5s2bhytXrmDevHmIjIzUuGdQ9+7dERsbW2ovUJ06dbB//34cO3YMcXFxePvtt4v0LL2Izz77DAcOHMClS5cwatQoODs7o3///gCASZMmYc+ePZg/fz6uXbuGxYsXY8+ePUXybdmyBdHR0bhw4QKGDh1abC/TkSNHigwKJ6LyYwFERBWycuVKNG/eHL1790ZISAgEQcCuXbvUp4vatm2Ln376Cd9++y2CgoKwZ88evPfee0XG6jRt2hRNmzbF2bNnsX79ejRt2hQ9e/ZUr2/Tpg02btyIlStXIjAwEKtWrcKmTZs0xh8FBASgRYsW2Lx5c4l5P/74YzRr1gzdu3dHp06d4Obmpi5QtGHu3LmYPHkymjdvjpSUFOzYsQNmZmYAgODgYPzyyy/44Ycf0KRJE+zbtw8fffSRxvYLFy6Eg4MD2rRpgz59+qB79+5Fenru3r2LY8eO4Y033tBabiJDJREqclKeiKgCxo4diytXrlTKZdy7du3CtGnTcOnSJRgZ6ef/7aZPnw6ZTIbly5eLHYWo2uMgaCKqNN988w1CQ0NhZWWF3bt3Y/Xq1Vi6dGmlvFbPnj0RHx+Pu3fvwsvLq1JeQ2wuLi6YNm2a2DGI9AJ7gIio0gwePBiHDh1CRkYGatWqhYkTJ2LcuHFixyIiYgFEREREhkc/T5QTERERlYIFEBERERkcFkBERERkcFgAERERkcFhAUREREQGhwUQERERGRwWQERERGRwWAARERGRwfk/uSp9OVlaRXAAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -201,7 +201,7 @@ "# Generate the data.\n", "nsamples = 100\n", "x = np.random.randn(nsamples)\n", - "y = 3*x**2 + np.random.randn(nsamples)\n", + "y = 3*x**2# + np.random.randn(nsamples)\n", "\n", "## Cross-validation on Ridge regression using KFold only\n", "\n", @@ -213,7 +213,7 @@ "lambdas = np.logspace(-3, 5, nlambdas)\n", "\n", "# Initialize a KFold instance\n", - "k = 10\n", + "k = 5\n", "kfold = KFold(n_splits = k)\n", "\n", "# Perform the cross-validation to estimate MSE\n", diff --git a/doc/src/Projects/2023/Project1/Project1.do.txt b/doc/src/Projects/2023/Project1/Project1.do.txt index a1053a8f2..12bfb642d 100644 --- a/doc/src/Projects/2023/Project1/Project1.do.txt +++ b/doc/src/Projects/2023/Project1/Project1.do.txt @@ -324,13 +324,13 @@ term which measures the deviation from the true data and the mean value of the m That is, show that !bt \[ -\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2, +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, \] !et with !bt \[ -\mathrm{Bias}[y]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right], +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right], \] !et and diff --git a/doc/src/week38/exercisesweek38.do.txt b/doc/src/week38/exercisesweek38.do.txt index a04146c53..5134c86e1 100644 --- a/doc/src/week38/exercisesweek38.do.txt +++ b/doc/src/week38/exercisesweek38.do.txt @@ -42,13 +42,13 @@ term which measures the deviation from the true data and the mean value of the m That is, show that !bt \[ -\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2, +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2, \] !et with !bt \[ -\mathrm{Bias}[y]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right], +\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right], \] !et and