From 47843ca4804826628a18a0ba21c3785151f4a42d Mon Sep 17 00:00:00 2001
From: Morten Hjorth-Jensen
Date: Tue, 19 Sep 2023 09:50:35 +0200
Subject: [PATCH] typo in p1
---
.../2023/Project1/html/._Project1-bs000.html | 6 +-
.../2023/Project1/html/Project1-bs.html | 6 +-
doc/Projects/2023/Project1/html/Project1.html | 6 +-
.../2023/Project1/ipynb/Project1.ipynb | 106 +-
.../Project1/ipynb/ipynb-Project1-src.tar.gz | Bin 193 -> 193 bytes
doc/Projects/2023/Project1/pdf/Project1.p.tex | 6 +-
doc/Projects/2023/Project1/pdf/Project1.pdf | Bin 270379 -> 270582 bytes
doc/Projects/2023/Project1/pdf/Project1.tex | 6 +-
doc/pub/week37/ipynb/week37.ipynb | 966 ++++++------------
doc/pub/week38/ipynb/week38.ipynb | 755 ++++----------
.../Projects/2023/Project1/Project1.do.txt | 6 +-
11 files changed, 599 insertions(+), 1264 deletions(-)
diff --git a/doc/Projects/2023/Project1/html/._Project1-bs000.html b/doc/Projects/2023/Project1/html/._Project1-bs000.html
index 5d0c5bb1e..24af4358b 100644
--- a/doc/Projects/2023/Project1/html/._Project1-bs000.html
+++ b/doc/Projects/2023/Project1/html/._Project1-bs000.html
@@ -489,17 +489,17 @@ term which measures the deviation from the true data and the mean value of the m
That is, show that
$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=(\mathrm{Bias}[\tilde{y}])^2+\mathrm{var}[\tilde{f}]+\sigma^2,
+\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2,
$$
with
$$
-(\mathrm{Bias}[\tilde{y}])^2=\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2,
+\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right],
$$
and
$$
-\mathrm{var}[\tilde{f}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
+\mathrm{var}[\tilde{y}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
$$
The answer to this exercise should be included in the theory part of the report. This exercise is also part of the weekly exercises of week 37.
diff --git a/doc/Projects/2023/Project1/html/Project1-bs.html b/doc/Projects/2023/Project1/html/Project1-bs.html
index 5d0c5bb1e..24af4358b 100644
--- a/doc/Projects/2023/Project1/html/Project1-bs.html
+++ b/doc/Projects/2023/Project1/html/Project1-bs.html
@@ -489,17 +489,17 @@ term which measures the deviation from the true data and the mean value of the m
That is, show that
$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=(\mathrm{Bias}[\tilde{y}])^2+\mathrm{var}[\tilde{f}]+\sigma^2,
+\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2,
$$
with
$$
-(\mathrm{Bias}[\tilde{y}])^2=\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2,
+\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right],
$$
and
$$
-\mathrm{var}[\tilde{f}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
+\mathrm{var}[\tilde{y}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
$$
The answer to this exercise should be included in the theory part of the report. This exercise is also part of the weekly exercises of week 37.
diff --git a/doc/Projects/2023/Project1/html/Project1.html b/doc/Projects/2023/Project1/html/Project1.html
index 09ea2b53f..41ec77b79 100644
--- a/doc/Projects/2023/Project1/html/Project1.html
+++ b/doc/Projects/2023/Project1/html/Project1.html
@@ -525,17 +525,17 @@ term which measures the deviation from the true data and the mean value of the m
That is, show that
$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=(\mathrm{Bias}[\tilde{y}])^2+\mathrm{var}[\tilde{f}]+\sigma^2,
+\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2,
$$
with
$$
-(\mathrm{Bias}[\tilde{y}])^2=\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2,
+\mathrm{Bias}[y]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right],
$$
and
$$
-\mathrm{var}[\tilde{f}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
+\mathrm{var}[\tilde{y}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
$$
The answer to this exercise should be included in the theory part of the report. This exercise is also part of the weekly exercises of week 37.
diff --git a/doc/Projects/2023/Project1/ipynb/Project1.ipynb b/doc/Projects/2023/Project1/ipynb/Project1.ipynb
index 2ecd55826..467bad587 100644
--- a/doc/Projects/2023/Project1/ipynb/Project1.ipynb
+++ b/doc/Projects/2023/Project1/ipynb/Project1.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "markdown",
- "id": "6060626f",
+ "id": "7424fea1",
"metadata": {
"editable": true
},
@@ -14,7 +14,7 @@
},
{
"cell_type": "markdown",
- "id": "fd9f7098",
+ "id": "c9ca1d90",
"metadata": {
"editable": true
},
@@ -27,7 +27,7 @@
},
{
"cell_type": "markdown",
- "id": "75f7ac9b",
+ "id": "3e4164b6",
"metadata": {
"editable": true
},
@@ -63,7 +63,7 @@
},
{
"cell_type": "markdown",
- "id": "988b2436",
+ "id": "dd4dcdf5",
"metadata": {
"editable": true
},
@@ -85,7 +85,7 @@
},
{
"cell_type": "markdown",
- "id": "0725f6b3",
+ "id": "71ddc478",
"metadata": {
"editable": true
},
@@ -100,7 +100,7 @@
},
{
"cell_type": "markdown",
- "id": "de583594",
+ "id": "d4184afb",
"metadata": {
"editable": true
},
@@ -133,7 +133,7 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "130a5d79",
+ "id": "62880fe6",
"metadata": {
"collapsed": false,
"editable": true
@@ -185,7 +185,7 @@
},
{
"cell_type": "markdown",
- "id": "9bf030f4",
+ "id": "6f7e7c97",
"metadata": {
"editable": true
},
@@ -207,7 +207,7 @@
},
{
"cell_type": "markdown",
- "id": "3be47d8d",
+ "id": "7377112f",
"metadata": {
"editable": true
},
@@ -220,7 +220,7 @@
},
{
"cell_type": "markdown",
- "id": "f7f14618",
+ "id": "72f83fe7",
"metadata": {
"editable": true
},
@@ -232,7 +232,7 @@
},
{
"cell_type": "markdown",
- "id": "913d72b1",
+ "id": "7a3578ff",
"metadata": {
"editable": true
},
@@ -244,7 +244,7 @@
},
{
"cell_type": "markdown",
- "id": "7a403d11",
+ "id": "e0aef819",
"metadata": {
"editable": true
},
@@ -254,7 +254,7 @@
},
{
"cell_type": "markdown",
- "id": "108b7a27",
+ "id": "c7f751af",
"metadata": {
"editable": true
},
@@ -266,7 +266,7 @@
},
{
"cell_type": "markdown",
- "id": "f1e64231",
+ "id": "c026a4bd",
"metadata": {
"editable": true
},
@@ -295,7 +295,7 @@
},
{
"cell_type": "markdown",
- "id": "30bca1e7",
+ "id": "2635bcbd",
"metadata": {
"editable": true
},
@@ -313,7 +313,7 @@
},
{
"cell_type": "markdown",
- "id": "ae6a3eeb",
+ "id": "92892df2",
"metadata": {
"editable": true
},
@@ -330,7 +330,7 @@
},
{
"cell_type": "markdown",
- "id": "5ba7dff9",
+ "id": "35987aab",
"metadata": {
"editable": true
},
@@ -346,7 +346,7 @@
},
{
"cell_type": "markdown",
- "id": "84ca72c8",
+ "id": "e61048c2",
"metadata": {
"editable": true
},
@@ -358,7 +358,7 @@
},
{
"cell_type": "markdown",
- "id": "0d734865",
+ "id": "5ef456d4",
"metadata": {
"editable": true
},
@@ -369,7 +369,7 @@
},
{
"cell_type": "markdown",
- "id": "1dac6c19",
+ "id": "52e6fbad",
"metadata": {
"editable": true
},
@@ -381,7 +381,7 @@
},
{
"cell_type": "markdown",
- "id": "e04687a9",
+ "id": "db6028a0",
"metadata": {
"editable": true
},
@@ -393,7 +393,7 @@
},
{
"cell_type": "markdown",
- "id": "f99dbe69",
+ "id": "9477ad5e",
"metadata": {
"editable": true
},
@@ -405,7 +405,7 @@
},
{
"cell_type": "markdown",
- "id": "0b3e0564",
+ "id": "8c2aeccd",
"metadata": {
"editable": true
},
@@ -416,7 +416,7 @@
},
{
"cell_type": "markdown",
- "id": "f660901d",
+ "id": "6a774b74",
"metadata": {
"editable": true
},
@@ -428,7 +428,7 @@
},
{
"cell_type": "markdown",
- "id": "7a1c7f34",
+ "id": "98fde9b7",
"metadata": {
"editable": true
},
@@ -441,7 +441,7 @@
},
{
"cell_type": "markdown",
- "id": "becad040",
+ "id": "87a4d11d",
"metadata": {
"editable": true
},
@@ -453,7 +453,7 @@
},
{
"cell_type": "markdown",
- "id": "242ede2f",
+ "id": "b4844f1e",
"metadata": {
"editable": true
},
@@ -463,7 +463,7 @@
},
{
"cell_type": "markdown",
- "id": "d4999c5f",
+ "id": "f7662ade",
"metadata": {
"editable": true
},
@@ -475,7 +475,7 @@
},
{
"cell_type": "markdown",
- "id": "46386eef",
+ "id": "1d61362e",
"metadata": {
"editable": true
},
@@ -486,7 +486,7 @@
},
{
"cell_type": "markdown",
- "id": "9f27e396",
+ "id": "74bbb4ab",
"metadata": {
"editable": true
},
@@ -519,7 +519,7 @@
},
{
"cell_type": "markdown",
- "id": "902b759d",
+ "id": "ac939724",
"metadata": {
"editable": true
},
@@ -531,7 +531,7 @@
},
{
"cell_type": "markdown",
- "id": "41a0b8d7",
+ "id": "4066d012",
"metadata": {
"editable": true
},
@@ -550,7 +550,7 @@
},
{
"cell_type": "markdown",
- "id": "89cced85",
+ "id": "7e5c5ef4",
"metadata": {
"editable": true
},
@@ -562,7 +562,7 @@
},
{
"cell_type": "markdown",
- "id": "22bb8b90",
+ "id": "690337b9",
"metadata": {
"editable": true
},
@@ -576,19 +576,19 @@
},
{
"cell_type": "markdown",
- "id": "e88ba22d",
+ "id": "92dabf3c",
"metadata": {
"editable": true
},
"source": [
"$$\n",
- "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=(\\mathrm{Bias}[\\tilde{y}])^2+\\mathrm{var}[\\tilde{f}]+\\sigma^2,\n",
+ "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[y]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n",
"$$"
]
},
{
"cell_type": "markdown",
- "id": "e2156ab0",
+ "id": "7da90966",
"metadata": {
"editable": true
},
@@ -598,19 +598,19 @@
},
{
"cell_type": "markdown",
- "id": "7d08fd36",
+ "id": "33a81d54",
"metadata": {
"editable": true
},
"source": [
"$$\n",
- "(\\mathrm{Bias}[\\tilde{y}])^2=\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2,\n",
+ "\\mathrm{Bias}[y]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n",
"$$"
]
},
{
"cell_type": "markdown",
- "id": "5cdcf0c6",
+ "id": "abb9bd28",
"metadata": {
"editable": true
},
@@ -620,19 +620,19 @@
},
{
"cell_type": "markdown",
- "id": "3235550b",
+ "id": "5b4668fd",
"metadata": {
"editable": true
},
"source": [
"$$\n",
- "\\mathrm{var}[\\tilde{f}]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n",
+ "\\mathrm{var}[\\tilde{y}]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n",
"$$"
]
},
{
"cell_type": "markdown",
- "id": "813e8659",
+ "id": "b453f8fc",
"metadata": {
"editable": true
},
@@ -651,7 +651,7 @@
},
{
"cell_type": "markdown",
- "id": "adf1396f",
+ "id": "6fe323b0",
"metadata": {
"editable": true
},
@@ -676,7 +676,7 @@
},
{
"cell_type": "markdown",
- "id": "a7cb0fd1",
+ "id": "5fd2f486",
"metadata": {
"editable": true
},
@@ -704,7 +704,7 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "3be6c174",
+ "id": "e88e39ea",
"metadata": {
"collapsed": false,
"editable": true
@@ -716,7 +716,7 @@
},
{
"cell_type": "markdown",
- "id": "9544f6e7",
+ "id": "29817da1",
"metadata": {
"editable": true
},
@@ -728,7 +728,7 @@
{
"cell_type": "code",
"execution_count": 3,
- "id": "c2dc1c15",
+ "id": "560fc9f4",
"metadata": {
"collapsed": false,
"editable": true
@@ -754,7 +754,7 @@
},
{
"cell_type": "markdown",
- "id": "81854928",
+ "id": "3fc05383",
"metadata": {
"editable": true
},
@@ -779,7 +779,7 @@
},
{
"cell_type": "markdown",
- "id": "9a8a9dc4",
+ "id": "bb21261e",
"metadata": {
"editable": true
},
@@ -793,7 +793,7 @@
},
{
"cell_type": "markdown",
- "id": "df82eef8",
+ "id": "c5ab9dbf",
"metadata": {
"editable": true
},
@@ -823,7 +823,7 @@
},
{
"cell_type": "markdown",
- "id": "ba32d247",
+ "id": "7d0dbb14",
"metadata": {
"editable": true
},
@@ -845,7 +845,7 @@
},
{
"cell_type": "markdown",
- "id": "cb0788db",
+ "id": "e301049d",
"metadata": {
"editable": true
},
diff --git a/doc/Projects/2023/Project1/ipynb/ipynb-Project1-src.tar.gz b/doc/Projects/2023/Project1/ipynb/ipynb-Project1-src.tar.gz
index a86558f6cdb0f816ae2c15ec1fbb5a827a091586..c506344768b15cc92f05a46c8e1126a120afbb7d 100644
GIT binary patch
literal 193
zcmV;y06za8iwFR`QVC@M1MSaC3c@fD2H>uHia9|^V$!SyUAPd6c!AWWHq|CINx|OU
zK0sHBn<7HK&Cf8yFmu?hH~Vek?><@#LMWva#^foR5|O!{V9Wtih6rJpa0&>o7~v71
zZ<1;4;=!DUSQ2N*K)^pSID=|
z52#boMIzYW<2uT=&DbpmLlY~n>qcjD|SQ>|El!O!rjbcVT
zAnTp<(i_Y5cuHG0LK&gn&5dJK^6OyGWbm=fI@73qeFN{)ad;4V0->%0FOvu8c)12SN3H(v
zH%LWh19CGy%s}JdtFGQkLcY_J9l=k%NYLZ72VJf^{PiQEqQ=1KUH(XU&9Gw)a|JZ>
z7%lRg3f<}9d3PG7rc7=52z_yPa26W@I(I^&BCS05%^!Dgz#l+k#qh3Yq1;qN%Glta*~AvOMT*W9W1Z+0Lz}O`6<4jB?8^{b+~ZO1
z$tj}Ms!OpoTl_`zwwTMnJDb^gez}b0#9$Pn%9ca!Vku<|wns2~F{NGyfR)+8Pv(Cf7f8Vud
zLsZ??@odfn2bqf`h-P}N;s*BLPrY^s!;69!r56hRo|b!X56`Vzk}(mBVIj+Os@`To
zJqsW3xrI^kl8j#Hyvvy1Qcb#lyXNI8GjaJPJ@c)yp-~ZZ(2Tz~*AU=?Z%OIfl*1?6
z3FPm$@K6AG(rK{Jpp(wj{Kte7tucdi0`4|!8P#G%5H_13Qrnvco~lcYbeSyPhAC1jPZ7*a~X
zN!SY5qGI?Gz(uAa$30<9YINNar+Y(=QZo}jXD@)n+H8|c$b`tszjZD8J_X;SKEK5|
zz>FJq>X_H#voB`z&;DzEd>D4?@8bnm9+uzkJw9YnDbSM
zOCTHV5nu9v_C0Ia5{~YwXXq}3;3pj`asnHh=_Gdaq&W*6J((eR!AT$J?XxVRhOg~7
z?QOP+e64plq~y*n-^+5rM<|dv&xR*d5;4u3!(^-725ZqGwP#+?omo33Sm6msxjR6<
zG4L$Lz3bV5LtgTqBVfJH?jKnUO5`a4lgH5#{bcc1%v3q%OYc-)&?9nLPXO+y>Yt<4
zoM>@>a8#e;mb3;r&=(^Od7zc$%gw5I&0uM)akK!BX<#DY%jx}#?N8U1(0Hg~GNKAf
zNpB&(fP*Vdhq|8&*)*>xm>d-%Fhzhn?Fw=`tcrBNJ2wyuZ&6k%T)<_BfMk5MFJ*{!
zz9jFX;asCQ9;48Utv{A(Vt|aWAIJ@{r$VMcV;;~GvsjF#6`tDT*4~iya|vw%O;9|$
z*tC~2(#U!~Bx{;wpTtq1p+zGmxC(QiIjuAWG1=s%Uye>Rw*w&wbBg6m}qh--a$
z+`p$)>rt#yQ0>~;!lu{_$W;7qa)QcW2tr^OEnES%$?`~VY%-!M0Dsqt?k%|X^`Y_a
zhL}7SAnzW&ll6rrYZjqrW?F!d02M;KPuAd{e>H9mmTDTp0f_>W-QWQ*g91je7?nbJ
zxp#IhPs#8n%y`YmsyfeRL=xJLYV#H#zFcDc;jO+U?>NT_W?qGG_&MI2mdefvYS
zLj+KpP-wGE2d(RQ4dHvNcKEfUQ8{#6FA9+t$b<4Eyrq;>&WuTQ8VaFj+^}hxfQ8D0<{EfZvgoT1>pu_=s6}@;6S&-5F&?=)o`Zi
z*NSMF5nSb69HinKBv7qGh(?eyVZ)4wKiKYFO>oKo6GQGy4~`BhElaT!M99G!DCkx+
zeT2VTukAMfO=*4!@k~F?E@AeR0JtS!BjqE8L0l{`S|Hv=MGjEj8(E4Iu_Qr%wZd*-
zJSWFY#R1e0R72qjklxp#vU*RFc~c_?c4u5%sjhJwFC!EAr?!aTuAJUj;XH_FYOql;
zRigI6vr(OXZisSAVh=qZSH*amcFXuy0t^~Zm{DT&lVmU`T@K#K*|c12Go;9d$H@~|
z6-*2*BdG&(#0Y7nlq(HGBHJwKbu)3@Mp)nhwi+T^nE=x;FR*H=NK!L(A;N*Zt$cPa;c3HOvT7m;Q
zY2WqE;x%A?LsaisCu}PiIowq84pCsT^gYxL=OysG++Pf-ruV~MX#Spam3If2)+wk2U0+Sc?&(FN87;&*sEWWfX<~Q%Vtg3tLXtLfO%2t*A
zGy_3Shi|UuGZ>*=bXMk0?@!%hLKG@Tep~q-iw}Ctd`752K6ZN*VK{MC(?6t~`}V%z
zM@4Z`s8nb729FRP4)8K1!rTQYvr;H!n*cUVk$7}h0za5w0;7HdXHLC|?$%$03>|l`
zk~4yGRDr`}?thRm*AR2jMKKMy*1(1y79TRbIk}wt(1Mrg!ej)0&fm_qa1T@t6gPLv
zI0?TTe7AiL+Po1%jdNSL)dI49xy&b}`fGuIU4c9ARYe3u~
zepME3+hK@BnweIHTy!j9^mJ|IB;iD%IBNSkM|L*hggn4@QjpUSnDe)jSL5?~x>oSY
zfurjHxs?Qa#vk;LbS-M`$Jv>LU^3!o=Hvd!Pu|PfucqsJ&iw=B9YvX<_>B>;S2e$7c`vH#jku;DN@CurPTEW{SyRH*nQaO}~x~
zh^Fo3MPgsitxbH$%VfuU`I#?wo7#S3iAGg{e^V!;N2m)*&
zI(`c$TpoUL@t)G$TkD`y^@8WcH_DdZV%Q*2Lk|h)CQ55=I|_R8Sx6UBH$Wv0munWE
zI+B5+zgz)S;E0n!^L!5YDCnDF?q{71aO#isa>Z-a*)sOs4x*-oYe=mQ4?%$_e6kQy
zLb5&?b!}+)*Nq_9*s!2myuQXiP|I~IZ|A!|F|qKi0FWzZXFviIc98m*6p30$!`UeDgpBy8Y7h}df!O{ku%Z`{6z3i;
zen5EMv1zP@&sKQf)%hGa{(eot)PX<{2v5w~pBQ9ZL0g!EFytD`$+J{iOr=)Sca}14
zsp~-xe4-ho*0SG_#34UEGKdfW(V18KzDl5Rn8fO){BywtDkc-D1x1{MfPR8nNYO=O
zNLIi!2(elUw``~uIm%zPu?%eZrO{uAP(?V)rb3adEf5Zl|4l7`YaJa2-0=qg)jBL9
zMraWEU55dZfI3{#)}{tdA9qqLNHq2&ELFTPy!VIOM|C$uZZdMhs3&`l0pRK^b452d
zb+_$KpK2QI%!%>|4Vv-^^?%_qBNMWk?=*i@E7YjZ%|h+*flN!Pqf=EI9z1zD&?}*TR3m_lzPm=73AGVj$kpMArjg4OsaumTQoZ
zoxrmxm0({H(j;kgGxKfS$;EI}};~`i_HS$7YVIY>dL9!`g`Envr
zH8HM0$x}rDT1GrVmimIR`Y%2nu=(wX{^7VW2cBiv$H!5QIFbl{X8$G>`JeLW~xRxH(0#87{du
zL|6rB>;o9;GwvR;&AF?Vk
zaYnL3Z^iJYjPIuZuEv3toB5u+1I@kfLQmDtn`E~D@-85cGxg2hbvY-3^19l_e7jL5
z#DipqZSuvP**@(K=f-ncS56{+Q*k*{EIhnaWZ_nU$v4pHE+OW{lEX&$7*0XkA$EtI@){yb=>1X@gUSQh%n(1pPA#PlGU;#?T#N)x28b;
zQr?dAU7}pQx!=dfXD6`KlRL8qGrspd%mOL?-*$0sWZQv*gZ|vELtU5s4;Gr6JpW%bwmW93ri2_qVv?NtG&X30U;fuRTmP#
zl{64VJVop}f>Q;4&R>l1Yql_`WDVv3bJQXo+ydX~{;^x%>X7Btb(ZIcvprQ7`vx<4
zrWSh@ELUt)os#W#?fW*1J{6)QJK4}8fz*oNtZcTYT2sSKm9OiX?BvIq31b~$Pt4#@
zwpjbV6sbcm!9oH&?CtNwZRm|(4)_9`-=foQs-C#Bqinzp?QVx#VWD=EX^bWNI11F6g}
zo~FJLz--LG>Tt_!UXztPi#9I((dsE{T^&RK7q%<(V??X$QQEje#P`8N%c%f36oI4|
z$U^_Kr5z|AN-=!;p(yjH84}aBK?{AQvZX&Ob+gqP7V;7;K}aoA3d;HsFBV%MRYo~y
zM}*%kxqgnqE4S+^va3R%i(ygZAW#Q7-?I@lBlKd{C{biun^*ed=ZRMu(2~Q5WRndp
zd05rK<*@lm0*+`#eaj|jdl?4BAFRL+mkJLhqRFv+yRh~vPf
zOby}V_VWEbWvORa-Y?#!hN^(_jeK{&oW5+eNjm_{I#4dc3j44y_YzDiQ{(Iue3y3F
zE6h?B9iNBK8?$7^W-lV30~Yo52?%GJliXajfcX-Tn>E~EZUn_GohzFAUQ4<}(yNUu
zKDl<8(sxhsK0`X;9eMz>qL6u9Pb)^i&aU7mhpnXEed(kSO~(-ekEcyCK@;(ZTa9f>
zlnwI;DG9#0^_2=GJY%3=C=yUbA8dbHVNJI-kJ0H`z8#m+Xjh`*K>D%qsa^ty#+4v(
z0j)izZI)TQuStKb7k^a_&dYK6|H@?_-zyJF=qxrEG8TwCCC~#7`XmMoFN-UF)Z9X<
z{wx(RhZz)wM3@R(;XiRGyUe+i>+MT?iuJ?>u)^5~TXJm?!@F}sN|e;Q1mO-EDN`Sn
zqczeoMw>(P0j4%D$9C8P}mDJQ+43j?vx$L#=B!$zV
zjAV+m{#V%PbO{egI}WY`W%;taw|XeB$J8`-?M0g?I7yd&cMZ8sL}}R4Fp*6))P?X7
z!AfSYXgy!;F9>t6;vpA9m{2aHLgpUB@0=36WN0a0A|e+*-j9N^DK=g!OFvIvF9~b+
zaNQ82?^3gkINPyf)cu$y^T*yF(_GgYx
zy>iMfQaQweMHHIQ(gSM`fgcJaP_nPgT3>c;?8cZhzc&I=9w|nh9=Xn_V;c5k`(*kP
zRksHVMLV$Q*IuFSj0FCIzQ{PiGevg|kW0AXqNvwcUD(NPf=u)1^V;!C@yahzDcxmJ
z?~D)_(@+JRa2#*tu*6qjIWn-GHGY*}i#fp0thmb7lvLD|8jmbb;3-uXS?jD6ty^x*
zFi7Cx#B-|S`^|);**lzlMms7|kjaJ?)@^Q9UB-E3Gyci2kStxF5d^?%v{cHw`o4q@
z>nT^NA@)FmS54j@eHZiQsu)dm_^ijy#CIKBa;E?)-PM|8;>m8#O|Vih!>Lgz`fM@N
z7hgypJ^5UkXQ7_JkuY*}nG52U5c^8A0XG{XZyz?y;@XH)clPmdt?bqJK(Rdz7)9i{
zzqS%WA42B4NbR9I_K?Aq=dd~72}T*#ii2u23pWmork(F?n{AP=#JKg<(|L5b!(^D-
z0*cAsk;1XAET#;`qN_@obT0L;mCWy)EE8l7H<-c+?eyzx{L|o8K{V
z1&pbC%)JJGIQTfYEvuGf?7*n(`oa>u~DeM|^!`OAg)Ku
zCclqTMO2C$ihNTg+XMrB`5CPXW-(q%*bO$p=dP@{Gd|qnFe?DX
z0rw#2c{Ha)Vp_<3HO?Bhe8wG*cjJhV6?nl|5>J2U35Q=}IOM#ADtw4o9sLW{7%#4)
zi?A()$Y|D{Y_g~@#Ye`x3S#TuF=Nw*IIVlbX!{3S`Auc{tmGw!ZjuR=M4~_yg7;>V
z!g9)7ijhANR&Qddy@_Is5aJ&^$%26H{98hDY4D(ONOHGdGZO%&j8$C^w$YNKY_Z{Cp`a<5eGARfgeE?JD6)f72$
z0KF)8Vs-F~^V4`4F+y-5lF51;8M4HAh3C0Yl6Y&>H(g=63YZU{T4XcOP^EHHPRVes
zabzZ34~Z!2LX-St?}+szRtX?n-KOWL@44>FeLEdc=fh5AMBJ3=#d4j5vGT+=g2dCl
zKbs6a^iv}wBb7*>2%38G
zh1^awA5*awh6?E-OM8Q5jtEB(hdJ()Vm=g~6=W}!lE>%w)!z^(fQZN@#p093w`?;?
zXW5VTs8$+cJ~U2Y8{rL!S3dP8N};RXlF;0gyA)%L4;{-@8YFw}m}=H>%>nM!LfnFM
zib*+VoCzyQaH#qp2qb_loQa45{x`++WHMOQGjK?`WNwco
zx3Skl?q;%n4|Ai68=?wBJ6-0Oe9D?pnwVE4PR!to+)K!~Xo`$w`7^&U=1+4y_ckWM
zd7Sc9LFShIY`-}b(?oY8xX*P;6;oRAOepEdqS4eOQb-g~NDe?plx~P-b`CxWqR+&(
zC(C+QuxCj8%uBmNG?LP^y~BLQ*h%9qu?;^C3^oTJI^LRYM7k{asg8~o+9o$5XDC&>kr?#nu1?{vzlzj(F0CG^r?K=iro66&Q7K!LxkF%Vt3$mx%1A1Dx5o-Uw+Sa>UnMf
zS~4cDbAucQIy?HtfaAAA1rgkQdT#2z4bwzKw?{*y4HICzgzeVztU90(>vsAqAhl+O
zIgh1Ue$t1o2WB@?lf)VMsxee@;du^}Ro8~R45t#Zfntqq&<9TS$koa#*N#lV4$2H)
z9)VQ%29@ADym3rj`G|%Upd^I7;|NkLl5;~oKiidkU=g^=;zM{Vfqz?RTfu|xd;9zA
z-zvD^hzAIP2^(^4ZuJR{mHuk`_qKiTqDgSL3X&42@N;4E#Q~V8OF3wjc`gWQs(ipu
z?ONWIIObA1Qpy!=7X~o7Yo$K7VB_lOk5%`ySC&Qdb#-p1HF3`1+rOPfaJGmigZd%t
z&mDyyLjKu47DtTrsA?tnInJWBF)2SKp2ArR^#Zs-c=ve#xbA|wGV;yn#83PbvbeWW
z-Jv|9TkK@JlAYs&g-S^AA+cn~>Drn=2|-zN;Y4M+v$aJh!{bXJp}Fs*iSdKxB9Xtt
z^=5A$BLAd#xyX1@937&M6j9&XC`@;jC6B^exaQ5)sVVXwG7nP(<(khOXTB-Hm*j3o
zH~~0P#D4dAZVNiZpHr3H?^?^#KhBIBzv5_HS2EsvQnN1k)jWJ1JN=Wv$Lzvfr8477
zAvXJc^%95)FXvxKRc;Jkk2M4~ia-V1oGe=fUM6qhEE>1`&jO7RS$l|Gb(>Q2e9One
z^WPUpf)3TDp9xRK_)IdYrYziJ1`Z{5_Eo(cJa-_Pqsg#Nz|%h@mH
zY^o1WPP{$sI{LU)%&67b-2~NHfx~16t9sLt2MwMic~PeaI`<-9+TCb%!v@112eF>C
zy_mnCeqUq~2f8sM?c|{`o4VQAiS0)3L
z0AR{jU8nShV^Gw#B4*$m3nXrKCKyIJGkXhHOBNy)HjZ?~OkfO94(9*)ou?;c0)qp5
zaO-rJT*o!Kr6k>%raeuV=`ISNo2$_|s0*5*I5);n
zQZYTa?pY;oKa&cRzn#{ZCk3PWX0DKfiiU*W|ksSxC`6)EQn5!uU_z)lJx-FI<@Q8lLxxZY#)AnM5)Hy!iMr-WOf(Tj
z`Bk(LWQID2z?2CXQ6Y~anLtpBz?6L0i0vf%8Hv!ABuq$9;K6hd0Zc$NNALhol`B4~
zfZ*`#GG1}mc<|CYZoH6B`Dpmii}QTYwLcvy*FcuZ2zz_d1u*^XGICMzvpcViQ;_15j2t{L$P4;Y1-@LH
z@S3e0J0hjjUwYI@)F?;N=bw&IQ6nA@AG-#VJ(8RGQ^SmI)
zX5^H9jh>ls1
zdbE;w4*tx}vE{+>S^shNn(fj7auJCK6#*oIj1=g|c|a;hV1-l+1v-Ml5L76f#5@~n
zZfFQ9N*M@&Fctue1%*jT#Wo)Rbq4O-P$)zNYn;C_MHDzhXD*Zl3H}3w3=j7+XkH~4
zZ^1_RnMk$gBxH;Gg_kInf%qlF{1x^k$%JxGc?1~MDFuF=AK!+FYci>&`{^T5v1w(GZ?_d23ouIF5XacIvpGL`sJ@
z|7pm)zXLezSVK>0FZ!`#AKMY0;jSVEG#qJxJMmVfnPlJP@w$x1AIMy4*z~ybXq>q$
z-09zxfTpJa&8Bg3l(syhS`G8ECYfkgeakRDV?c#t3KFSJI%=yR$CDk^QSn*#H6rC^
zCakB!=r>&~^b#s%v=@#@hq$!p557hhx>#vEB?2<;#rYtMUhz@7E1u%mzbyi_BKOJ@
zMXPKl`6+RHOs`|3$0D}?W@X2;a$77Bq?loyuJPD7v2lIM+FLCBYg+~vs_ps_!k6CZ
z4diZaX8Eo{+l6_fH7ASFZUPl_vfIxinJ4Jo6f`H_E9k_kX+94gxLyi<3hh8weK{C7
zynvm@pVyaV^_{?CnQ1&D7v+w!f}sW$VuX~qKc0bWY**;5;UT3ubf&n(ePxb0F%fiz?2$d+MxAYJr_HCEdWQ0`c
zuj3#u068tW6>p6C3SHmn%1g~~n7JN{?crJl#L-n6t{*@WUSb8AOAfLaL+JC||GK<7@YZ&({SU_qxlQy6)!%Z_quZ
zEVjn+=FPfl`}Ps33ARV$#_#Q!CS0EEy?R&aiML4dv_$uPSo_`xm>w!!OO%Ce@>*W!
zhz+!T#`SB3Ry$_MWBU#ov2h+u7Jx2tOB*%jvBuFvRis%JxZNp5lO6%un?63xx(-VNE9N@bLSyEtnC1pE=wQ>
zFXSzEV)7l1gw5JAS}(KNW#gd7dKdDOyS?UqIJFtj-jbaIY-@mr0K68B6lg436dWY#
zG!2W-VWHsqi2_wx78+8m=GX*ikJ`LyP@T^`c>F(`-1VrZcOgs_HI+@K1|*icb@q10
zU!6uh({VW>-20oEvK5##F$E+XedR}1zy8-5Rb3_T7P$n32e`r&mq}%GKKwM+Ec}>K
zH?fOUdrSgL>%CejfJ_?69ImR=fEc$m#T=S*Jx@LTBbUmviSUkdcwLQuAjl@nDipeV
zP_g*cmRuT@iPlm#m1Y_>iR_5_TAhb08wVmC8-+)cqC?PsRg3w$hFP`_d1weFQS(mK
z6FyRKK1w_Eg|u2bw(EQb1Ci>KwT5p|y4=3_H-C0U)r`NZ0Qr)%FC0y}>^ACj_95ya
z&V*fdd=T(zw@aQ)L?S;k-~9l)*SvhFU8HGugshZ5AF^|+UEft%C3Gr-!6|VEkBu`t
zhbcKJkDX`awZx4c3uTz~~~KQPg2L>f`CnIHO0>2HxhG9dhMX*W4i{JflUu`g;zl
zh;&1m)0Vszfa%J}rH|j9=TdzIirZf~cIae_;y0D|{dEFDY-_I9PQCao``l7XLKq14#>vWGjK
zgYmJ!b&y%Eery7?r!vXGuc^E3o?Ok_mB4PJ+{UR1;_*AsOfgnvR0lSyQy55Mv1aVNNME*
zl%&Vzt}igY&`){@^l|vQ4UNrN{Y~+Fu(aD$fMkExgCI^nMqm%u$Nu6W5l!)S`#M9G
zkqdTc(4WhYs@NO_X_^|0LewUQmW5lzw2T!76owRn*l`6;6OWeUvFJNL-eqk)%IYOG
zy|#Ba!1Y~2)8X+CAX#T&Sl#4i01>>k!2M;j$#53z99odl$zm9*Pk&`>D&^NhZsFWL=Jb9>!?a(R}`mjKQGpGlv+6%e>T%^xTb#H{t#Z
zkYOC&QfdA}ism%wJ{=9ESp^zWw>)_L2YFReyr;8iWc)#Zt~TbVxYm1mX7N3HQPa+B
zsRvwqcVQxZ$w3-39ypcew&M^~SEgHs-~D;msoU@GQ(7$5&-l%mX3f!ya0__v$SN1~
zj!3K+M7NvEj9q6$iD!*;?}AlRWX__*@lzU_xozNPY)DyKaQn80Nr=O
zS?h}*Ce96uv2#HEWy;gbEbqI)WzZ@J_I-RCn_qjE_@7~W!@uRyt-2~x;|B^lFR~Wr
z{tRo*f*ThPh7+jc6X|#J=f10(#y4E)4HWL1Go
zWq%f?*IBZL+RMq+9EL75{4$@z!sz+U-}*=C2cE^ALFu35PZHeYvSHC?RF&LLqfu55cUxjcQ;D)(#vhY?F?Zzb
zRHT-(_j6?PVR!rGgvj+FH1ZMtk>bYe$SDGV>HESVFU1c$U$biX|IlUlS1Is+1=99a
z3TkEp#mvq0A84{LrCXtZ;r_oZ1_i7Qf%E_H=YROg$)2u^3PuC4bJ=Le_?fAj8$EGv
zZ!{fGGIw{UKw2=xmC7!*+Q{{H>?hs{l5aStcwOBk9S^WI*>Nvpg(lLLn)RBQcZo|u
zl1L1PIvgtl*#U<^3-$<&I7JrY`cnBj2uX+x91ndOK?OY9A(ZlQXI6`31_A*Aro0bH
zkMb5lgDiwNHAw~V#rOa2_k~KRBcl^WGZq^(6(c%I<2s13T#Ili(G=1JtFIQ)Wkytv1OY)bnhv1R!rFi{R1OUacuYgQ
zU06*f+CLxXMhTH+0vULK33Lkbfs?j`1eqe2k^%xp3NZ%MBdihgPvidAh;_suq*N*i
zp~+&^RY=D9p$XBF0$X*xq+~=%g;mZ60`$Dl;8X>C4Xd9w9q4^ud7_a76qCQ`Cc+*)
z)5;#xtPlsO`8xbLv9|m@eU|&A%Zx-UyOI?om<&j=UOh6`G`&T4UqqE|wjOBk@B7;`U2ow(XtyuEJ#%%UC`LcFIJW%}bs2+BxrVrB{+=p_JOcVuyZZ(2ci+%|8MDeLSGFFph)^Vm
ziXHy+{q|zUVg@f}4}i|CAwHh&$P$oi=THpW?1~+ZkmWd-r?3#Iftz0nJwbI+q
z%^Nb`^Mbd9V8|438u!f!AG%epT|XN(X?t(C#|h&}ZT0UpZrs4)U8>Ea*D=YV?^owW
zVg^Jtt&{occ^u8l>X&BYi@b*aoR9jq;97L^JYZL$=YsEJczUGaryxcl8hy7#TFY
zTS=*~B3Q44dVt*^h?WwYOW{3-Z+O9(FArEFB3-@^3<9dZ19x$)aI1%}5ded)Z>;|P
z1~GHoI?c_5_lL$TkePb
zzL~tD;rUrBuAHG5a%_3KC-!nHbXP?J|
zTH(&pwDaFq@Q%n`T@9Z6yjf6a83HJ@HmjE5)T{oO@Txa`o!9?_tY*D}DRTt&%S3{oUCn@IoqZ6L02Bo~YOVBm2$sX%^)*qh*1S+90tLu7fmuAT
z%As==$^IA6CO0hn;qU)g0uZ1=ZJ>>J+8e+VHUvOR|H2b2VGs<;RT;kE)8ty%c#QAt
z5ixgN==*keFTRvYce3UNrrH_!Y(l#5qg2ZuSE8MjcCe(Xc+08il(A(`q(x4Z)|11F
z&+Ct?OF;dNDqhM+j8aRXD+R)__=&M_r8v9
z{@9f%c25fglj>$GbD4kS&qeW=VZdq?aj>Hkm%!t^)%JOt@p7EmF=Ce%abXX!+%Kwv
zH6s+keM^T{kQl4-@r{hVMn8bOZ#lu^)u$r_40TL?N*L4(4tb7CNC5VK2mMy0H$bam
ztiH_-M#SQ%8Db>u?oA3uhH-M%?Gg$CZY%rt>h*9DLk`7XWiSnn$AplMtPc8R-FPTQL;2<8tK8}bFX)g;CqW@6;Y%~+XZKYBY&3bXo
zzvIvDrD_8LE_^m{KY9Ak*dJs`5d3w`8HRhzWJF2&HGu-{<$OPDB(0QPhyHE1%L}gz_wA`y(w$qIcUi^0^0_8_tE#BS)(@CD<+q`|%23S{{e$lEU!Om0?^-u+`LN{a
zBN>-GTWbeT@A)QMK8SNi>jn#fsdV04_$L0c7GILEzA${J3MMsD>^tB7e@G)!D$0}hmX)zmTC~3!b1QOz<
z40#xlI*#QMRtL_jwqomoDng;W2B6oMw+B~+q
zC;NPY!f9wvCoJ)$zvPr--u0HMmaJvhL!OL|JQ+7g3XrS#3k?6IKIXDCS9B1nj99({
ze>$B{P;m-cey8E&mXRjy6g`;AAcb_688?77Rt^~okF6((*||yyr%Y~^4Drhpa4Dc7
z76A~=fzBH!Z!QCssKu7(6V4^a2{zgU~FFROlrePqiD*JB$}L=x}I3WqBC
z=gVl`>NyN>9+s^7)4p!j@>VT~mS*bBLiDJq0{Q8hqrfA9yHLx5)bMs;FnP$+7_=TZ
zqoVG0=YSk$+~$1U-_01m^pUPu=)!J0Tpp7}kbHk8h2MC2`O;3}CHM1oGjzimq7}A6
znD<4+&wRHxRSZ4G2k9Kkj`lq9@d|B(n$jP^thsBj|51PG@DrTw}sygRwp{H3K5lFXGdvV;WcHImWZ%e2V24MveQ^0z8#WEe@Mus*Uj$h09^Qe<=y
zQlb+gDz84a0`K3qhhMXTzoaCMdty1|9N!p8L7<4}2_ZP><7k9e~dtFgSusQ
zQF+^tqm%+TXml!2Y{Tg1z^`dU|6Xr!Pz|}C1t_e+@RkFN@m)n3uhg(-H@3IuK)VbX
zpM(YB`iZWBk=Ke?GTXX7_kr+2h`GRq95Y@w5V6RY6XW`ox&CE+Zyb{SRN@9Z5!gsk
z?vdpf*QBulm+iylgt|BqQ0xayEaR^1il_RJ%ciTD-|BQ$Z`@~~iJ`(f9HI%<$3s{0
zJWeI4`XvIIj?{NObY#lASj;O#wilIUQ!kQt>ByH~`5M(q{NK_`Qa&OJORS$6f6JOb
zIC~0zwx`F!HxuLS5)~WZe~;~tY=|Ge-Ar#3jwmHhbVvdE;y-Dmu!{{&1~FukBVpk-s*w$a$OZQC}QPDn!SnN)mscINA+@z36+djxA+-{R|aadPnye+b(rQ;rUs%L~5G
zqXNc2x;9nW|Wj%D`;|?Flr2obrN%4Gg)1>
zOy}m5L%@B
z2;hO%8il2*86mFcb
zi@9YE&-#LjO^dW^iTp>XiPX`@C78iapqIWsUY-<++TI2+oQF&rvl@@2ROPBO=7N!9>7`>-K?BzD
z#fdL5B85o(y)?KgJk_K&l2(S~t}o0;qq?ijp2Wavogk-UpuGBL9h&KBs3-CD#{4lt
zhc)M7fwiJz6Mb}(TmRFs*=+K^tc!6+yGkQQt}Dn6B}Reh$zKd|7wN=f3e%g_S_D((
zRty@EcA|&n`CDSYBA{uA8S*OpnHY6E8v$M$+KoVl-w}^tVmN;BMdHWvlvk#&9w^ba
zFM0q~X?J-rfknO3<_v8-R6=BwBIJS^=Ovp;kQICJNDcZ`jn!9&a`ue+O-4nwOI)()
zoOa0_C)gq9q8fwpOtO_*Br_&_*N88}UCX~EDiEg}yX2p4-pxy*F*zrP
zL+!cEl2?C9S!eIvp8KF}XVeWn=<>)JW`6<;PN)S*42!Lq1)PHDP1!Xq_Eev9$)NPgf}eyP%Z!_Px3#y1
zK3c@cTN+N2y*}5~#BH`lpJ$+!oGaciiKKKd+o=j_Tvn3QtlmUYFNp)N!DT$)6%Ig7
zC2CIWrsGDI$EL4YOZ&yPY$-xvXoeedI-lzq-C~TtxFrVL&%9$J*^$6`!=BiF+Q~aR
z#S@0^z5zp?f2Zp3#(COobC2DF6AyN`UTFJi~_EV+o?7uT3VU7#dHm{gD%rUq9BZu!ksy&N!j|)dMPCQ^pEzjxQj(vriLS^z~q@=F@uRQXi7kZiXvh
z0piOdbTm4bWXmf2NJ>vajPVXc8Oau^Ph!Ay^33WB9TIvy;HIE;g
zikWcHNr!D6p^>fZB(vDPSXQ>DjUl~T`*iAgo`xO{t@q+Xh`yqE`jgYTu-jiKa0PfL>p2q`UCZy&I`-FrkB3W~jHv$9k)s8)Xy2X=GFusuH=
zN6y1B*BF8}4&vsotYIP?#33u%%5bQFw&nOBOg#8Yp7NzkeQuZt)_)ET>RnLv!;T
zni?J_l~tZ3`LZwD2%3SzmmR5+Fc5M@?&F>i@RLXGmP^ECmLF}AZZ=UJgZ5ApQD1pb
zs<)~n7I_r$6-p@5t%yYzW?WAALTSURV4WH)Fc$WvfTi|C82sGOG%vzZoWdN
zmE8
z;QLt;0fWz}F=C}^lELo_CQTt%46p;CTQwhAzV5iEZ=EHl8-sy)RAJa}_uQ`zw@CK!
zyYbYJKv*$FNp-+wyZ}n<#)fZB{4;&GhbR9C;%q9i@L?B7K`BL1n}=O+@BZD4m{KSz
zLSa-!kif<&gPMSIA^pLXVzv`nVR$&7=$z?izZ*$DaX;ReUDGUGtLtcKkG8v8sVrws
z^t-*=tM%#og6{Zmu$W|=X&Dh@M>97t1!V{)n;O2EfC~Uc!;DyJju@Te{0?86;!U`3OE2CK*X
zqklU?-X-K#jt$zA>g!ZTL{;M@m3<#;V;1C
znX|U06T%wzwA@H*l2k;Pihi00Pt#kId_%~XaY;kIv?G>3k_iZyL+C2fZbZu%!O*dLou-E_ja
z%?Xg@n|i1G9QHdk@bKbYc&}J#Eh&FB9zTx7e_nug+Zygn7=Vtm99v#a1vhhAtNJ)g
zSppo_&z4>*JOu@teYf{Y2zb;(;kWXCWT(w~c+_xOS*8l5L`oLhxMXQ5Qru7Z=22tz
z@>g%)=xiX`&GXAi2pYUEMcV!_z%f`dhz16U_j>u5O%cjLi
z%U+0wUhVNA^9=AIL)v}Q1~ANX;KHECKf&6&c*s9DGjJPT57vb&P4!sjFb1g9
z+isPHq0hG><)Zi;yeN$T=Wdtc`;-K7xz|6fMxndgU7%&Cwd0~tBxDNy>{wpw9su$s
z_VV)39C1^m%sFSL6pu@X!r!;GD{fNVVmuOiRjCi<%I3v_k1hs0GqB|>U;j`4)~h&$
z*EBpV=g;bOv&MCOr}ob)-%duN8qCzpRpR^<^OVR52`Rp}!S|Xh9pb9%s}G`0qof~s
z^{pHWH53p{7|89_55fME({VUr@PN>p3*@{jtRlIwwj7(`D`XMb>)kn&V2Je(g_=sv
zg5yH}m|35iD&w;w4Xr9I=Jm^hw^walt$}gV`dbL9qm=-Ptv!Uqu0waKGO(?sR`dmE
z%JBH^F^QFn)qgtEV&JAxw9#2J40LqcRcmcSG?kTbn}urv{?G}lBrDLKc0h8vz+2U#
z??Q^JpSh-z4Osne;U@Fy%zdR>nSp{mAG5C??!{7xy>ov#{i;EfADJq
z;Qi?L45xksc|(~A1`L=hT0MmXgi`Sx-p5kQO_#ffXA3=W)RYTj7li(Uc+YB?$-zge;KR`hZ@76SI@Uol-6T1MqC7Y*nAw`qVfvPt{YMO<7-si@K
z`p^-NM!G7V%#Lyy%#eKY7-~X*3
z<-wqgAnVNP6a%SWy(q7$s;IJh9x0hi`I}wtXOUPgdT3f9U-bt`qq5E`g*IhYMGL11
zHnu7Txs|nqRpcBU{T7iq`Zk`Wm~}RF%+wSB)5Q-&6sjEbib?7BqHS^|VymmxFJrht
zgT#W!_Kq$WzV@*2|CKVG65FdmbhCOo1};eb5tqV9r6IV>7wo6TgO~}v!a_oA3`@cb
z+h7jXrOE?M2G$h;++)1QJ1u)ggkn71DK5>rO+>Vk|QeQRxTt
z?m@sw{?)G7ku6WG0;5tl=J1XLmy0L^y{@H5mnEsS*0YeGWyua4vD{n$=y3|G&I9;xetEVZ;OG{J)ibJM|B#sqPjCd2szQtJ?
z`|6Dn2%e=ADe0^u9OBtZr#+5ly5kOz=HNWI&8%4biEHW;p5Vb!g|znqtCuumt;r8r
z#r2}hO9fIpV9?|34r_92=*?P+mEnt6rY=tW~+
zm^bb&!{HjCsr4FeF3=xa46?c44W+xa02QVM^p8OP
zEp(IpDqkzVDw#dfur<+4!L~K4x>$9wVUXviqe^haM_1POd?Np6!s>jY*mfB{HK)by
zZXc06A$^a{52KW~hf_6wHk)&uzp0y(uSGg#N50`tz6+C*Ic+a>>eB$=b1n~K%gW~@
zn8#T^{#MbR`-O@?FcjiZqL>-Ozo?khJ;L#dsdaOC(eJGt95pWE<>99Jd)?0d`0*p+
zFVpy_ce!EgkXDk;eH|{NWzuRfZ&s7lj@@cd=e=iTfOf+-WN_-)>hSD?rT9pCEscXJo
z@nSsJRQqqftQrd{o1z9A@}&m%t!iAFu_*GUQ;#1Xrosnd3+tOo@zR3^#jXL+RkZ${_>;FJ03r
zf3L}?E-*1QH8xP$f<1ZW9b#Bmh
zpmACCw$c12VdfR6t-%)jQmXQB4YA>0IfJ{-uSdgk@Ay3@y^SF>WcEm7PlP`NM*+cG
zKh>H!~>V
zhNbcLd4QgBml7UdCUShY<&Cn47Yi=CObcx@z9@6*OU$J)`m4ru
zr~l;6b*<(s+iW{c!7J2s{m(8{!jyLrmkFR|;v@LVOET~qE03D^+|#98(n`((EHS`9
zoI+T}drA6CEJAMe9UC4e2Xb|M;6)-YHEes>+F4
z0@Ex>79JpJ5Rk>naoE{8&uMK`@`ns;@6*^!1ZE4J3GJikxnB-?yEVO&|6u1O{tA$~
zSK-lFEthlSnRvG_`TMe8Q7Aa1Buxmgf(NN!BIbcYVK2|Um``kkG>;i9SGiBQanUbpR4p^h1Z{=xd5_-Sq<0d1~jv1JTm9JZ7q%QWG(g
zKd%DK33b-%h$bcXJv{|Ym%TeG)(9xkeA~JTu$kvX65`o0gS%{qf{_YWN4Y@1SBVlev+!8bxPLOvY_*&&}27
z*WjKct8D@@o;Nb25&l2uu7SU^PgBBM#i7hEUk3(~aADkEKO{qC{Gb=)kY4{?ucMr7
z{?kvR!-)hIn_>OCBuksM-(~}deoLlr+7^ZCw=Yg`Gzh;O*wV?hNLV-NDbLMsgr{NQYzq$-R#nlO2^V*hc~
zmZaUV#vaF@s-tt*&vEwPc#K-*!`|h(>;W@S$K)o^-aGrl-
zC$i=J6I~gPRiP7kgaE)7X>H+%|1_nlv(7(mFz($akikX|!)zf_+5Vgsa
zYfDDrd!bTDa%JDRj=u*9OK2DTLk5F)ZVd4M{m4&ZB(soBaHVMGt0vT#v+OR8NPgVDa
z^dud8T}VW-X)3e_@8nP#C1Yg{os-y&F#)eO7B!3D?Nz$X8cB~=@8+N7_;aMo^(XGs
z;m3e)gCK7WjPuxWv3~uQzS-mlilI7bWhj^PYVSl>rm2{byL`g
z?~yjVn?kiy-3S18jgo6Bj&4rTELW|LW{nCAza`PEJrOytI8u=3qRFn{-3b%todV4A!m
z(M&*pX^>m_Xu)OuqBbpx4ab-~9pU|_C4iMB{6Nm2OAK%ToZSbfmhu5kl8nqI0LLWtP$ioCQfA#BHZT{m_qhcU!s@sswGk0lk
zR*F88x=)T!SO=%&^7>u1v>t;H$kkn#4&Ce(tov&bVf2mM;cAlMSw+%aOgG3_bJgMT
zGEiEFF$I{&9Elrq{gBR%*A8c|><`OnEhR+W_yt)4(?4#`Jyf@6C8x-R38$2>6=1v#
zB5D#Aw6#Wk$FF1C5F}H>(Ye4V*Sf{Qhs0?Uy~wu~G}1wR-LU>N7lG&VRPH~{{^(HW
zkFw$z;_c%Yu|IX&0;8L(ye1kTxpoi<*hs|q^8r0P_G&YF#Dy6zU(sUT+elWAxLb-@
zI%DnsQ1Q>YS;zl5jdmwleYbUZsb8HtAaCy*R(KiNEV=7u
z*qf=!6fetdcFW@tDMsrs_WgJ9u$I3Q9@|+?_+0SpA`JssZMnGbW|)^}z;;)>^-cWh
zNJC2ddS;Zlcv&8H33j~kL@jk+aTPMMy?}%56C}PHmo|cA*{|k1N)s?fUQ>EuF(roFClrfA)sh$q6M7yo8@C*$&w7n<9HotzII
zE$Z2htBcfDRj;xLN)K{f8>gF%~~zw?YP
z#n7g}wR$g^MeQH93N%N)O9cbCwt|gh&6a`MzqO@gN$LZ-Rd#C$W)hR;T7xswr3ByEiAEfny
zG2~<9qBgl`Lb~gJ?p`km*>XiJ$@evwuoq2reBiCydUc4EJ3q)?oLDF)JAi5Fus#FF
zh)c&hPrdo8fsWvy!nzDP_hZ_086$+3bj=&-EZJQ!vw2@Z~#1?$?HDhr?
zU2Xc`&%1;4rr{BXZm%piy{V4ZW~R>Fd0deGI~QJy^IWBWj7}gTZclb6i6D9@I$22@
zcH&TGffK_;w08x}&p^G|d$TLL$odg(92O_lS{J>;gpGp7inN&LeA*=2v=*4AC@wI5
zTUI)+od6R1q#8fFr~Th~So7htg{PVOPNFHv_5t-PFDkTX`trSZk=cMw&>?ky(0r>A
zzSZ1Mplv8=dHU;XxwQKG;b3t;FLo&8^u$%iC&+uE+C4j58_3ZWOpZH%-Mw5aJWMyd
zh=hz+lDzL`Pa3EHn79?6Wqr6@Q$hbLj|3dw|D94LY@F{a!G86c^B#UVJ_ad*gak?NkLcgvz+i
z{=KPuLLb~lwnF9RVS{6mw{Wy_w`L__W##5c>M6&7U}IJk(k6Czz+t37V%s=sh_%ii|{5ma)*0~BY*#^D6
zl0PYJs!JbJ%FT!G?c3j7)jl~q$M+q1$FCjJ?>@QHy%p`sE)-A%2y1;P!Q?xL6QI;^
zVc|6+DljmxE6n1|J*`0isVGt`GRjRj6ci4_QYP{o3PxQ4X+DoIvxqhb<^d@Q7tR5A
zSu`#b{7MkYh?ENy3IiAoiKrX)tuR==Se8yp4;TctJbFGeOjr%6GB#puxDS~iUuamQ
z8_urtp8TFQJ^zkyVMK32Ff*rc2u>h)ZZLHAuJpnPl)Etu43rMQj)WeV5y*Ff=&d~J
zsNOSm$%Twv@q+0BX*#9bSvk8OGRcTCJc
z1Pmk$(NV;XPc|+PaK+FB5*hkx0^)ldLf|ZkLpmUQD6YL@59SgCLd{9|4~_<@F3d{g
z2OT9|=vqQ|RDmeTUUs&QmoP}MkVF3;SWD9!ef?~ZAF+r9jU^)P3#bL8qM#`3W@1e7
z1p;y?5S3wmUEauJT#?_;oZDr3y(d|(E~b0d3Ei48P_U4|tdRBnz1_S?!y-haF()|;
z^D2Szgds^4E`DaMiu6>L{iwQEHu~C6lsK@$GhuHExQtt@WAat*3~(yTArR2c0e{(W
zoF(S8=sGOpfxFcjl^m3Grn&E9BIJ-AX_}*MZst{#5ZWq8v61=bz?EH5=cLJH=7rE#
zM|e=w9uN?qAC&pm)EqW?`YCa31_@_j-@xFFF;P#E3xqL27VIA``ItZ~P%s!Hn4!TB
zn2(M=lqE0(PC^kqVnX5x`6mSwbf^i-Ntz_5ndOohmVrXG;^?>^;Nje4fuO7`Z&S>{
zO_5XImUjWGksbS=PyP{sHn~k)QYic`JM9NIj@A3IV+{O~nx+`NAML)8Rp7
z%-zgVe@TxJPuACe@_kyiALXsv4CNurnn2;hfg(gvfy7RiMlnNK|HeU!5=Hi^Ukwo&ZndJ;ZOJ#eFQnApu~`u+qviRC;pAK
zkAs6IV8gIkpuEmcw=Rii8auH~{%-`rO|Knw4ZuH5Xg
zGe7teju8poN9)JvTijpX)T?M794=l!$=HhZLs^qK<c6;mL&bFQS%tg=9E
z*3*I1f)9{Td?QV%l`LdT4rVT$>W)#^`rWmZZ0|Gl(b>FEYP|19)w#c!8^|8Z);k{j
zW9r=3b$lGgh9<~!@RQX?3yiJKc;yzaQjv)opASGT8PQW|^ZI@JXe_?#;y
zVw}eVR+)mDBb}Zi)$=CLZ{9>2JD*_N
zW~PyPQQyfiJFOa*>*GAhIp}vd7N!ZK>dZFFFB^{fDl6Doy9={>jC3wb^X51Aity0P
z?ga|kkvm$q3VzN;wwy1`#(&k*iGH0jt4UvbLx$Oki%lQl3U>3K*f~Jw!%2e1a~@Ag
zY9WE8eC+ubgj4%j?>%V6{U;R7@v>9PDwW5z`E!Qu7f9=xRO1@xFv8_(0O6%-E+b|JN
z=Tug^P~5C;rBEoM$QZVRmNJD`Fi$C|_oWKK(xt0!oRTmJ_3F#>L~OdMTtL)6Bru?-
zbyGSsy|Foq=80_T#i5h?v_gweM+OXHI#_T>l5}}Ut_J>O&
z@^KI35p=+`qY%X~ED7X~%0Vbwc88o46JH=Sf|+nlsRC4K`k?0kfYGg!ENI
zP$T*P^pBhgT>!YvPqfJ2KI%Wz_;-|r6p3^-m0lKL0Y4QbYuHtk<0q~sWL&lIs}RwO
zcGRob9AJNhwkX5Du2`k;*YRvwVdmv7qZGXqF+`mjXHTU>X!+$mvAO{WEDBAiQ_DFG
z1W@9u$FIr#HRwrziuAQ%r2@?OTj(=IJ(#A#+k3*oPcHEBBL%SEbkZTpW1Rb<4A0ZQeIx{yYkbl>^
z?fue3#N>&->?{=n_jM-LxrH}?mY>^5^Q39nLfB`xAV!Yori_)FcgEfBBdB(K02Xly
z)V5umuz76fbK?|M0d
zB&FMBvtRRdcH6A2-+$V#22~a(c|c*VGC}W6z&M$Y38DbhA0|8$dZ=h01kL>>n>bcg
z%^NpxnNV6k72+;5pBa}d&{mRefl&z`tx_HlZ3}+I7T2Axr!({nONw>Gp`{4yZ7$oW
z%P}&<=4hgf|C(+TF{koCX!-BQrN{0`#re%sJN3_}`e^Ks;XEk6RhMc9r>cWe<~ei$
zo7br0iW1G)C4c=g_swLrZqyStIHGvc>lB{=>*kdG2iRdx)qIJOclvZCfN7|vF?{yx
zNk;ZPiK!#*q%^aGSKUEfyT*;xSE8focCz&vfjuI~Fkz1#->dqxzA=?)JGWx;5KRUx
zLT`_Lgv(YjEmgx%aZY({UjOCoep6)J0QO))3f0-^Z7FpybNbt(YKgU*sJ;W?ODoN-
zu$If|?_6-inQvlryifZbARRR=So|+TI>E;y=*idtWcRnn7G;)_?~GcZTT|ESXRp#v
zF)ywD=*bGMg-lgeDlw;Oce)NQy!$ih;LYAgtaG)SQCe!4cD6GmYAbqMg9b##d8c%F
zI9|bPOUha`6OXUkT7fQQ`xEG6zR{<;MlLEiC#7=;PO02$ecbaBfX2?MFgu#GFNoSh
zAk9j@aIJkw_y1{Yu(feNFXu&@dkwgA8?#A
z$K!KIEU|(1S0)&oT}aiJyP`NgpQfT+Pd0z$3;_#coWp2*+S^;pvZb+#BpDOKV20gB
zgGKFWmedsmq){y(1VvAh)q5l&%Z}iFP?)OQwai4iAca6m$S2l_nLw48$kG|F31_&DF^r1!+W9@H$HD9KPwoLY*npLH9I)4Y3;$)rKW^CWtb`sY9b_L|#oPVeg&u2V
z<%OMADr_c1m#df?%{`gKP<(JgM>IJ+YcEh3176^H%2OsK+eDU(o~N`|OABLTqV{4A
zwLpCzfQ8sS9A>gFh|8FniZ*Yks^JvFf0f$Js7szbZNdl_`9L~ax(}RKEUFAlOQa}u
zA?Fdl4HLh2CMscHaI%6!nTPijuU4{B!S7_LU{HXpwnQ(4>oxm>@5Q5z#K1hJ-qu(b6-WMPZD8+IHOp
z+P>P%`7cobd?xqc^$qUJ67fstV^!~5r{fK^U$oFZ-^UlI@P12;A~fsW
z$CM!6u+Y&g&hWHYJcjhe?1x;<=QBFLOAK-S>jdxq{P+W{A|McwE#8kip?y0#QC>ax
zL++pA@IAqIkI<10YPBExZ=mnN=dMHD`SkDTi^*=){R3(4VZS|=UZFpy<~qBc@-7h?
zclSkiyWUYHBiS(>Gh846o+v96q_dFSX~oo|qK;e>d@p0%?qcY6OwuL#@!{dlM^kg8
znp0E~(;j;(1hA+6&8@iU2uLpF-HhE6iCCDSXlhbvXQ}@!xIX-*Hezb)oZkB31)?X4
ztkF6OUY1Sra5FO_Ko2ircavx0B$zDH9uKeVOTN>&@&0o3*7jWNetn)c>d!gVoQjM`
zxOx^OD#KWe&C&8|0yVXs_R-g--tq8v4kq6WY<^EB5tvdC&7ufwmK`ws3h;d_26)TR
zY8iir^@IqR^?W7%`o!lyLF5;2b?N=!Q|DFsaV6Vf%EZ=wlH?arUbK*zFs7b7Mg1I4
zm8jCz3iw92n}b!96gSG0XU0ECOPDf6ckzDt{=6H``toR28hzskx!*yxa=wD^Dd4yt
zWBB3ym#YzzGzF0k4Cqv>aH+gqPjXKepg4sgO5ztB5^B38xUB04
zoNsDocMdjW0Lkw1-&(F(R3-9E$z~mUWJ;T61QK)$v;jaIZloM*0^E@54UJadbyVgS*uP$>Vg~t
z3FHZ@vdYyIZO_^JjmO&~+0ti{MxkbB9>kX3j1sOMH6|4JYI**{EzIAWsgaIu2oZ$x@&k@*L{T)Ajy6dJc2Q
zRFtOMr1?F`1cR*DW!Od7ar)TqwZu$61qYc;pLt`$Iu38*7A
zLHEWV`Nt+`6$P~Ny6@NDLPY$7J5ATsAJ(U!e;fqr7H7wJbrI{}C?FY5|EcNTS^Z8nvO*T}x>d>g!`tr62_yp|PN
z>HVK9PCik(5tZlhVwYl^r{=I37htb}AB~TbBL45E3S;>cH_Q=;6LnFqfe_7?@kZMr
z?*H%MiBM1JFgSZz9!kHMN_e)e!UAq6k@m4@{M)6I(`h5Df{QHZy@0BLI_Pd*#0m
zGh6HyXo|Y0<#CUO*o7W{$1o>tK9}(T#%5)Q_XDUeL0fg$2A#DEj*JU2_Fwl%R~Byf
zca7ke89WXd-v>?2-AY6|FUC&dKU&GP5=gX2$)1fsM_Lz}5tHtlMhymIH-1R1O+r(5
zI-qCzUs^g_@VjRC`b7Keg9AcczFCOBtD>znJ0#;U9?m)6ZCrPCi5pdJwf)P
zZg^Wx;|-;(d6R$AbOo~6sr=kB)do+x#6I%8oTNq+#s2KBf0MRN2;rj4Db6j<#wsGoCc@6aEy>Br#lk7V#m2+)-&@bg&BiS$NW%aBMGXF*
z2y(EbiEM+@0hDuc(hDjW!?$i%gjU(NJiHdENt%T=zgg%x80#+_Qd4xF7FOJHKsW?rG161m}P*tuX)9@_9m`kXEa8-tuTOEl%lYx~M#kGS}1u)giihfqAQ{vMbV=_eWS6Vh>
zRcN?sT{vUet{6aFghOeMLUnS>wr0zIg>UB551Uvas%+^b#m?lO!Z$k}Uo67*+mr^8?!E@uXk
za0XG&7{J%447r>^?xZt=Y+r}wGl9Mc2VDXGX+nGvQCK5q@n`&sC@{ns25OfXp%-<_
zPRKZ2U4a*Bco)L%$xcf8LxT&>j6XF=v5ywV>R+%s-c*kcH}vr?n0!ys*x~xpwnR$%
zXX=k~{)}4o;2b}f`&B*>cFnT4iJl0Xo{aWb>RQBq1MNy7avf_soM
diff --git a/doc/Projects/2023/Project1/pdf/Project1.tex b/doc/Projects/2023/Project1/pdf/Project1.tex
index 9a3ed69b4..0381df84e 100644
--- a/doc/Projects/2023/Project1/pdf/Project1.tex
+++ b/doc/Projects/2023/Project1/pdf/Project1.tex
@@ -421,15 +421,15 @@ Show that you can rewrite this in terms of a term which contains the variance o
term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise.
That is, show that
\[
-\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=(\mathrm{Bias}[\tilde{y}])^2+\mathrm{var}[\tilde{f}]+\sigma^2,
+\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2,
\]
with
\[
-(\mathrm{Bias}[\tilde{y}])^2=\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2,
+\mathrm{Bias}[y]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right],
\]
and
\[
-\mathrm{var}[\tilde{f}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2.
+\mathrm{var}[\tilde{y}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2.
\]
The answer to this exercise should be included in the theory part of the report. This exercise is also part of the weekly exercises of week 37.
Explain what the terms mean and discuss their interpretations.
diff --git a/doc/pub/week37/ipynb/week37.ipynb b/doc/pub/week37/ipynb/week37.ipynb
index d31eb4073..39771732f 100644
--- a/doc/pub/week37/ipynb/week37.ipynb
+++ b/doc/pub/week37/ipynb/week37.ipynb
@@ -3,9 +3,7 @@
{
"cell_type": "markdown",
"id": "9e3d6c33",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"\n",
@@ -15,9 +13,7 @@
{
"cell_type": "markdown",
"id": "6c4bf45e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"# Week 37: Statistical interpretations and Resampling Methods\n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n",
@@ -32,9 +28,7 @@
{
"cell_type": "markdown",
"id": "af4e71fa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Plans for week 37\n",
"\n",
@@ -75,9 +69,7 @@
{
"cell_type": "markdown",
"id": "a30b83bd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Material from last week and relevant for the weekly exercises"
]
@@ -85,9 +77,7 @@
{
"cell_type": "markdown",
"id": "603e5939",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Linking the regression analysis with a statistical interpretation\n",
"\n",
@@ -114,9 +104,7 @@
{
"cell_type": "markdown",
"id": "c6d1b655",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*} \n",
@@ -130,9 +118,7 @@
{
"cell_type": "markdown",
"id": "0ac423bb",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The randomness of $\\varepsilon_i$ implies that\n",
"$\\mathbf{y}_i$ is also a random variable. In particular,\n",
@@ -149,9 +135,7 @@
{
"cell_type": "markdown",
"id": "a321bc7e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Assumptions made\n",
"\n",
@@ -163,9 +147,7 @@
{
"cell_type": "markdown",
"id": "a058a44f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n",
@@ -175,9 +157,7 @@
{
"cell_type": "markdown",
"id": "7ca5aeb9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We approximate this function with our model from the solution of the linear regression equations, that is our\n",
"function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with"
@@ -186,9 +166,7 @@
{
"cell_type": "markdown",
"id": "7a064201",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n",
@@ -198,9 +176,7 @@
{
"cell_type": "markdown",
"id": "fc70ead9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Expectation value and variance\n",
"\n",
@@ -210,9 +186,7 @@
{
"cell_type": "markdown",
"id": "9beae550",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*} \n",
@@ -226,9 +200,7 @@
{
"cell_type": "markdown",
"id": "c0ab7cc5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"while\n",
"its variance is"
@@ -237,9 +209,7 @@
{
"cell_type": "markdown",
"id": "a6900e18",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n",
@@ -260,9 +230,7 @@
{
"cell_type": "markdown",
"id": "fe9b0d2f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n",
"mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)."
@@ -271,9 +239,7 @@
{
"cell_type": "markdown",
"id": "be36f9aa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Expectation value and variance for $\\boldsymbol{\\beta}$\n",
"\n",
@@ -283,9 +249,7 @@
{
"cell_type": "markdown",
"id": "3717afd4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n",
@@ -295,9 +259,7 @@
{
"cell_type": "markdown",
"id": "f75bbc6c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This means that the estimator of the regression parameters is unbiased.\n",
"\n",
@@ -309,9 +271,7 @@
{
"cell_type": "markdown",
"id": "501aab1c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{eqnarray*}\n",
@@ -340,9 +300,7 @@
{
"cell_type": "markdown",
"id": "882b7267",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where we have used that $\\mathbb{E} (\\mathbf{y} \\mathbf{y}^{T}) =\n",
"\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n",
@@ -362,9 +320,7 @@
{
"cell_type": "markdown",
"id": "b7993235",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}.\n",
@@ -374,9 +330,7 @@
{
"cell_type": "markdown",
"id": "27dfe1f9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We see clearly that \n",
"$\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big] \\not= \\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}$ for any $\\lambda > 0$.\n",
@@ -387,9 +341,7 @@
{
"cell_type": "markdown",
"id": "d1ccc9c9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mbox{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n",
@@ -399,9 +351,7 @@
{
"cell_type": "markdown",
"id": "4b51caf8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n",
"\n",
@@ -411,9 +361,7 @@
{
"cell_type": "markdown",
"id": "6b6c2347",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mbox{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}]-\\mbox{Var}(\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n",
@@ -423,9 +371,7 @@
{
"cell_type": "markdown",
"id": "7b622b81",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The difference is non-negative definite since each component of the\n",
"matrix product is non-negative definite. \n",
@@ -437,9 +383,7 @@
{
"cell_type": "markdown",
"id": "bc281f57",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Material for lecture Thursday September 14"
]
@@ -447,9 +391,7 @@
{
"cell_type": "markdown",
"id": "b1262baf",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Deriving OLS from a probability distribution\n",
"\n",
@@ -470,9 +412,7 @@
{
"cell_type": "markdown",
"id": "a14cfe86",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n",
@@ -482,9 +422,7 @@
{
"cell_type": "markdown",
"id": "56fe0849",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Independent and Identically Distrubuted (iid)\n",
"\n",
@@ -495,9 +433,7 @@
{
"cell_type": "markdown",
"id": "afed97f2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n",
@@ -507,9 +443,7 @@
{
"cell_type": "markdown",
"id": "e4b5884d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n",
"\n",
@@ -519,9 +453,7 @@
{
"cell_type": "markdown",
"id": "0c949c06",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n",
@@ -531,9 +463,7 @@
{
"cell_type": "markdown",
"id": "113e3658",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n",
"in case we have a simple one-dimensional input and output case"
@@ -542,9 +472,7 @@
{
"cell_type": "markdown",
"id": "6851f92d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n",
@@ -554,9 +482,7 @@
{
"cell_type": "markdown",
"id": "9266f3e0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n",
"We can now rewrite the above probability as"
@@ -565,9 +491,7 @@
{
"cell_type": "markdown",
"id": "287d1ae7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n",
@@ -577,9 +501,7 @@
{
"cell_type": "markdown",
"id": "5b27ae15",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$."
]
@@ -587,9 +509,7 @@
{
"cell_type": "markdown",
"id": "05685291",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Maximum Likelihood Estimation (MLE)\n",
"\n",
@@ -618,9 +538,7 @@
{
"cell_type": "markdown",
"id": "9219f05a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## A new Cost Function\n",
"\n",
@@ -630,9 +548,7 @@
{
"cell_type": "markdown",
"id": "fba0d995",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n",
@@ -642,9 +558,7 @@
{
"cell_type": "markdown",
"id": "c19bfc19",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which becomes"
]
@@ -652,9 +566,7 @@
{
"cell_type": "markdown",
"id": "6de82f09",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n",
@@ -664,9 +576,7 @@
{
"cell_type": "markdown",
"id": "adfc6771",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely"
]
@@ -674,9 +584,7 @@
{
"cell_type": "markdown",
"id": "a1ecd0dd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n",
@@ -686,9 +594,7 @@
{
"cell_type": "markdown",
"id": "91a37c0d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which leads to the well-known OLS equation for the optimal paramters $\\beta$"
]
@@ -696,9 +602,7 @@
{
"cell_type": "markdown",
"id": "11677560",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n",
@@ -708,9 +612,7 @@
{
"cell_type": "markdown",
"id": "8099810c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics."
]
@@ -718,9 +620,7 @@
{
"cell_type": "markdown",
"id": "f9232f8c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## More basic Statistics and Bayes' theorem\n",
"\n",
@@ -738,9 +638,7 @@
{
"cell_type": "markdown",
"id": "9088ea5d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n",
@@ -750,9 +648,7 @@
{
"cell_type": "markdown",
"id": "5a43a59f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"**The product rule (aka joint probability) is given by.**"
]
@@ -760,9 +656,7 @@
{
"cell_type": "markdown",
"id": "4cf876a2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n",
@@ -772,9 +666,7 @@
{
"cell_type": "markdown",
"id": "19f3bd61",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n",
"\n",
@@ -784,9 +676,7 @@
{
"cell_type": "markdown",
"id": "f4df8724",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Marginal Probability\n",
"\n",
@@ -796,9 +686,7 @@
{
"cell_type": "markdown",
"id": "a137b317",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n",
@@ -808,9 +696,7 @@
{
"cell_type": "markdown",
"id": "83858731",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Conditional Probability\n",
"\n",
@@ -820,9 +706,7 @@
{
"cell_type": "markdown",
"id": "ee5dab96",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n",
@@ -832,9 +716,7 @@
{
"cell_type": "markdown",
"id": "cb03bfdd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Bayes' Theorem\n",
"\n",
@@ -844,9 +726,7 @@
{
"cell_type": "markdown",
"id": "72b69902",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n",
@@ -856,9 +736,7 @@
{
"cell_type": "markdown",
"id": "8a876747",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which we can rewrite as"
]
@@ -866,9 +744,7 @@
{
"cell_type": "markdown",
"id": "2424f759",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n",
@@ -878,9 +754,7 @@
{
"cell_type": "markdown",
"id": "f33b3f29",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$."
]
@@ -888,9 +762,7 @@
{
"cell_type": "markdown",
"id": "b73e0693",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Interpretations of Bayes' Theorem\n",
"\n",
@@ -907,9 +779,7 @@
{
"cell_type": "markdown",
"id": "a2bc10db",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Example of Usage of Bayes' theorem\n",
"\n",
@@ -928,9 +798,7 @@
{
"cell_type": "markdown",
"id": "f41d3f9b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X=1\\vert Y=1) =0.8.\n",
@@ -940,9 +808,7 @@
{
"cell_type": "markdown",
"id": "fe20e517",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n",
"It is however not correct, as the following Bayesian analysis shows."
@@ -951,9 +817,7 @@
{
"cell_type": "markdown",
"id": "d003d5d8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Doing it correctly\n",
"\n",
@@ -964,9 +828,7 @@
{
"cell_type": "markdown",
"id": "07a5ca9c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(Y=1) =0.004.\n",
@@ -976,9 +838,7 @@
{
"cell_type": "markdown",
"id": "91485638",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have"
]
@@ -986,9 +846,7 @@
{
"cell_type": "markdown",
"id": "be6de757",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(X=1\\vert Y=0) =0.1.\n",
@@ -998,9 +856,7 @@
{
"cell_type": "markdown",
"id": "51263519",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute"
]
@@ -1008,9 +864,7 @@
{
"cell_type": "markdown",
"id": "5b2dc226",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n",
@@ -1020,9 +874,7 @@
{
"cell_type": "markdown",
"id": "e241fbc1",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!"
]
@@ -1030,9 +882,7 @@
{
"cell_type": "markdown",
"id": "42db70df",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Bayes' Theorem and Ridge and Lasso Regression\n",
"\n",
@@ -1044,9 +894,7 @@
{
"cell_type": "markdown",
"id": "ed75a460",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n",
@@ -1056,9 +904,7 @@
{
"cell_type": "markdown",
"id": "c826a95f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"is given by"
]
@@ -1066,9 +912,7 @@
{
"cell_type": "markdown",
"id": "70a426f9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n",
@@ -1078,9 +922,7 @@
{
"cell_type": "markdown",
"id": "a90b1b91",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In Bayes' theorem this function plays the role of the so-called likelihood. We could now ask the question what is the posterior probability of a parameter set $\\boldsymbol{\\beta}$ given a domain of events $\\boldsymbol{D}$? That is, how can we define the posterior probability"
]
@@ -1088,9 +930,7 @@
{
"cell_type": "markdown",
"id": "3f5a6f0b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D}).\n",
@@ -1100,9 +940,7 @@
{
"cell_type": "markdown",
"id": "2530803f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Bayes' theorem comes to our rescue here since (omitting the normalization constant)"
]
@@ -1110,9 +948,7 @@
{
"cell_type": "markdown",
"id": "34940159",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D})\\propto p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})p(\\boldsymbol{\\beta}).\n",
@@ -1122,9 +958,7 @@
{
"cell_type": "markdown",
"id": "1f497f2c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We have a model for $p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})$ but need one for the **prior** $p(\\boldsymbol{\\beta}$!"
]
@@ -1132,9 +966,7 @@
{
"cell_type": "markdown",
"id": "c780a9b9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Ridge and Bayes\n",
"\n",
@@ -1148,9 +980,7 @@
{
"cell_type": "markdown",
"id": "68ddd4cc",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta})=\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\beta_j^2}{2\\tau^2}\\right)}.\n",
@@ -1160,9 +990,7 @@
{
"cell_type": "markdown",
"id": "b85ade85",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Our posterior probability becomes then (omitting the normalization factor which is just a constant)"
]
@@ -1170,9 +998,7 @@
{
"cell_type": "markdown",
"id": "6386a328",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta\\vert\\boldsymbol{D})}=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\beta_j^2}{2\\tau^2}\\right)}.\n",
@@ -1182,9 +1008,7 @@
{
"cell_type": "markdown",
"id": "316194e8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We can now optimize this quantity with respect to $\\boldsymbol{\\beta}$. As we\n",
"did for OLS, this is most conveniently done by taking the negative\n",
@@ -1195,9 +1019,7 @@
{
"cell_type": "markdown",
"id": "f7e39504",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta})=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\frac{1}{2\\tau^2}\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_2^2,\n",
@@ -1207,9 +1029,7 @@
{
"cell_type": "markdown",
"id": "6c3b01de",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and replacing $1/2\\tau^2$ with $\\lambda$ we have"
]
@@ -1217,9 +1037,7 @@
{
"cell_type": "markdown",
"id": "5c4a05c9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta})=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_2^2,\n",
@@ -1229,9 +1047,7 @@
{
"cell_type": "markdown",
"id": "6f26b763",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which is our Ridge cost function! Nice, isn't it?"
]
@@ -1239,9 +1055,7 @@
{
"cell_type": "markdown",
"id": "08efa297",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Lasso and Bayes\n",
"\n",
@@ -1251,9 +1065,7 @@
{
"cell_type": "markdown",
"id": "97765f2c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta})=\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\vert\\beta_j\\vert}{\\tau}\\right)}.\n",
@@ -1263,9 +1075,7 @@
{
"cell_type": "markdown",
"id": "b2e48903",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Our posterior probability becomes then (omitting the normalization factor which is just a constant)"
]
@@ -1273,9 +1083,7 @@
{
"cell_type": "markdown",
"id": "fae174cf",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\vert\\beta_j\\vert}{\\tau}\\right)}.\n",
@@ -1285,9 +1093,7 @@
{
"cell_type": "markdown",
"id": "dc3ff171",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Taking the negative\n",
"logarithm of the posterior probability and leaving out the\n",
@@ -1297,9 +1103,7 @@
{
"cell_type": "markdown",
"id": "af3046b7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta}=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\frac{1}{\\tau}\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n",
@@ -1309,9 +1113,7 @@
{
"cell_type": "markdown",
"id": "fbf017df",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and replacing $1/\\tau$ with $\\lambda$ we have"
]
@@ -1319,9 +1121,7 @@
{
"cell_type": "markdown",
"id": "23214eb3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta}=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n",
@@ -1331,9 +1131,7 @@
{
"cell_type": "markdown",
"id": "f3c6b69a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which is our Lasso cost function!"
]
@@ -1341,9 +1139,7 @@
{
"cell_type": "markdown",
"id": "386493f5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Why resampling methods\n",
"\n",
@@ -1360,9 +1156,7 @@
{
"cell_type": "markdown",
"id": "727bee7f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods\n",
"Resampling methods are an indispensable tool in modern\n",
@@ -1388,9 +1182,7 @@
{
"cell_type": "markdown",
"id": "7e34ba8e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling approaches can be computationally expensive\n",
"\n",
@@ -1414,9 +1206,7 @@
{
"cell_type": "markdown",
"id": "fc35fdde",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Why resampling methods ?\n",
"**Statistical analysis.**\n",
@@ -1431,9 +1221,7 @@
{
"cell_type": "markdown",
"id": "dc071fc4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Statistical analysis\n",
"\n",
@@ -1451,9 +1239,7 @@
{
"cell_type": "markdown",
"id": "ef1325b9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods\n",
"\n",
@@ -1480,9 +1266,7 @@
{
"cell_type": "markdown",
"id": "340ea11c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods: Bootstrap\n",
"Bootstrapping is a [non-parametric approach](https://en.wikipedia.org/wiki/Nonparametric_statistics) to statistical inference\n",
@@ -1505,9 +1289,7 @@
{
"cell_type": "markdown",
"id": "74bd7468",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The Central Limit Theorem\n",
"\n",
@@ -1525,9 +1307,7 @@
{
"cell_type": "markdown",
"id": "93013ca8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"z=\\frac{x_1+x_2+\\dots+x_m}{m},\n",
@@ -1537,9 +1317,7 @@
{
"cell_type": "markdown",
"id": "92fa2b15",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"the question we pose is which is the PDF of the new variable $z$."
]
@@ -1547,9 +1325,7 @@
{
"cell_type": "markdown",
"id": "1031aebe",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Finding the Limit\n",
"\n",
@@ -1562,9 +1338,7 @@
{
"cell_type": "markdown",
"id": "cc495848",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\tilde{p}(z)=\\int dx_1p(x_1)\\int dx_2p(x_2)\\dots\\int dx_mp(x_m)\n",
@@ -1575,9 +1349,7 @@
{
"cell_type": "markdown",
"id": "28b2bcff",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where the $\\delta$-function enbodies the constraint that the mean is $z$.\n",
"All measurements that lead to each individual $x_i$ are expected to\n",
@@ -1588,9 +1360,7 @@
{
"cell_type": "markdown",
"id": "5d3eb73d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Rewriting the $\\delta$-function\n",
"\n",
@@ -1600,9 +1370,7 @@
{
"cell_type": "markdown",
"id": "75b47bae",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\delta(z-\\frac{x_1+x_2+\\dots+x_m}{m})=\\frac{1}{2\\pi}\\int_{-\\infty}^{\\infty}\n",
@@ -1613,9 +1381,7 @@
{
"cell_type": "markdown",
"id": "a99105b0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and inserting $e^{i\\mu q-i\\mu q}$ where $\\mu$ is the mean value\n",
"we arrive at"
@@ -1624,9 +1390,7 @@
{
"cell_type": "markdown",
"id": "da28e9b0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\tilde{p}(z)=\\frac{1}{2\\pi}\\int_{-\\infty}^{\\infty}\n",
@@ -1638,9 +1402,7 @@
{
"cell_type": "markdown",
"id": "5bd0da08",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"with the integral over $x$ resulting in"
]
@@ -1648,9 +1410,7 @@
{
"cell_type": "markdown",
"id": "8dcbd91d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\int_{-\\infty}^{\\infty}dxp(x)\\exp{\\left(iq(\\mu-x)/m\\right)}=\n",
@@ -1662,9 +1422,7 @@
{
"cell_type": "markdown",
"id": "3ab26352",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Identifying Terms\n",
"\n",
@@ -1675,9 +1433,7 @@
{
"cell_type": "markdown",
"id": "9e232448",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\int_{-\\infty}^{\\infty}dxp(x)e^{\\left(iq(\\mu-x)/m\\right)}=\n",
@@ -1688,9 +1444,7 @@
{
"cell_type": "markdown",
"id": "15cdad60",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"resulting in"
]
@@ -1698,9 +1452,7 @@
{
"cell_type": "markdown",
"id": "7b857d11",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\left[\\int_{-\\infty}^{\\infty}dxp(x)\\exp{\\left(iq(\\mu-x)/m\\right)}\\right]^m\\approx\n",
@@ -1711,9 +1463,7 @@
{
"cell_type": "markdown",
"id": "a7ba76a4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and in the limit $m\\rightarrow \\infty$ we obtain"
]
@@ -1721,9 +1471,7 @@
{
"cell_type": "markdown",
"id": "27abee5c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\tilde{p}(z)=\\frac{1}{\\sqrt{2\\pi}(\\sigma/\\sqrt{m})}\n",
@@ -1734,9 +1482,7 @@
{
"cell_type": "markdown",
"id": "450fbae6",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which is the normal distribution with variance\n",
"$\\sigma^2_m=\\sigma^2/m$, where $\\sigma$ is the variance of the PDF $p(x)$\n",
@@ -1746,9 +1492,7 @@
{
"cell_type": "markdown",
"id": "583a9a42",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Wrapping it up\n",
"\n",
@@ -1765,9 +1509,7 @@
{
"cell_type": "markdown",
"id": "cdd93e78",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\sigma_m=\n",
@@ -1778,9 +1520,7 @@
{
"cell_type": "markdown",
"id": "dfe218b6",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The latter is true only if the average value is known exactly. This is obtained in the limit\n",
"$m\\rightarrow \\infty$ only. Because the mean and the variance are measured quantities we obtain \n",
@@ -1790,9 +1530,7 @@
{
"cell_type": "markdown",
"id": "5d3fd1f7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\sigma_m\\approx \n",
@@ -1803,9 +1541,7 @@
{
"cell_type": "markdown",
"id": "3b8a47f4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In many cases however the above estimate for the standard deviation,\n",
"in particular if correlations are strong, may be too simplistic. Keep\n",
@@ -1823,9 +1559,7 @@
{
"cell_type": "markdown",
"id": "c57ef602",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Confidence Intervals\n",
"\n",
@@ -1846,9 +1580,7 @@
{
"cell_type": "markdown",
"id": "cdea97fa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Standard Approach based on the Normal Distribution\n",
"\n",
@@ -1861,9 +1593,7 @@
{
"cell_type": "markdown",
"id": "efc3abe4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\left(\\mu_{\\beta}\\pm \\frac{z\\sigma_{\\beta}}{\\sqrt{n}}\\right),\n",
@@ -1873,9 +1603,7 @@
{
"cell_type": "markdown",
"id": "1679ffac",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $z$ defines the level of certainty (or confidence). For a normal\n",
"distribution typical parameters are $z=2.576$ which corresponds to a\n",
@@ -1893,9 +1621,7 @@
{
"cell_type": "markdown",
"id": "fc2481bb",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods: Bootstrap background\n",
"\n",
@@ -1913,9 +1639,7 @@
{
"cell_type": "markdown",
"id": "a18c7fde",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods: More Bootstrap background\n",
"\n",
@@ -1937,9 +1661,7 @@
{
"cell_type": "markdown",
"id": "4af5f00a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods: Bootstrap approach\n",
"\n",
@@ -1958,9 +1680,7 @@
{
"cell_type": "markdown",
"id": "f40537a7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Resampling methods: Bootstrap steps\n",
"\n",
@@ -1988,9 +1708,7 @@
{
"cell_type": "markdown",
"id": "c6459716",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Code example for the Bootstrap method\n",
"\n",
@@ -2012,10 +1730,7 @@
"cell_type": "code",
"execution_count": 1,
"id": "61ebf590",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
@@ -2051,9 +1766,7 @@
{
"cell_type": "markdown",
"id": "2bcfb7ee",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We see that our new variance and from that the standard deviation, agrees with the central limit theorem."
]
@@ -2061,9 +1774,7 @@
{
"cell_type": "markdown",
"id": "bb8e2e4c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Plotting the Histogram"
]
@@ -2072,10 +1783,7 @@
"cell_type": "code",
"execution_count": 2,
"id": "4d167410",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# the histogram of the bootstrapped data (normalized data if density = True)\n",
@@ -2092,9 +1800,7 @@
{
"cell_type": "markdown",
"id": "5b04a99c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The bias-variance tradeoff\n",
"\n",
@@ -2110,9 +1816,7 @@
{
"cell_type": "markdown",
"id": "df8b5b83",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{y}=f(\\boldsymbol{x}) + \\boldsymbol{\\epsilon}\n",
@@ -2122,9 +1826,7 @@
{
"cell_type": "markdown",
"id": "8b1cae6d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $\\epsilon$ is normally distributed with mean zero and standard deviation $\\sigma^2$.\n",
"\n",
@@ -2139,9 +1841,7 @@
{
"cell_type": "markdown",
"id": "347294eb",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right].\n",
@@ -2151,9 +1851,7 @@
{
"cell_type": "markdown",
"id": "0536e454",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We can rewrite this as"
]
@@ -2161,9 +1859,7 @@
{
"cell_type": "markdown",
"id": "4edf3a9e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\frac{1}{n}\\sum_i(f_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\sigma^2.\n",
@@ -2173,9 +1869,7 @@
{
"cell_type": "markdown",
"id": "95b5e144",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The three terms represent the square of the bias of the learning\n",
"method, which can be thought of as the error caused by the simplifying\n",
@@ -2190,9 +1884,7 @@
{
"cell_type": "markdown",
"id": "4ec0202c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}})^2\\right],\n",
@@ -2202,9 +1894,7 @@
{
"cell_type": "markdown",
"id": "3729c884",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and adding and subtracting $\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]$ we get"
]
@@ -2212,9 +1902,7 @@
{
"cell_type": "markdown",
"id": "09d292c0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}}+\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right],\n",
@@ -2224,9 +1912,7 @@
{
"cell_type": "markdown",
"id": "9393b969",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which, using the abovementioned expectation values can be rewritten as"
]
@@ -2234,9 +1920,7 @@
{
"cell_type": "markdown",
"id": "31400952",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right]+\\mathrm{Var}\\left[\\boldsymbol{\\tilde{y}}\\right]+\\sigma^2,\n",
@@ -2246,9 +1930,7 @@
{
"cell_type": "markdown",
"id": "fab9fd56",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"that is the rewriting in terms of the so-called bias, the variance of the model $\\boldsymbol{\\tilde{y}}$ and the variance of $\\boldsymbol{\\epsilon}$."
]
@@ -2256,9 +1938,7 @@
{
"cell_type": "markdown",
"id": "f6bbceee",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## A way to Read the Bias-Variance Tradeoff\n",
"\n",
@@ -2272,9 +1952,7 @@
{
"cell_type": "markdown",
"id": "2486e572",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Example code for Bias-Variance tradeoff"
]
@@ -2283,10 +1961,7 @@
"cell_type": "code",
"execution_count": 3,
"id": "af100ade",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
@@ -2348,22 +2023,104 @@
{
"cell_type": "markdown",
"id": "e4b4ea82",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Understanding what happens"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 1,
"id": "13bb228b",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Polynomial degree: 0\n",
+ "Error: 0.32149601703519115\n",
+ "Bias^2: 0.3123314713548606\n",
+ "Var: 0.009164545680330616\n",
+ "0.32149601703519115 >= 0.3123314713548606 + 0.009164545680330616 = 0.3214960170351912\n",
+ "Polynomial degree: 1\n",
+ "Error: 0.08426840630693412\n",
+ "Bias^2: 0.0796891867672603\n",
+ "Var: 0.004579219539673834\n",
+ "0.08426840630693412 >= 0.0796891867672603 + 0.004579219539673834 = 0.08426840630693413\n",
+ "Polynomial degree: 2\n",
+ "Error: 0.10398646080125037\n",
+ "Bias^2: 0.10077114273548984\n",
+ "Var: 0.0032153180657605116\n",
+ "0.10398646080125037 >= 0.10077114273548984 + 0.0032153180657605116 = 0.10398646080125036\n",
+ "Polynomial degree: 3\n",
+ "Error: 0.06547790180152352\n",
+ "Bias^2: 0.062082386342319454\n",
+ "Var: 0.0033955154592040923\n",
+ "0.06547790180152352 >= 0.062082386342319454 + 0.0033955154592040923 = 0.06547790180152355\n",
+ "Polynomial degree: 4\n",
+ "Error: 0.06844519414009445\n",
+ "Bias^2: 0.06453579006728322\n",
+ "Var: 0.003909404072811221\n",
+ "0.06844519414009445 >= 0.06453579006728322 + 0.003909404072811221 = 0.06844519414009444\n",
+ "Polynomial degree: 5\n",
+ "Error: 0.05227921801205679\n",
+ "Bias^2: 0.04818727730430286\n",
+ "Var: 0.004091940707753925\n",
+ "0.05227921801205679 >= 0.04818727730430286 + 0.004091940707753925 = 0.05227921801205679\n",
+ "Polynomial degree: 6\n",
+ "Error: 0.03781367141738902\n",
+ "Bias^2: 0.03365768507152769\n",
+ "Var: 0.0041559863458613296\n",
+ "0.03781367141738902 >= 0.03365768507152769 + 0.0041559863458613296 = 0.03781367141738902\n",
+ "Polynomial degree: 7\n",
+ "Error: 0.027609773491022394\n",
+ "Bias^2: 0.022999498260366198\n",
+ "Var: 0.004610275230656182\n",
+ "0.027609773491022394 >= 0.022999498260366198 + 0.004610275230656182 = 0.02760977349102238\n",
+ "Polynomial degree: 8\n",
+ "Error: 0.017355848195593312\n",
+ "Bias^2: 0.010331721306655165\n",
+ "Var: 0.007024126888938144\n",
+ "0.017355848195593312 >= 0.010331721306655165 + 0.007024126888938144 = 0.01735584819559331\n",
+ "Polynomial degree: 9\n",
+ "Error: 0.026605727637184558\n",
+ "Bias^2: 0.010018312644139219\n",
+ "Var: 0.016587414993045335\n",
+ "0.026605727637184558 >= 0.010018312644139219 + 0.016587414993045335 = 0.026605727637184554\n",
+ "Polynomial degree: 10\n",
+ "Error: 0.021592704588021178\n",
+ "Bias^2: 0.010516485576646504\n",
+ "Var: 0.01107621901137467\n",
+ "0.021592704588021178 >= 0.010516485576646504 + 0.01107621901137467 = 0.021592704588021174\n",
+ "Polynomial degree: 11\n",
+ "Error: 0.07160048164232538\n",
+ "Bias^2: 0.014436800088896381\n",
+ "Var: 0.05716368155342902\n",
+ "0.07160048164232538 >= 0.014436800088896381 + 0.05716368155342902 = 0.0716004816423254\n",
+ "Polynomial degree: 12\n",
+ "Error: 0.11547777218876518\n",
+ "Bias^2: 0.016285782696017142\n",
+ "Var: 0.09919198949274803\n",
+ "0.11547777218876518 >= 0.016285782696017142 + 0.09919198949274803 = 0.11547777218876518\n",
+ "Polynomial degree: 13\n",
+ "Error: 0.2284246870217162\n",
+ "Bias^2: 0.01975416527168255\n",
+ "Var: 0.20867052175003364\n",
+ "0.2284246870217162 >= 0.01975416527168255 + 0.20867052175003364 = 0.2284246870217162\n"
+ ]
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABx/klEQVR4nO3dd3xUZd7+8c/MJJk00kMCpBB6VZr0asHeVhdsuK6oy4K7Ivv8VFZdy65iWQUbKu4q6gqii21dXIlKbyKCjd6SEFJIAul15vz+mGSYIQkQSDKT5Ho/r3klOXPm5Dtxn+TiPvf9vU2GYRiIiIiIeDGzpwsQERERORUFFhEREfF6CiwiIiLi9RRYRERExOspsIiIiIjXU2ARERERr6fAIiIiIl5PgUVERES8no+nC2gsdrudw4cP065dO0wmk6fLERERkdNgGAaFhYV07NgRs7n+cZRWE1gOHz5MfHy8p8sQERGRM5CWlkZcXFy9z7eawNKuXTvA8YZDQkI8XI2IiIicjoKCAuLj451/x+vTagJLzW2gkJAQBRYREZEW5lTTOTTpVkRERLyeAouIiIh4PQUWERER8XqtZg6LiIi0HTabjcrKSk+XIafBYrHg4+Nz1i1HFFhERKRFKSoq4tChQxiG4elS5DQFBgbSoUMH/Pz8zvgaCiwiItJi2Gw2Dh06RGBgINHR0WoU6uUMw6CiooIjR45w4MABunfvftLmcCejwCIiIi1GZWUlhmEQHR1NQECAp8uR0xAQEICvry8pKSlUVFTg7+9/RtfRpFsREWlxNLLSspzpqIrbNRqhDhEREZEmpcAiIiIiXk+BRURERLyeAouIiEgTu+222zCZTLUel1xyiadLazG0SugU3l5/kO2HC/j9+K50jgrydDkiItJCXXLJJbz11ltux6xWa53nVlZW4uvre8pjp+NMX+dtNMJyCh9tTWfJd2nszCzwdCkiInICwzAoqajyyKOhjeusViuxsbFuj/DwcMCx6um1117j6quvJigoiL/97W88+uijDBgwgDfffJMuXbpgtVoxDIPU1FSuvvpqgoODCQkJYdKkSWRlZTm/T32va+k0wnIKnSMD+SHtGAdzSzxdioiInKC00kafv3zpke+9/fGLCfRrvD+jjzzyCHPmzGHu3LlYLBbeeust9u7dywcffMDSpUuxWCwAXHPNNQQFBbFq1SqqqqqYPn06kydPZuXKlc5r1fW6lk6B5RQSIwIBSFFgERGRs/D5558THBzsduz+++/n4YcfBuCmm27i9ttvd3u+oqKCd999l+joaACSk5P58ccfOXDgAPHx8QC8++679O3bl82bN3PeeefV+brWQIHlFBIjHfNWUnKLPVyJiIicKMDXwvbHL/bY926ICRMm8Oqrr7odi4iIcH4+ZMiQWq9JTEx0Cx07duwgPj7eGVYA+vTpQ1hYGDt27HAGlhNf1xoosJxCYqRGWEREvJXJZGrU2zJNKSgoiG7dup30+VMdMwyjzi6/Jx6v61otnSbdnkLNCEtGfinlVTYPVyMiIm1Znz59SE1NJS0tzXls+/bt5Ofn07t3bw9W1vRaRiz1oKhgPwL9LJRU2Dh0tJSu0cGnfpGIiMgJysvLyczMdDvm4+NDVFTUaV/jwgsv5JxzzuHmm29m3rx5zkm348aNq/OWUmuiEZZTMJlMzlGWVN0WEhGRM/S///2PDh06uD1Gjx7doGuYTCY++eQTwsPDGTt2LBdeeCFdunRhyZIlTVS19zAZrWFxNlBQUEBoaCj5+fmEhIQ06rWnvfMd/9ueySNX9uW3o5Ia9doiInL6ysrKOHDgAElJSfj7+3u6HDlNJ/vvdrp/vzXCciqLb2RuyjUMMu3RxFsREREPUWA5lYoiAmyFdDZlammziIiIhyiwnEpEFwASzVmk5GmERURExBMUWE6lOrB0NmWRlleCzd4qpvyIiIi0KAospxLRFXAElkqbQUZ+qYcLEhERaXsUWE6leoQlyezYCVMTb0VERJqfAsuphHcGIIQiQilSYBEREfEABZZT8QuEdh0BSDJlkpKnlUIiIiLNTYHldNSsFDJlkpKjERYREWmY8ePHM3PmzHqf79y5M/PmzWu2eloi7SV0OiKSIGUtnU1ZLNfSZhERaWSbN29ulTssN6YzGmGZP3++s73u4MGDWbNmTb3nrl27llGjRhEZGUlAQAC9evVi7ty5tc5bunQpffr0wWq10qdPHz7++OMzKa1puPZiyS2mlexmICIiXiI6OprAwEBPl+HVGhxYlixZwsyZM3nwwQfZunUrY8aM4dJLLyU1NbXO84OCgrj77rtZvXo1O3bs4KGHHuKhhx5iwYIFznM2bNjA5MmTmTJlCj/88ANTpkxh0qRJbNq06czfWWOKrFnanElJhY2cogoPFyQiIi1NVVUVd999N2FhYURGRvLQQw85/wF84i2h559/nv79+xMUFER8fDzTp0+nqKjI+XxKSgpXXnkl4eHhBAUF0bdvX5YtW9bcb6lZNfiW0PPPP8/UqVO54447AJg3bx5ffvklr776KnPmzKl1/sCBAxk4cKDz686dO/PRRx+xZs0a7rrrLuc1LrroImbPng3A7NmzWbVqFfPmzWPx4sVn9MYalXNpczYAKbnFRLezerIiEREBMAyo9NCtet9AMJlO+/S3336bqVOnsmnTJr777jvuuusuEhMTufPOO2udazabefHFF+ncuTMHDhxg+vTp3HfffcyfPx+AGTNmUFFRwerVqwkKCmL79u0EBwc32lvzRg0KLBUVFWzZsoUHHnjA7fjEiRNZv379aV1j69atrF+/nr/97W/OYxs2bODee+91O+/iiy8+6QSk8vJyysvLnV8XFBSc1vc/I+GOHZrDKSCEYlJySxjSOaLpvp+IiJyeyhJ4sqNnvvefD4Pf6c87iY+PZ+7cuZhMJnr27MlPP/3E3Llz6wwsrhN0k5KS+Otf/8rvf/97Z2BJTU3luuuuo3///gB06dLl7N5LC9CgW0I5OTnYbDZiYmLcjsfExJCZmXnS18bFxWG1WhkyZAgzZsxwjtAAZGZmNviac+bMITQ01PmIj49vyFtpGGswBDvqSzRlaRNEERFpsOHDh2NyGZEZMWIEe/bswWaz1Tp3xYoVXHTRRXTq1Il27dpx6623kpubS3Gx4+/PH//4R/72t78xatQoHnnkEX788cdmex+eckarhEwnDIEZhlHr2InWrFlDUVERGzdu5IEHHqBbt27ceOONZ3zN2bNnM2vWLOfXBQUFTRtaIrpAUZZj12atFBIR8Q6+gY6RDk997yaQkpLCZZddxrRp0/jrX/9KREQEa9euZerUqVRWVgJwxx13cPHFF/Pf//6X5cuXM2fOHJ577jn+8Ic/NElN3qBBgSUqKgqLxVJr5CM7O7vWCMmJkpIct1X69+9PVlYWjz76qDOwxMbGNviaVqsVq7UZ55FEdIHUDSSaslijbrciIt7BZGrQbRlP2rhxY62vu3fvjsVicTv+3XffUVVVxXPPPYfZ7LgR8sEHH9S6Xnx8PNOmTWPatGnMnj2bN954o1UHlgbdEvLz82Pw4MEkJye7HU9OTmbkyJGnfR3DMNzmn4wYMaLWNZcvX96gaza5ml2bzVmk6paQiIg0UFpaGrNmzWLXrl0sXryYl156iXvuuafWeV27dqWqqoqXXnqJ/fv38+677/Laa6+5nTNz5ky+/PJLDhw4wPfff88333xD7969m+uteESDbwnNmjWLKVOmMGTIEEaMGMGCBQtITU1l2rRpgONWTXp6Ou+88w4Ar7zyCgkJCfTq1Qtw9GX5+9//7pYC77nnHsaOHcvTTz/N1VdfzaeffspXX33F2rVrG+M9Ng6XbrdHSyrJL60kNMDXw0WJiEhLceutt1JaWsrQoUOxWCz84Q9/cK6WdTVgwACef/55nn76aWbPns3YsWOZM2cOt956q/Mcm83GjBkzOHToECEhIVxyySV19jhrVYwz8MorrxiJiYmGn5+fMWjQIGPVqlXO537zm98Y48aNc3794osvGn379jUCAwONkJAQY+DAgcb8+fMNm83mds0PP/zQ6Nmzp+Hr62v06tXLWLp0aYNqys/PNwAjPz//TN7SqaVvNYxHQowjjyQYifd/bvyYdqxpvo+IiNSrtLTU2L59u1FaWurpUqQBTvbf7XT/fpsMo3W0bS0oKCA0NJT8/HxCQkIa/xuU5cNTCQD0K/sHT900iivO8dBSOhGRNqqsrIwDBw44u61Ly3Cy/26n+/dbmx+eLv9QCIwCINGUTYom3oqIiDQbBZaGcN21WRNvRUREmo0CS0PUrBQyZXFQIywiIiLNRoGlIao3QUw0ZZGqwCIiItJsFFgawtmLJZPMgjLKKmu3UxYREZHGp8DSEBGObr1J5iwAUtWiX0REpFkosDRE9QhLe44SQBkHczTxVkREpDkosDREQLjjgWNps0ZYREREmocCS0O5LG0+qKXNIiLiQSaTiU8++cTTZTQLBZaGinCsFOpsylLzOBEROaUrr7ySCy+8sM7nNmzYgMlk4vvvvz+ja2dkZHDppZeeTXkthgJLQ7mMsOiWkIiInMrUqVP55ptvSElJqfXcm2++yYABAxg0aFCDrllRUQFAbGwsVqu1Uer0dgosDVUdWJJMWRw6Wkqlze7hgkRExJtdccUVtG/fnoULF7odLykpYcmSJVxzzTXceOONxMXFERgYSP/+/Vm8eLHbuePHj+fuu+9m1qxZREVFcdFFFwG1bwndf//99OjRg8DAQLp06cLDDz9MZWWl8/lHH32UAQMG8O6779K5c2dCQ0O54YYbKCwsdJ5jt9t5+umn6datG1arlYSEBJ544gnn8+np6UyePJnw8HAiIyO5+uqrOXjwYOP9wOqhwNJQLr1YbHaDw8dKPVyQiEjbZRgGJZUlHnmc7t7BPj4+3HrrrSxcuNDtNR9++CEVFRXccccdDB48mM8//5yff/6Zu+66iylTprBp0ya367z99tv4+Piwbt06Xn/99Tq/V7t27Vi4cCHbt2/nhRde4I033mDu3Llu5+zbt49PPvmEzz//nM8//5xVq1bx1FNPOZ+fPXs2Tz/9NA8//DDbt29n0aJFxMTEAI6QNWHCBIKDg1m9ejVr164lODiYSy65xDnq01S0W3NDFefCs47Q0rNsIQtuH824HtFN9/1ERMTpxF1/SypLGLZomEdq2XTTJgJ9A0/r3J07d9K7d2+++eYbJkyYAMC4cePo1KkTixYtqnX+5ZdfTu/evfn73/8OOEZY8vPz2bp1q9t5JpOJjz/+mGuuuabO7/vss8+yZMkSvvvuO8AxwvLss8+SmZlJu3btALjvvvtYvXo1GzdupLCwkOjoaF5++WXuuOOOWtd78803eeaZZ9ixYwcmkwlw3J4KCwvjk08+YeLEiXXW0Ri7NfvU+4zULTACrKFQnk+CKZvU3GJAgUVEROrXq1cvRo4cyZtvvsmECRPYt28fa9asYfny5dhsNp566imWLFlCeno65eXllJeXExQU5HaNIUOGnPL7/Pvf/2bevHns3buXoqIiqqqqaoWAzp07O8MKQIcOHcjOzgZgx44dlJeXc8EFF9R5/S1btrB3716314MjkOzbt++0fhZnSoGloUwmiOwCh7fS2ZSpTRBFRDwowCeATTdtOvWJTfS9G2Lq1KncfffdvPLKK7z11lskJiZywQUX8OyzzzJ37lzmzZtH//79CQoKYubMmbVusZwYYE60ceNGbrjhBh577DEuvvhiQkNDef/993nuuefczvP19XX72mQyYbc75mMGBJz8PdntdgYPHsx7771X67no6Kb9x7sCy5mIcASWRO3aLCLiUSaT6bRvy3japEmTuOeee1i0aBFvv/02d955JyaTiTVr1nD11Vdzyy23AI5QsGfPHnr37t2g669bt47ExEQefPBB57G6ViadTPfu3QkICODrr7+u85bQoEGDWLJkCe3bt2/a6Rd10KTbM1Ez8daUSYqax4mIyGkIDg5m8uTJ/PnPf+bw4cPcdtttAHTr1o3k5GTWr1/Pjh07+N3vfkdmZmaDr9+tWzdSU1N5//332bdvHy+++CIff/xxg67h7+/P/fffz3333cc777zDvn372LhxI//85z8BuPnmm4mKiuLqq69mzZo1HDhwgFWrVnHPPfdw6NChBtfcEAosZ8IlsKTmlWC3t4p5yyIi0sSmTp3K0aNHufDCC0lISADg4YcfZtCgQVx88cWMHz+e2NjYeifRnszVV1/Nvffey913382AAQNYv349Dz/8cIOv8/DDD/OnP/2Jv/zlL/Tu3ZvJkyc757gEBgayevVqEhIS+NWvfkXv3r25/fbbKS0tbfIRF60SOhOpG+HNizlkRDG6/EU2zD6fDqENu5cpIiINd7LVJuK9GmOVkEZYzkT1CEtHUy5+VKpFv4iISBNTYDkTQdHgF4wZg3hTtuaxiIiINDEFljNhMrnsKaRNEEVERJqaAsuZck68zSJFmyCKiIg0KQWWM6WlzSIiIs1GgeVMuQWW098ES0REzp5+57YsjfHfS4HlTLnMYSksq+JoSeUpXiAiImfLYrEANPnOwNK4SkocUydO3BagIdSa/0xVB5Y4cw6+VJGSW0xEkJ+HixIRad18fHwIDAzkyJEj+Pr6Yjbr393ezDAMSkpKyM7OJiwszBk4z4QCy5lqFwu+gVgqS+hkOkJKbgkDE8I9XZWISKtmMpno0KEDBw4caPA+OeI5YWFhxMbGntU1FFjOVM3S5qyfHSuFtLRZRKRZ+Pn50b17d90WaiF8fX3PamSlhgLL2YhIqg4sWikkItKczGazWvO3Mbr5dzZcm8epF4uIiEiTUWA5G+rFIiIi0iwUWM6GywhLTlEFReVVHi5IRESkdVJgORvVgSXefAQLNlI18VZERKRJKLCcjXYdwccfX2x0NOXotpCIiEgTUWA5G2YzhCcB2gRRRESkKSmwnC1NvBUREWlyCixnK8JlhEVzWERERJqEAsvZcq4UylRgERERaSIKLGfLeUsoi8P5pZRX2TxckIiISOujwHK2IrsCkGDKxmTYScsr9XBBIiIirc8ZBZb58+eTlJSEv78/gwcPZs2aNfWe+9FHH3HRRRcRHR1NSEgII0aM4Msvv3Q7Z+HChZhMplqPsrKyMymveYV0AosffqYqOpBLap4m3oqIiDS2BgeWJUuWMHPmTB588EG2bt3KmDFjuPTSS0lNTa3z/NWrV3PRRRexbNkytmzZwoQJE7jyyivZunWr23khISFkZGS4PVrExlZmC4R3BqCzOZODOZrHIiIi0tgavFvz888/z9SpU7njjjsAmDdvHl9++SWvvvoqc+bMqXX+vHnz3L5+8skn+fTTT/nPf/7DwIEDncdNJhOxsbENLcc7RHSBnN10NmWRql4sIiIija5BIywVFRVs2bKFiRMnuh2fOHEi69evP61r2O12CgsLiYiIcDteVFREYmIicXFxXHHFFbVGYE5UXl5OQUGB28NjXHdtVi8WERGRRtegwJKTk4PNZiMmJsbteExMDJmZmad1jeeee47i4mImTZrkPNarVy8WLlzIZ599xuLFi/H392fUqFHs2bOn3uvMmTOH0NBQ5yM+Pr4hb6VxuTWP0wiLiIhIYzujSbcmk8nta8Mwah2ry+LFi3n00UdZsmQJ7du3dx4fPnw4t9xyC+eeey5jxozhgw8+oEePHrz00kv1Xmv27Nnk5+c7H2lpaWfyVhqHywhL2tESbHbDc7WIiIi0Qg2awxIVFYXFYqk1mpKdnV1r1OVES5YsYerUqXz44YdceOGFJz3XbDZz3nnnnXSExWq1YrVaT7/4puQSWKpsNg4fKyU+ItDDRYmIiLQeDRph8fPzY/DgwSQnJ7sdT05OZuTIkfW+bvHixdx2220sWrSIyy+//JTfxzAMtm3bRocOHRpSnueExoPZB39TJTEc1cRbERGRRtbgVUKzZs1iypQpDBkyhBEjRrBgwQJSU1OZNm0a4LhVk56ezjvvvAM4wsqtt97KCy+8wPDhw52jMwEBAYSGhgLw2GOPMXz4cLp3705BQQEvvvgi27Zt45VXXmms99m0LD4Qlgh5+0gyZ3Iwt5hR3aI8XZWIiEir0eDAMnnyZHJzc3n88cfJyMigX79+LFu2jMTERAAyMjLcerK8/vrrVFVVMWPGDGbMmOE8/pvf/IaFCxcCcOzYMe666y4yMzMJDQ1l4MCBrF69mqFDh57l22tGEV0gbx+JpixSNfFWRESkUZkMw2gVM0QLCgoIDQ0lPz+fkJCQ5i9g2X3w7eu8VnUlW3vO5PUpQ5q/BhERkRbmdP9+ay+hxqJdm0VERJqMAktjqd4EsabbbSsZuBIREfEKCiyNxWVpc0lFFUeKyj1ckIiISOuhwNJYQuPBZCHQVE57jum2kIiISCNSYGksPn4Q5tgeQC36RUREGpcCS2OquS1kziJVmyCKiIg0GgWWxuSyCeJBjbCIiIg0GgWWxhThWCmUaMoiRe35RUREGo0CS2NyjrBkkaJbQiIiIo1GgaUxudwSOlZSQX5JpYcLEhERaR0UWBpTeCJgIthURhQFpORplEVERKQxKLA0Jh+rox8LatEvIiLSmBRYGltEEqB5LCIiIo1JgaWxOXuxaIRFRESksSiwNDaXTRAVWEREpDVI3p7F2j05VFTZPVaDAktjc9kEUZNuRUSkNZjzxQ5u+ecmvtqR5bEaFFgaW3VgSTJlklVQRmmFzcMFiYiInLmU3GL2HynGx2xidPcoj9WhwNLYwjsDEGIqIZxCUtXxVkREWrAVO7MBGNI5nBB/X4/VocDS2HwDIKQToJVCIiLS8q3YdQSACT3be7QOBZam4DqPRRNvRUSkhSqpqGLD/lwAzu+lwNL61LToN2dq4q2IiLRYG/blUlFlp1NYAN3aB3u0FgWWpqARFhERaQW+qZ6/cn6v9phMJo/WosDSFFxWCimwiIhIS2QYBitr5q/0ivZwNQosTcNlhCX9WCmVNs812hERETkTu7OKSD9WitXHzIgunlvOXEOBpSlU7ycUbioi2F5I+tFSDxckIiLSMCt2OW4HjegaSYCfxcPVKLA0Db8gCI4FHKMsB7W0WUREWhjX+SveQIGlqbjsKaTmcSIi0pLkl1ayJeUo4Pn+KzUUWJpK9W2hRFMmB3MUWEREpOVYs+cINrtBt/bBxEcEerocQIGl6bj0YklVLxYREWlBVuys6W7r+dVBNRRYmkpNYFEvFhERaUHsdoNVux3zVyZ4yfwVUGBpOq7N4/JKsNsNDxckIiJyaj+l55NTVEGw1YchiRGeLsdJgaWphDvmsESZCrBWFZFVWObhgkRERE6tZjnzmO5R+Pl4T0zwnkpaG/8QCHLc+0swZWnirYiItAgrqpcze8vqoBoKLE0pwnVpsybeioiIdztSWM4Ph/IBGO9FE25BgaVpOeexZHJQE29FRMTLrdrtWB3Ur1MI7UP8PVyNOx9PF9CquWyCuFKBRUREvFzN/JUTbwct2bkEk8nEBQkXEBkQ6YnSFFiaVE3zOLPa84uIiHertNlZvbtmd+bjgcUwDN746Q2ySrKIDYplbNxYj9SnW0JNyaUXS2puCYahpc0iIuKdvk85SmFZFRFBfpwbF+Y8vjNvJ1klWQT4BDCswzCP1afA0pSqA0t70zFs5UXkFVd4uCAREZG6fVN9O2hcj2gsZpPz+MpDKwEY3mE4VovVE6UBCixNKyAMAh33+moayImIiHijldXt+E9cHbQ6bbXjePz45i7JjQJLU3PpeJuqibciIuKF0o+VsiurELPJMcJS40jJEX7O/RnAY3NXaiiwNDWXlUKaeCsiIt6oplncoIRwwgL9nMdXH3KMrvSL7EdUQJRHaqtxRoFl/vz5JCUl4e/vz+DBg1mzZk2953700UdcdNFFREdHExISwogRI/jyyy9rnbd06VL69OmD1WqlT58+fPzxx2dSmvdx6cWiERYREfFGzu62J2x2WDN/ZVz8uOYuqZYGB5YlS5Ywc+ZMHnzwQbZu3cqYMWO49NJLSU1NrfP81atXc9FFF7Fs2TK2bNnChAkTuPLKK9m6davznA0bNjB58mSmTJnCDz/8wJQpU5g0aRKbNm0683fmLWpWCmlps4iIeKGyShvr9uUA7v1XyqrK2Hh4I+D5+SsAJqOBa22HDRvGoEGDePXVV53HevfuzTXXXMOcOXNO6xp9+/Zl8uTJ/OUvfwFg8uTJFBQU8MUXXzjPueSSSwgPD2fx4sWndc2CggJCQ0PJz88nJCSkAe+oiR36Dv5xARlGBFf6LuC7hy7ydEUiIiJOK3dlc9tbm4kN8WfD7PMxmRwrhFYfWs2Mr2cQGxTL8uuWO483ttP9+92gEZaKigq2bNnCxIkT3Y5PnDiR9evXn9Y17HY7hYWFREQc37J6w4YNta558cUXn/Sa5eXlFBQUuD28UvUISwdTHkVFhRSVV3m4IBERkeNW7qppFhftFkpWpa0CYFzcuCYLKw3RoMCSk5ODzWYjJibG7XhMTAyZmZmndY3nnnuO4uJiJk2a5DyWmZnZ4GvOmTOH0NBQ5yM+Pr4B76QZBUaAfxgACaZsUnRbSEREvIRhGHxTx+7MhmGw6pAjsHh6dVCNM5p0e2LSMgzjtNLX4sWLefTRR1myZAnt27tP7GnoNWfPnk1+fr7zkZaW1oB30MycHW8zSdHEWxER8RL7c4pJzSvBz2JmVLfjq4C8pbutqwbtJRQVFYXFYqk18pGdnV1rhORES5YsYerUqXz44YdceOGFbs/FxsY2+JpWqxWr1XMd9xokogsc/l6BRUREvErN6qBhXSIIsh6PBN7S3dZVg0ZY/Pz8GDx4MMnJyW7Hk5OTGTlyZL2vW7x4MbfddhuLFi3i8ssvr/X8iBEjal1z+fLlJ71mi+Kyp5BuCYmIiLeo2Z15/Am7M9fMX/GG1UE1Grxb86xZs5gyZQpDhgxhxIgRLFiwgNTUVKZNmwY4btWkp6fzzjvvAI6wcuutt/LCCy8wfPhw50hKQEAAoaGhANxzzz2MHTuWp59+mquvvppPP/2Ur776irVr1zbW+/Qsl263/9EIi4iIeIGi8iq+PZAHwISe7t1tf8n9BfCe+StwBnNYJk+ezLx583j88ccZMGAAq1evZtmyZSQmJgKQkZHh1pPl9ddfp6qqihkzZtChQwfn45577nGeM3LkSN5//33eeustzjnnHBYuXMiSJUsYNsw77pudtZrAYtYIi4iIeIe1e3KotBl0jgykS3Sw83hNd9v+Uf093t3WVYNHWACmT5/O9OnT63xu4cKFbl+vXLnytK55/fXXc/31159JOd4vsisAHcklt6CA8iobVh+Lh4sSEZG2bGU9t4Nq5q940+gKaC+h5hEYiWENwWwyiOMIaXmlnq5IRETaMMMwnPNXzu/lvd1tXSmwNAeTCVNEEuDYBFG3hURExJO2ZxSQVVBOgK+FoUnHG7l+m/ktZbYyYoNi6Rne04MV1qbA0lxcNkHU0mYREfGkmuXMo7pF4e97fIrCyrSVgPd0t3WlwNJctLRZRES8xAqXdvw1XLvbjovz/O7MJ1JgaS4uS5tT8jTCIiIinnG0uIKtqUcB93b8O/N2kl2STYBPAEM7DPVUefVSYGkuEY6VQup2KyIinrR6zxHsBvSKbUfHsADncW/sbutKgaW5VI+wdDLlkHW0gCqb3cMFiYhIW+Tc7LCX93e3daXA0lyC22P4BmExGcTas8jIL/N0RSIi0sbY7AardlfPX3G5HZRdku2V3W1dKbA0F5MJk+s8Ft0WEhGRZrYt7SjHSioJ8fdhUEKY87i3drd1pcDSnKp7sXQ2ZXJQK4VERKSZrdjpGF0Z2yMaH8vxCODNq4NqKLA0J5cRllStFBIRkWZWM3+lpXS3daXA0pzUi0VERDwkM7+M7RkFmEyOEZYart1te4T38GCFJ6fA0pyqN0FUt1sREWluNZsdnhMXRlTw8WXL3tzd1pUCS3OqHmGJM+WQnluAYRgeLkhERNoK52aHLquDvL27rSsFluYUHIvhE4CvyUZEVRZHCss9XZGIiLQB5VU21u7JAdzb8Xt7d1tXCizNyWx27trcWS36RUSkmXx38CjFFTaigq306xjqPF7T3XZEhxFe2d3WlQJLc3PZtflgjibeiohI06vZnXl8z2jM5uPzVLy9u60rBZbm5jLCoqXNIiLSHL7ZVXs5s2t32zFxYzxSV0MosDS3iJqVQlkc1EohERFpYim5xew/UoyP2cTo7se72LaE7rauFFiam7MXSyap6sUiIiJNrOZ20JDO4YT4+zqP19wO8vbVQTUUWJpbdWCJN2WTllPo4WJERKS1W7Gr9maHZVVlbMzw/u62rhRYmltIJwyLFT+TjcDyTPJLKj1dkYiItFIlFVVs2J8LuM9faSndbV0psDQ3sxlTeGegZmmzbguJiEjT2LAvl4oqO53CAujWPth5vKV0t3WlwOIJLvNYNPFWRESaiutmhzXBpCV1t3WlwOIJrrs2a+KtiIg0AcMwWFkzf8Wlu+2OvB0tprutKwUWT4g8vmuzRlhERKQp7M4qIv1YKVYfMyO6HF+2XLM6qCV0t3WlwOIJbkubFVhERKTx1Wx2OKJrJAF+FufxmttBLWV1UA0FFk+oDiwJpixScwo8XIyIiLRGrvNXatR0tzVhahHdbV0psHhCSByG2RerqQpzUSYlFVWerkhERFqR/NJKtqQcBWB8j+OBpaV1t3WlwOIJFh9M4YkAJJq1p5CIiDSuNXuOYLMbdI0OIiEy0Hm8Zv7K2LixnirtjCmweIrLPJYUzWMREZFGtGKnY3WQ6+2gltjd1pUCi6e4bIKYoqXNIiLSSOx2g1W7HfNXXNvxt8Tutq4UWDwl4vjSZo2wiIhIY/kpPZ+cogqCrT4M6RzhPN4Su9u6UmDxFN0SEhGRJlCzOmh0tyj8fBx/5l2727bE20GgwOI5EUlATbdb7dosIiKNY+Wu2suZXbvbnhd7nqdKOysKLJ4SloBhshBgqqDyWAYVVXZPVyQiIi3ckcJyfjiUD8D4nsfb8bfU7rauFFg8xeILYQkAJJBF+rFSDxckIiIt3ardjtVB/TqF0D7E33l85aGVQMu9HQQKLB5lqtkE0ZyplUIiInLWatrxu64Oyi7JZnvu9hbZ3daVAosnRTqWNmulkIiInK1Km53Vu2t2Z24d3W1dKbB4klYKiYhII/k+5SiFZVVEBPlxblyY83jN/JVx8eM8VFnjUGDxJLdeLLolJCIiZ+6b6ttB43pEYzE7+qy4drcdF9cGA8v8+fNJSkrC39+fwYMHs2bNmnrPzcjI4KabbqJnz56YzWZmzpxZ65yFCxdiMplqPcrKys6kvJajZg6LSXNYRETk7KysbsfvujpoU8amFt3d1lWDA8uSJUuYOXMmDz74IFu3bmXMmDFceumlpKam1nl+eXk50dHRPPjgg5x77rn1XjckJISMjAy3h7+/f73ntwphCRgmM0GmckqOZmC3G56uSEREWqD0Y6XsyirEbHKMsNSoWR3UUrvbumpwYHn++eeZOnUqd9xxB71792bevHnEx8fz6quv1nl+586deeGFF7j11lsJDQ2t97omk4nY2Fi3R6vnY4XQOAA62g6TWdDKR5RERKRJrKjubjsoIZywQD/A0d12dZpjwm1LXs5co0GBpaKigi1btjBx4kS34xMnTmT9+vVnVUhRURGJiYnExcVxxRVXsHXr1rO6Xkthqt4EsbM5i4O6LSQiImegJrBMOLG7bWnL7m7rqkGBJScnB5vNRkxMjNvxmJgYMjMzz7iIXr16sXDhQj777DMWL16Mv78/o0aNYs+ePfW+pry8nIKCArdHi+Scx5JFqlYKiYhIA5VV2li3Lwdw779SszpoZMeRLba7raszmnR74n0wwzDO6t7Y8OHDueWWWzj33HMZM2YMH3zwAT169OCll16q9zVz5swhNDTU+YiPjz/j7+9R1YElyZRJSp4Ci4iINMzG/bmUVdqJDfGnd4d2zuOu81dagwYFlqioKCwWS63RlOzs7FqjLmdVlNnMeeedd9IRltmzZ5Ofn+98pKWlNdr3b1ZaKSQiImdh5a6aZnHRzsGD1tLd1lWDAoufnx+DBw8mOTnZ7XhycjIjR45stKIMw2Dbtm106NCh3nOsVishISFujxbJ5ZZQSo4Ci4iInD7DMPimev7KeNfbQYcct4NaendbVz4NfcGsWbOYMmUKQ4YMYcSIESxYsIDU1FSmTZsGOEY+0tPTeeedd5yv2bZtG+CYWHvkyBG2bduGn58fffr0AeCxxx5j+PDhdO/enYKCAl588UW2bdvGK6+80ghv0cuFd8bARIiplIK8rLO+vSYiIm3H/pxiUvNK8LWYGN3teDCpWR3U0rvbumpwYJk8eTK5ubk8/vjjZGRk0K9fP5YtW0ZiYiLgaBR3Yk+WgQMHOj/fsmULixYtIjExkYMHDwJw7Ngx7rrrLjIzMwkNDWXgwIGsXr2aoUOHnsVbayF8/SGkExQcIrriEHnFFUQGt/zJUSIi0vRqVgcNS4okyOr4k96autu6anBgAZg+fTrTp0+v87mFCxfWOmYYJ2+INnfuXObOnXsmpbQKpsguUHCIRFMWB3NLFFhEROS0OHdndlnOXNPdtkNQhxbf3daV9hLyBjV7CpkzSc3TPBYRETm1ovIqvj2QB8CEnrW7246NG9uqphgosHgDl00QD+ZoabOIiJza2j05VNoMOkcG0iU6GGh93W1dKbB4A5elzanqxSIiIqdhRR2rg1pbd1tXCizewHWERb1YRETkFAzDcM5fOb9X6+1u60qBxRuEJwEQZirmWE6Wh4sRERFvtz2jgOzCcgJ8LQxNinAeb23dbV0psHgDv0Ds7RxN8kJK0ygsq/RwQSIi4s1qbgeN6haFv68FaJ3dbV0psHgJc82uzaZMUrQJooiInMQKl3b8NVpjd1tXCizeIsJxW6izKUsTb0VEpF5HiyvYmnoUqHt35tbU3daVAou3qFkpZNbEWxERqd/qPUewG9Arth0dwwIAKK0qbZXdbV0psHgL50qhTFJ1S0hEROpR12aH32Z8S7mtvNV1t3WlwOItXHZt1giLiIjUxWY3WLXbMX/FdTmz6+qg1tTd1pUCi7eoDiyRpkLyco54uBgREfFG29KOcqykkhB/HwYlhAGtu7utKwUWb2ENxh7kSMvWohTKKm0eLkhERLzNip2Of9CO7RGNj8XxJ3x73nZnd9shsUM8WV6TUmDxIqbI6qXNZHLoqOaxiIiIu5r5K3WtDmqN3W1dKbB4EZPrPBZtgigiIi4y88vYnlGAyQTjetbuv9JaVwfVUGDxJjW9WMxZpKgXi4iIuFhZvXfQOXFhRAU7RlJcu9uOjRvryfKanAKLN3HZtTlFK4VERMSFc7ND19tBNd1to/sTGRDpkbqaiwKLN3HZtVnt+UVEpEZ5lY21e3KAE9rxV89fGR833hNlNSsFFm9SHViiTfkcyc3xcDEiIuItvjt4lOIKG1HBVvp1DAXcu9u29ttBoMDiXfxDsVUP6fkcO0CVze7hgkRExBsc724bjdnsaAzXFrrbulJg8TLm6qXNcUYmGfllHq5GRES8gXP+ShvrbutKgcXLmFzmsahFv4iIpOQWs/9IMT5mE6O7RwFtp7utKwUWb+PSi0UTb0VEZEX17aAhncMJ8fcF3Lvbnhd7nifLazYKLN6mZoTFrKXNIiICK3Y52vHX193Wz+LnkbqamwKLt4nUCIuIiDiUVFSxYX8uABNc56+krQRaf3dbVwos3qZ6hCXWdJTMnDwPFyMiIp60YV8uFVV2OoUF0L19MABZxVnsyNvRJrrbulJg8TYB4dis4Y7Pj+7HMAzP1iMiIh7j3OywV7RzJdDqdMdk27bQ3daVAosXMlXfFupgyyC7sNzD1YiIiCcYhsHK6vkrrsuZ21J3W1cKLF7IHKkW/SIibd3urCLSj5Vi9TEzootjObNrd9tx8W1n/goosHgnl00Q1YtFRKRt+nRbOgAjukYS4GcBYFPGJmd32+5h3T1ZXrNTYPFGEY5ut51NWaRqhEVEpM3ZfDCP11fvB+BXg+Kcx11XB7WF7rauFFi8Uc0IizmLlDwFFhGRtiSvuII/LNqKzW5wzYCOXHlOB6C6u+2httXd1pUCizeqDiydTLlkaGmziEibYbcb/OmDbWQWlNElOognru3vHEnZnredI6VHCPQJbDPdbV0psHijwAhsfiEA2HMPerYWERFpNgvW7GfFriNYfcy8ctMggqw+zufaYndbVwos3shkgogkACIrDnGspMLDBYmISFP77mAez365C4BHr+pL7w4hbs8756+0sdVBNRRYvJQl0jHxVi36RURav6PFFfxhsWPeytUDOnLDefFuz7t2tx3TaYyHqvQsBRZvVbMJopY2i4i0ana7wZ8+/IGM/DK6RLnPW6nRVrvbulJg8VYuIyxa2iwi0nq9sWY/3+zMxs/HzMs3DSLYZd5Kjbba3daVAou3qh5hSTJnclCBRUSkVdqSksczNfNWruxLn44htc5py91tXSmweKvqwNKRXA7nHvVwMSIi0tiOuvRbufLcjtw4NL7O82q623YM6tjmutu6UmDxVkHR2HyDMJsMKrW0WUSkVTEMg//78AcO55eRFBXEk9f2q7dzrevqoLbW3daVAou3Mpkg3DHKElKSRklFlYcLcqiy2fnn2gOMe3YF/1iz39PliIi0SP9Yc4CvnfNWBtLO37fO8+yG3dnddlxc270dBGcYWObPn09SUhL+/v4MHjyYNWvW1HtuRkYGN910Ez179sRsNjNz5sw6z1u6dCl9+vTBarXSp08fPv744zMprVWxRB3ftTnVC1r0b9qfy+UvruWvn28nJbeEv/13B59sTfd0WSIiLcqWlKM8/b+dAPzlij707Rha77k78na06e62rhocWJYsWcLMmTN58MEH2bp1K2PGjOHSSy8lNTW1zvPLy8uJjo7mwQcf5Nxzz63znA0bNjB58mSmTJnCDz/8wJQpU5g0aRKbNm1qaHmtS0TNSqFMDuZ4LrBkF5Qx8/2tTF6wkV1ZhYQF+nJh7xgA7vv3j2w+qO0DREROx7GSCv64eCtVdoMrzunAzcMSTnp+W+9u66rBgeX5559n6tSp3HHHHfTu3Zt58+YRHx/Pq6++Wuf5nTt35oUXXuDWW28lNLTuFDlv3jwuuugiZs+eTa9evZg9ezYXXHAB8+bNa2h5rUuE6whL8/diqbn9c/5zq/hk22FMJrhpWAIrf9ebBV3Wcmv3Sipsdu565ztS1CtGROSkauatpB8rpXNkIHN+VbvfyonaendbVw0KLBUVFWzZsoWJEye6HZ84cSLr168/4yI2bNhQ65oXX3zxSa9ZXl5OQUGB26PVcWse17wjLK63f4rKqzg3LpRPp4/kycRthL01CvPXj/BY3n2c36GCoyWV/HbhZvJLKpu1RhGRluSfaw/w1Y7j/Vbqm7dSQ91t3TUosOTk5GCz2YiJiXE7HhMTQ2Zm5hkXkZmZ2eBrzpkzh9DQUOcjPr7u5WAtWnVgiTMdIT0nv1m+ZXZBGfcu2eZ2+2fOr/rz8Q0dOeebW+Gzu6EsH8y+mIqzeN3yDN1CDPYfKWbav7ZQUWVvljpFRFqS71OP8tQXjnkrD1/Rh36d6p+3UuOr1K8AOCf6nDbb3dbVGU26PXEIyzCMs15q1dBrzp49m/z8fOcjLS3trL6/V2oXi83ij8VkUJ5zsEm/levtn4+3pjtv/6y4dzQ3Vn6M+bWRcGA1+ATARX+FuzdDUHt8c7bzceybtPMzsWF/Lg998hOGYTRprSIiLcmxEke/lSq7weXndOCWU8xbAdifv5+Xtr4EwMTEiac4u22o3f/3JKKiorBYLLVGPrKzs2uNkDREbGxsg69ptVqxWq1n/D1bBJMJIzwJcnYQUJRCRZUdP5/GX4m+aX8uf/n0F3ZlFQJwblwof72mH+dYUmDRJZDxg+PEpLFw5QvOkR9uXAwLL6dd6tcs6xXHuJ8u5oPvDtE5Kojp47s1ep0iIi2NY97Kj6QfKyUxMpCnTmPeSkllCfeuuJfiymKGxAzhpt43NVO13q1Bf/38/PwYPHgwycnJbseTk5MZOXLkGRcxYsSIWtdcvnz5WV2ztbBEOVYKJZDFoaONO4+l3ts/dw7inB1zYcEER1jxD4WrXoZbPzseVgDihsC1rwEQv/ttFg34BYBn/reLZT9lNGqtIiItkWPeShZ+FjOvnMa8FcMweHjdw+zP30/7gPY8O+5ZfMwNGltotRr8U5g1axZTpkxhyJAhjBgxggULFpCamsq0adMAx62a9PR03nnnHedrtm3bBkBRURFHjhxh27Zt+Pn50adPHwDuuecexo4dy9NPP83VV1/Np59+yldffcXatWsb4S22bKbqTRA7mzJJySuhS3TwWV+zymbn7Q0pzE3eTVF5FSYT3Dg0gf83sSfh2Zvg9ashr7opXJ9r4NJnoF09o119r4XcffDNXxm+8yn+2u9ZHv45lnuXbKNDqD8DE8LPul4RkZZoq9u8ld6nNW/lne3vsDxlOT5mH54b/xxRAVFNXWaL0eDAMnnyZHJzc3n88cfJyMigX79+LFu2jMTERMDRKO7EniwDBw50fr5lyxYWLVpEYmIiBw8eBGDkyJG8//77PPTQQzz88MN07dqVJUuWMGzYsLN4a62Ey0qhlJxi6Hl2l6v39k8kkPwn+L46aLbrAJc/B70uP/VFx/zJEVp+WMQtaY+yq8vz/Gt/IHe+8x0fTx9FfETg2RUtItLC5JdUcnfNvJX+HbhleOIpX7M5czNzt8wF4L7z7mNA+wFNXGXLYjJayQzJgoICQkNDyc/PJySk9m6XLdaB1fD2ley3x/LueR/xyJV9z+gy2QVlzPliJx9Xd6YNC/Tl/kt6MXlIPOad/4Fl/wdFWY6Th9wOFz7quBV0uqrK4d1rIWUd9tB4buZJNmRZ6BETzL9/P5KQUwyDioi0FoZhcNe7W0jenkVCRCCf/3H0KX8HZhVnMenzSeSV5XFFlyt4cvSTbWbfoNP9+629hLxd9QhLvOkIh3IKG/zyelf//Gk8N/byxfzBLfDBFEdYiewGv/0CrpjbsLAC4GOFyf+CiC6Y89N4O3Ae8cGwO6vI8a8Mm5Y7i0jb8Oa6gyRvPz5v5VRhpdJWyf+t+j/yyvLoEd6Dv4z4S5sJKw2hwOLt2nXEbvbD12SjtIFLmzftz+WKl9ybv30yfRRPXt2X8J2L4JVhsPNzMPvAmP+Daesg8SwmOgdGwE0fgH8ofhlb+E/C+wT4mlm9+wiPfPaLljuLSKu3Le0YT32xA4CHruhN/7hT/+Pv79/9nW1HttHOtx1zx88lwCegqctskTT12NuZzdjCOmPO241v/kFsdgOL+eTJO7uwjDnL6rn9k7cP3r4ZUqonNHccBFe9BLH9GqfeqO6OkZZ3ryVs/2d83j+eC7eO4r1NqSRFBXHHmC6nvoaISAvkmLfyPZU2g8v6xzLlNOatfL7/cxbtXATAnDFzSAg5dY+WtkojLC2AJcrR06STkUlmQVm959Xc/rng73Xc/hncAfO65+HVkY6w4hsIFz8Jd3zVeGGlRtJYuGIeAF23v8JbAx0rjp5YtoPk7VmN+71ERLyAYRj8v3//wKGjpSREBPLUdeec8rbOrrxdPLb+MQB+d87vtF/QKSiwtADmSMeoRJIps95NBr89kOe8/VPoevvn2v6EH/vZ0VPl68fBVg5dz4fpG2DEDDBbmqboQVNg1EwAxu16nNl9j2IY8MfFW/k5vXm2GRARaS4L1x9keQPmrRRUFHDvynsps5UxquMofn/u75up0pZLgaUlqJ54m2jKJOWETRCzCx3N3ya9voGdmS7N36aP4twYX/jyQfjHBZD1EwSEw7Wvwy0fQXjnpq/7gkeg95WYbBXcdfhhrkuqoLTSxtS3N5ORX9r0319EpBn8kHaMJ5c55q08ePmp563YDTsPrnmQtMI0OgZ15KkxT2Fpqn88tiKaw9ISOHuxZPFddWCpaf42L3k3hSc2fwvyg30r4D/3wLEUxzX6XQ+XPAXB0c1Xt9kM1y6A/MswHd7KM4FPsL/9Y2zNLmfqwu/4cNoIgqz6n6CItFz5pZXMqJ63cknfWG4dcep5K//46R+sPLQSP7Mfz094njD/sKYvtBXQCEtL4FzanE1qTsHJb/+YiuCT6fDuNY6wEhLnWLlz/T+bN6zU8AuEG9+HkE5YcvewOPRVYoLMbM8o4J73t2Kza+WQiLRMhmFw/79/5NDRUuIjAnj6+lPPW1mfvp6Xt74MwEPDH6Jv5Jn11mqLFFhagtA47GZfrKYqtu/aWfftn7hQ+HkpvDIUtr0HmGDoXTBjI/S42LP1t4uFm5aAbxD+aWv4b9dPsfqY+GpHNk/8d4dnaxMROUNvrz/I/37JxNdi4pWbBhEacPJ5K+lF6dy35j4MDK7rfh3Xdr+2mSptHRRYWgKzhaoQxzBjR3uG++qfoQmYCw/D4hvh37dD8RGI6gm3fwmXPQvWdh4uvlpsf7j+TTCZidq9mI8HbgXgzXUHeHfDQc/WJiLSQD8eOsYT1fNW/nxZb86JCzvp+eW2cmatnEV+eT59I/sye9jsZqiydVFgaSF8ox2bII6PKjx++yfAB759w9EAbvcXYPaFcQ/AtDWQ4IX7MPW8BCY+AUCfn55l/uDDADzy2S+s3JXtycpERE6b67yVi/vGcNvIzqd8zZxNc9ieu50waxhzx8/FarE2faGtjAJLC1Gza/NdfQ3OjQ+DI7vgrUsdewBVFELceY6gMmG2o02+txr+exgyFTC4dPdfuKdPMXYD7l60lZ2ZBZ6uTkTkpAzD4IGlP5KWV0pceADPXH/uKeetLN29lKV7lmI2mXlm7DN0CO7QTNW2LgosLUX1xFuO7IJVz8BroyFtI/gGwaXPOG4Bte/t2RpPh8nkqLfr+ZgqS5h55C9cmminqLyK29/aTHZh/Y3xREQ87Z0NKXzx8+nPW/k552ee2OQYWf7DwD8wouOI5iizVVJgaSkikhwf9ybDiifAVgHdLoIZm2DY75quAVxTsPjArxdCdC9MhRm8ZDxNn0gzh/PLuPPt7yitsHm6QhGRWn46lO9cKDD70t6O0e6TOFp2lFkrZ1Fpr2RC/ARu73d7M1TZeimwtBQRXY9/HhgJ1/0Tbv4QwuI9V9PZ8A91rBwKjMIn+yf+3f5NIgPM/HAon1kfbMOu5c4i4kUKyhzzVipsdib2ieG3ozqf9Hyb3cb9q+8noziDhHYJPDH6Ccwm/ck9G/rptRThnR3LlM+7A2Zshv7XO26vtGThneHGxWCxEnhgOZ/3/go/i5kvfs7kmS93ebo6ERHg+LyV1LwS4sIDePY05q28su0VNmRsIMAngLkT5tLOz0tWbLZgCiwthcnkWKZ8+XMQFOnpahpP/FC4Zj4AHbb/gyVDdgLw2qp9vP9tqicrExEB4N2NKSz7yTFv5eWbBhEaePJ5KytSV/DGT28A8OiIR+kR3qM5ymz1FFjE8/pfD+P/DMDAn/7Gc4PzAHjok59ZtzfHk5WJSBv3c3o+f/vcMW/lgUt7M+AU81ZSClL481rH77Nbet/CZV0ua+oS2wwFFvEO4+6DcyaDvYpf7f0zd/WupMpuMO1fW9ibXejp6kSkDXKdt3JRnxhuP8W8lZLKEmaumElRZRED2w9k1pBZzVNoG6HAIt7BZIKrXoL44ZjKC3jg6COcH2+isKyK3y7cTG5RuacrFJE2xDAMZi/9iZTcEjqFBfD3U8xbMQyDxzY8xt5je4kKiOK5cc/haz75rSNpGAUW8R4+VrjhPQjvjPnYQV73m0vXcF/S8kq5690tlFVqubOINI9/bUrlvz9l4GM28fJNA085b2XRzkUsO7AMi8nC38f9nehAD2w228opsIh3CYpy7C5tDcU3/Vs+jl9MiL+FLSlHue/fP2IYWu4sIk3r5/R8/vqf7QA8cGkvBiaEn/T8rdlb+fvmvwPwpyF/YnDM4CavsS1SYBHvE90TJr0NJgshuz/i83M24GM28dkPh5n71R5PVycirVhReZVz3sqFvWOYOjrppOfnlObwp5V/osqo4pLOl3BL71uaqdK2R4FFvFPXCXDF8wAk/DiPd4elAfDi13v46PtDnqxMRFqxp7/YeXzeyq/POem8lUp7Jf+36v84UnqEbmHdeGzkY6fszyJnToFFvNfg22DE3QCM+PFh/jq4BID7l/7Ipv25HixMRFqjjftzeXdjCgDPXH8OYYF+Jz1/3pZ5bMnaQpBvEHPHzyXQN7A5ymyzFFjEu130OPS8DGzl3HLgAab0Mqi0GfzuX1s4kFPs0dLKq2ykHyvlh7RjfL0jiyWbU1mz54jm2Yi0QKUVNu5f+iMANw5NYFS3qJOe/7+D/+Od7e8A8MSoJ+gc2rmpS2zzfDxdgMhJmS3wqzfgrUsxZf7IY0WPs7fT39iQXsntCzfz8fSRp/xXUEOUVdrIKSrnSGE5OUUV5BSVk1NY7vhYVMGR6q+PFJVTWFZV5zX6dAjhjxd0Z2KfGMxmDQ+LtAR/X76LlNwSOoT68+fLep303L1H9/KXdX8BYGq/qVyQeEFzlNjmmYxW8s/BgoICQkNDyc/PJyQkxNPlSGMrOAxvnA+FGZQnjueizLtJza9gWFIE704dhp9P/YOFpRXVIcQZPiqqA4nro4KcwnIKy+sOIfXxtZiICrYSFWwlPMiPLQfzKK7ebbpXbDvuPr8bl/brgEXBRcRrbUk5yvWvrccw4K3fnseEnu3rPbeooogb/3sjBwsOMqzDMF678DV8zPq3/9k43b/fCizSchzeBm9dCpUlHO1zK2N+uZyichuXn9OBEV0ij4ePwgq3gFITIE6Xn8VMVLAf0e2szjAS1c7v+OfBVqKrvw4N8HWbZHe0uIK31h3grXUHneGnW/tg/nB+N644p6OCi4iXKau0cfmLa9h3pJjrBsXx3KRz6z3XMAzuXXkvX6d+TUxgDB9c+QER/hHNWG3rpMAirdPO/8L7NwMGewc9xMUb+2Kzn/p/wlYfc3XwsBId7FcdOlwCSbAfUdVfh/j7nN5Mf8OAyhIoyYWSPCg9CqHxENWN/NJKFq47yD/X7qeg+tZRl6ggZkzoxtUDOuJj0fQxEW/wzP92Mn/lPqLbWUm+d+xJbzG/+fObzN0yF1+zL29f8jb9o/s3Y6WtlwKLtF7rX4LlDwEmNgx7mbkpXQkN9HWEkOrgEV0dTmrCSLD1FCHEboeyY9XBI8/9Y0muy7Gj7s/Z6tgyoNcVMO5+6HAOhWWVvLMhhTfW7OdYSSUAiZGBzBjfjWsHdcJXwUXEY346lM8189dhsxu8PmUwF/eNrffcTRmbuCv5LuyGnYeHP8yknpOasdLWTYFFWi/DgP/cA9+/Db5BMPVLiHX5l05VuUuoyD0hhBytO4yUHgPO8P8VzL4QGAH+oZCz5/h1el3h2NSxw7kUlVfxr40pvLF6P7nFFQDEhQcwfXw3rh8cd9I5OCLS+Cqq7Fz18lp2ZhZy5bkdeenGgfWem1mcyeTPJ5NXlsc13a7h8ZGPq99KI1JgkdbNVgn/ug4OrILASAjpdDyMVJ7Fcme/dhAYDgERjhDi9jGy+vNw9+f8gh2bNwJk74TVz8DPH+EMLj0vdwSXjgMoqahi0aZUXlu1n5zqDR07hvrz+/Fd+fWQePx9LWf3cxGR0zLvq93M+2oPEUF+JN87lshga53nVdgq+O3/fsuPOT/SO6I371z6Dv4+/s1cbeumwCKtX+lR+OdEyNld+zmTuZ7QUUcYCYx0fB4QDj6NtET6yC5Y/Sz89G+cwaXHpTD+fug4kLJKG4u/TeW1VfvIKnAEl5gQK9PGdeXGoQkKLiJNaEdGAVe+tJYqu8FLNw7kynM71nvuXzf8lQ92f0CIXwhLrlhCXLu4Zqy0bVBgkbah6IhjlMUa4j76YQ0FsxfcZjmy2xFcfv43GHbHsR6XOOa4dBpEWaWND79LY/7KfWTklwEQFWxl2rgu3DQsgUA/LZcUaUxVNjvXzl/PT+n5TOwTw+tTBtd7e+eTvZ/w8LqHMWHilQteYUzcmGautm1QYBHxJjl7qkdcPjweXLpf7Bhx6TSY8iobS7ek88qKvaQfKwUgMsiPO8d2YcrwRIKsCi4ijWH+yr08879dhAb4knzvWNqH1H17Z0fuDqZ8MYVyWznTB0zn9+f+vpkrbTsUWES8Uc5eWPN3+HGJS3CZCOMegLjBVNrsfLzVEVxSch17J4UH+nLHmC7cOiKRdv6+HixepGXbm13IZS+upaLKznO/PpfrBtd9eye/PJ/Jn08mvSidMZ3G8PIFL2M2ecGIbSulwCLizXL3weqa4FLd2K7bhY7gEn8eVTY7n247zMsr9jr3TArx9+H20Un8dlQSoQEKLiINYbMb/Pq19XyfeozxPaN567bz6rwVZDfs3P313axJX0NccBzvX/E+odZQD1TcdiiwiLQEuftgzXPww/vHg0vXC2D8AxA/FJvd4PMfD/PSN3vZm10EQDurD78d1ZnbRyc16j5KIq3ZP9ce4K+fbyfY6sPye8fSMSygzvPe+vktnt/yPFaLlfcue4+eET2budK2R4FFpCXJ2w+rn4MfFrsEl/MdIy4Jw7DbDb74OZMXv97DrqxCAIL8LNw6sjN3jE6qd0mmiMDBnGIueWE1ZZV2nry2PzcNS6jzvO2527l52c1U2at4dMSjXNfjumautG1SYBFpifIOVI+4LAZ79UaMXSY4RlwShmO3GyzfnsWLX+9he0YBAAG+FqaMSOTOMV2IbqfgIuLKbje48Y2NbDqQx8iukbx3x7A6bwWVVJYw+fPJHCw4yIUJF/L8+OfVHK6ZKLCItGRHDzqCy7ZFx4NL0jhHcEkciWEYfL0jmxe/2cOPh/IB8Pc1c9PQRH43rgsx9ax8EGlr3t2YwsOf/EyAr4Xl944lPiKwzvMe3/A4H+7+kPaB7Vl65VLC/MOat9A27HT/fp/RtOf58+eTlJSEv78/gwcPZs2aNSc9f9WqVQwePBh/f3+6dOnCa6+95vb8woULMZlMtR5lZWVnUp5IyxfeGa56Cf7wPQz6DZh9HP1m3roU3r4SU8p6LuwTw6czRvHWb89jQHwYZZV23lx3gDHPrOCRT392Lo8WaasOHS3hqWU7ALj/kp71hpWvU7/mw90fYsLEk6OfVFjxUg0OLEuWLGHmzJk8+OCDbN26lTFjxnDppZeSmppa5/kHDhzgsssuY8yYMWzdupU///nP/PGPf2Tp0qVu54WEhJCRkeH28PfXvxKljQtPhKtedASXwbc59i06sBoWXgYLr8CUso4JPdvz8fSRvDt1KEMSw6mosvP2hhTGPrOCGe99z5aUo55+FyLNzjAMZn/0E8UVNoYkhnPriM51npddks2j6x8F4La+tzGsw7DmK1IapMG3hIYNG8agQYN49dVXncd69+7NNddcw5w5c2qdf//99/PZZ5+xY8cO57Fp06bxww8/sGHDBsAxwjJz5kyOHTt2hm9Dt4SkjTiWCmvnwvfvgt2x+zOJox23ipLGYBgGG/bn8sqKvazbm+t82YD4MKaOTuKSfrHaIVrahA82p3Hf0h+x+pj54p4xdIkOrnWO3bDzu+TfsTFjI70jevPeZe/ha1HLgObWJLeEKioq2LJlCxMnTnQ7PnHiRNavX1/nazZs2FDr/IsvvpjvvvuOyspK57GioiISExOJi4vjiiuuYOvWrSetpby8nIKCAreHSKsXlgBXzIU/boUhUx0jLilr4e0r4K3LMB1cw8gukbx3x3D+N3MMk4Y4doLelnaMPyzeythnVvDaqn3kl1Se+nuJtFCZ+WX89b/bAZh1UY86wwrAu9vfZWPGRvwt/jw19imFFS/XoMCSk5ODzWYjJibG7XhMTAyZmZl1viYzM7PO86uqqsjJyQGgV69eLFy4kM8++4zFixfj7+/PqFGj2LNnT721zJkzh9DQUOcjPj6+IW9FpGULi4crnod7tsF5d4DFD1LWwdtXwluXwf5V9IppxzPXn8v6B85n5oXdiQr2IyO/jKe+2MnwOV/z8Cc/s/9IkaffiUijMgyDBz/+icKyKs6ND+OOMV3qPG9n3k5e+P4FAO4beh9dQus+T7zHGY0Nn7jUyzCMky7/qut81+PDhw/nlltu4dxzz2XMmDF88MEH9OjRg5deeqnea86ePZv8/HznIy0t7UzeikjLFhoHlz8Hf9wG593pCC6p6+Gdq+AfF8DPS4kKsDDzwh6svf98nr3+HHrFtqO00sa7G1M4/7lVTF24mXV7c2glCwaljfvsh8N8vTMbX4uJZ68/B4u59t+m0qpS7l99P5X2SibET+D67td7oFJpqAbtqBYVFYXFYqk1mpKdnV1rFKVGbGxsnef7+PgQGRlZ52vMZjPnnXfeSUdYrFYrVqt6TogAENoJLv87jJkFa+fBloWQvgX+fTuExMGwu/Af9Bt+PSSe6wfHsWFfLm+uO8DXO7Odj16x7bh9dBJXndsRf1+Lp9+RSIMdKSznkc9+AeCP53enR0y7Os977rvn2J+/n+iAaB4b+Zj6rbQQDRph8fPzY/DgwSQnJ7sdT05OZuTIkXW+ZsSIEbXOX758OUOGDMHXt+77hYZhsG3bNjp06NCQ8kQkpCNc9gzc+7OjS25gFBQcguS/wPN9YNl9mI4eYGS3KP7xm/P45k/juXVEIgG+FnZmFnLfv39k9NPfMDd5N0cKyz39bkQa5JHPfuZYSSV9OoQwbXzXOs9ZmbaSJbuWAPC30X8j3D+8GSuUs9HgVUJLlixhypQpvPbaa4wYMYIFCxbwxhtv8Msvv5CYmMjs2bNJT0/nnXfeARzLmvv168fvfvc77rzzTjZs2MC0adNYvHgx113naHv82GOPMXz4cLp3705BQQEvvvgi7777LuvWrWPo0KGnVZdWCYnUobIMfvoQNs6H7O3VB03Q8zIYMR0SR4HJRH5JJe9vTuXt9Qc5nO/of+RnMXP1gI7cPjqJ3h30/1Pi3Zb9lMH0977Hx2zikxmj6Nep9oaFR0qOcN1n13G0/Ci/6fMb/u+8//NApXKi0/373aBbQgCTJ08mNzeXxx9/nIyMDPr168eyZctITEwEICMjw60nS1JSEsuWLePee+/llVdeoWPHjrz44ovOsAJw7Ngx7rrrLjIzMwkNDWXgwIGsXr36tMOKiNTD1x8GTYGBt8D+lbDhFdibDLv+63jEngMj7ia077X8blxXbh+dxJe/ZPLPtQfYmnqMD7cc4sMthxjZNZKpo5OY0LM95jrmBIh40tHiCv7y6c8A/H581zrDit2w89C6hzhafpReEb3446A/NneZcpbUml+krTmyCza+6tghuqq6G25wLAy9E4bcDoERAHyfepR/rj3A/37OxGZ3/JpIigrit6M6c92gOIKsDf73jkiTuHfJNj7emk739sF8/sfRWH1qz8F6d/u7PLP5GawWKx9c8QFdwrQqyFtoLyERObmSPPjuTfj2DSiqnhjvEwADboTh0yGqOwDpx0p5Z/1BFn2bSmGZY1+jEH8fbhyWwG9GdKZjWICn3oEIX+/IYurb32E2wdLfj2RgQu05KbvydnHjf2+k0l7JQ8MeYnKvyR6oVOqjwCIip6eqAn75yHG7KPPH48e7T3QEly7jwWSiuLyKf285xFvrDnAwtwQAi9nEpf1imTo6qc4/FCJNKb+0kolzV5FVUM5dY7vw58t61zqnrKqMGz6/gX35+xgfN54Xz39Rq4K8jAKLiDSMYTiaz22YD7uWAdW/Gtr3dUzQ7Xc9+Ppjtxt8szObf649wIb9x9v/D0yobv/fNxYftf+XZnD/v39kyXdpJEUF8cU9Y+pcjv/kpidZvHMxUQFRLL1qKRH+ER6oVE5GgUVEzlzuPtj0Gmx9DyqLHceCoh1ddYdMheBoALYfLuDNdQf4bNthKmx2ADqFBfCbkYlMPi+B0AC1OpemsWbPEab881tMJvjgdyM4r3PtILL60GpmfD0DgNcufI1RnUY1d5lyGhRYROTslR6FLW/DtwugIN1xzGKFc34Nw2dATB8AsgvL+NfGVN7bmEJucQUAgX4Wfj04jttGJZEUFeSpdyCtUFF5FRfPXU36sVJuG9mZR6/qW+ucnNIcrvvsOvLK8ril9y3cP/R+D1Qqp0OBRUQaj60Stn/q6OeSvuX48S4TYMQM6HoBmM2UVdr4bNth3lx3gJ2Zhc7TzokLZWKfGC7qE0uPmGDNIZCz8pdPf+adDSnEhQfw5cyxtVasGYbB9K+nszZ9LT3Ce7Do8kVYLeqM7q0UWESk8RkGpH0LG1+BHf8Bw3EbiKieMPz3cO4N4BuAYRis35fLP9ceYMWubFx/yyRGBnJR7xgm9o1lcGJ4nXu9iNRn4/5cbliwEYD37hjGqG5Rtc55b8d7PPXtU1gtVt6//H26hXdr7jKlARRYRKRpHT0ImxbA9+9ARfVoSkCEo5fL0DuhXSzguF309Y5skrdnsXZvDhVVduclIoL8uKBXey7qE8OY7tEE+GkPI6lfaYWNS15YTUpuCTcOjWfOr86pdc7uo7u58fMbqbBX8Odhf+bGXjd6oFJpCAUWEWkeZQWw9V+w6VU4Vt3l2uwL/a93LIvucPyPSnF5Fat3H2H59iy+2ZlNfmml8zl/XzNjukczsU8MF/SOISLIr7nfiXi5v32+nX+sPUCHUH++vHcsIf7uk7rLbeXc8PkN7D22l7FxY3n5/Jd1+7EFUGARkeZlt8HOzx3LotM2Hj+eONoxSbfXFRB0fPi+0mZn84E8lm/PInl7FunHSp3PmU0wpHNE9byXGBIjNWm3rfs+9SjXvboew4C3bjuPCb3a1zrnqW+f4r0d7xHhH8FHV31EZECkByqVhlJgERHPSd/iCC6/fAyGzXHMZHZsttjnakd4CTm+G7thGGzPKCB5exbLf8lie0aB2+V6xrRjYl9HeOnfKVT/am5jyiptXPHSWvZmF/GrQZ14ftKAWuesObSG6V9PB2D+BfMZEzemmauUM6XAIiKel3/IsWfRjs8g4wf35+KHQe+roPeVEJ7o9lRaXglf7XCMvGw6kOfcywigQ6g/F/aOYWLfGIYlReLnoyZ1rd0z/9vJ/JX7iAq28tWssYQFut8uzC3N5Vef/Yq8sjxu7n0zDwx9wEOVyplQYBER73L0oGNl0fbP4NC37s91GAB9roLeV0OU+4qOYyUVrNiVzfJfsli1+wglFTbnc+38fZjQ0zFpd3zPaNr5q1Fda/PToXyumb8Om93gtVsGc0m/WLfnDcNgxtczWJO+hm5h3Xj/ive1hLmFUWAREe9VcBh2fO4YeUlZd3x5NED7Po6Rlz5XOT53uf1TVmlj/b4ckrdnkbw9m5yicudzvhYTI7pGOee9xIT4N+c7kiZQUWXnqpfXsjOzkMvP6cArNw2qdc7inYt5ctOT+Jn9WHzFYnqE9/BApXI2FFhEpGUoOgK7/usYeTmwCuxVx5+L6Fo98nIVdBzoFl7sdoOtacdYvj2T5F+y2J9T7HbZc+PDmNgnhol9YujWXs3qWqIXvtrD3K92ExHkR/K9Y4kMdh852Xt0Lzf89wbKbeU8MPQBbu59s4cqlbOhwCIiLU/pUdj1P8fIy96vwXZ8BIXQBMd8lz5XQdxQMLvPXdmbXeSYtLs9k62px9ye6xwZyMS+sVzUJ4ZBCWpW1xLszCzgypfWUmkzePHGgVx1bke358tt5dz035vYfXQ3ozqN4tULXlUobaEUWESkZSsvhN1fOsLLnmSoLDn+XHAs9L7CMfKSOAos7q3ZswvK+GpHNsnbM1m3N9e5MSNAZJCfc9LuqG5Rde7w21YUlFWyJ6uIjmH+xIb4e80f/CqbnWvnr+en9Hwu6hPDgimDa9X29LdP868d/yLCP4KlVy0lKqB2x1tpGRRYRKT1qCiBfV87bhvt/h+Uuyx7DoyEnpc5lksnjQMf9xUkRTXN6n7J5Jud2RSUHb/lFOhnYVyPaCb2jeH8njGEBrbuSbt2u2P5+KrdR1i1+wjfpxylqnoFVqCfhaSoILpEB9MlKogu0UF0jQ4mKSqo1l49Te3Vlft4+n87CfH3IXnWuFrzkdalr2PaV9MAeOWCVxgbN7ZZ65PGpcAiIq1TVTnsXwU7PoWdy6A07/hz1lDoeYlj5KXbBeAb4PbSSpudbw/ksfyXTJZvzyIjv8z5nMVsYniXCCb2cdw66hjm/tqWKq+4gjV7HAFl9e4ct4nKANHtrOQVV7gtHT9RbIg/XaIdIaZLVLAzzHQMC2j022t7s4u47MU1VFTZefb6c/j1kHj391OWx3WfXUdOaQ439LyBB4c/2KjfX5qfAouItH62KkhZ6xh52fk5FGUdf843CLpf5Jjz0v1isAa7vdQwDH5OL2D59kyW/5LFrqxCt+f7d6reYbpvDD1j2nnN7ZJTsdkNtqUdc46i/HjomNvmk4F+FkZ2jWJcz2jG94gmPiKQSpud1LwS9h8pZv+RIsfHHMfH3OKKer+Xn4+ZpMjjozGOUOP4eGLb/NOt/devref71GOM6xHNwt+e5/ZzNwyDP37zR1YeWknX0K68f8X7+PtoNVhLp8AiIm2L3ebYSXrHZ44AU3Do+HMWq2PEpe+1ji67foG1Xp6SW+zstLs5Jc/tj3xCRKBjxZGX7jCdXVDGyuqAsnZPjtseTQC9Ytsxrmc043pEMyQxokHN9o6VVLCvJsjkHA80KbklbnODThQVbK0OMsdHZbpEBxMfHoCPpe7v/+baAzz++XaCrT58ee9YOp0wyrVk5xL+tulv+Jp9WXz5YnpG9Dzt9yHeS4FFRNouw4DD3zuCy47PIG//8eesodD/Ohg4pdZS6Ro5ReV8syOb5dszWb3HfYfpyCA/Lujdnol9Yhnd3TOTdiuq7GxJOeocRdlxwlYGIf4+jOnhCCjjekQ3SU8am93g0FHHqMy+E8JMdmF5va/ztZhIiAh0jsR0rQ4zVh8Lv359PWWVdp64th83D3Pvfrz/2H4mfT6Jcls59513H1P6TGn09ySeocAiIgKO8JL1C2z/FH58//iO0gAx/WDgLXDOZAiMqPPlxeVVrNlzhOW/ZPH1CTtMB/g6Ju1e1CeGC3q3r9UyvjGl5ZU4A8r6vTkUu3T8NZngnLgwZ0A5Ny603lGM5lBYVsmBnGLnLaZ9OcXsyy7iYG4xZZX1j8oAjOgSyXt3DMPsMopVYavgpv/exK6juxjZcSSvXvgqZpO2ZGgtFFhERE5kt8PB1fD9u45tAmr6vFj8oNfljvDSZQKY6x41qbTZ2Xwwj+W/1N5h2mI2MbRzhHOTxrjw2redGqKs0sbG/bnOkLL/iHtjvKhgP8ZWB5Qx3aOJCGq6sNRY7HaDw/mlx+fKuISaw/llhAX68tmM0SREuv/s/r7577y9/W3CreEsvWop0YHRHnoHbZhhOPokWUNqtRE4WwosIiInU5IHPy+F79+BzB+PHw+Jg4E3w4CbILxzvS83DINfDhewfHsWy3/JZGem+6Tdvh1DmNgnlol9Y+gVe+pJu4ZhsD+nmFW7HAFl4/5cyl1uRVnMJgYnhDvnovTpEOI2CtHSlVRUYTaZat1iW394Pb9L/h0AL054kQkJEzxRXutmq4TCTCjMgIJ0KMiAwsPVHzMcW2kUZkJVKUzfCO17N+q3V2ARETldGT84Rl1++gDK8o8fTxoHg251TNT1Pfk8kNTcEsc2Aduz2HwwD9dVwvERAY7w0ieGwYnhzts1ReVVrN+b4xxFOXS01O2aHUP9nQFlZLeoM1p505IdLTvKdZ9dx5HSI0zuOZmHhj/k6ZJaFsNw/O/ZGToyTggj1R+LjwCnGQVu/RS6jG/UMhVYREQaqrLMsTz6+3cc+xrV8A+DcyY5bhl1OPeUl8krruDrHVks357F6t1H3EZKwgN9Gd+zPRn5pWxJOUql7fivYD+LmWFdIpxzUdryHkiGYXDPintYkbaCLqFdeP+K9wnwaR29cRqFrQqKMusOIK4BxbVD9MmYfaFdBwjpUP2x4wkfq4/7Nv5/AwUWEZGzcTQFtr0HW99zXyIde45j1KX/9RAQfsrLlFRUsWZPTvWk3SyOlbgvOU6KCnIGlGFdIgj0a/qusoZhsD9/P6sPrWZn3k46Bneka1hXuoV1Iyk0CavFeuqLNLEPd3/I4xsex8fsw+LLF9MropenS2palWWO7SjKC6ofhVBWAGXHXEZGXIJIUTanPSriHwrtOjpCR0jH45+7fgyMrLU/V3NRYBERaQx2G+xfCVvfhZ3/BVt1IzWL1bEZ46Ap0Hnsaf2yr7LZ+S7lKGv35BDdzsr4ntEkRgY1bf3VSqtK2Zy5mdWHVrPm0BoOFx+u8zyzyUx8u3i6hnZ1hpiuYV1JCk3Cz9I8E3v35+9n8n8mU2Yr4/+G/B+/6fubZvm+Z8RW6RI0qkOGa/Bwfn3iOQXux2z1N+irl9nHsa9WvaMi1R/r6DvkTRRYREQaW0ke/LjEMd8l+5fjx8MSYMAtjom6YfH1v76ZHSo8xJr0Naw+tJrNmZspd9n92s/sx3mx5zGg/QCySrLYd2wfe4/tpbCisM5rWUwW4tvFOwNMzcfOIZ3xtTTe3JpKWyU3L7uZHXk7GN5hOK9f9HrzLGGuLHNMOD2WCvmHHCti3EJGvvvXNcGjqvTU124Iv3ZgrX74hzhW5bSLdbkt4zIqEhTtsVGRxqTAIiLSVAwDDm91jLr89G+XzRhN0PV8x1yXXpeDT/PeWqm0VbI1e6tjFCV9Dfvz97s9HxsUy9hOYxkTN4ahsUMJ9HX/l7dhGBwpPcLeY3vZd2yfM8TsO7aPosqiOr+nxWQhISTBGWC6hnWlW2g3EkMT8TU3PMg8/93zvPXLW4RZw1h61VLaB7Zv8DVqqVmSm5/mCCPH0qo/T6v+/BAUZ5/d9/ANPB40rCHugaPWsZqvTzjmF1zvkvrWTIFFRKQ5VJQ4erpsfRcOrjl+PCDC0ZBu4C0Q26/Jvv2RkiOsTV/LmvQ1rD+8nuLK4/1aLCYLA9oPYEynMYyNG0u3sG5nNInXMAyyS7KPB5j840HG9fu58jH5kBiS6DYa0y2sG/Eh8fUGmY0ZG7lz+Z0AzJswjwsSLji9Am1Vjnkd+YdOCCLVYST/EFTUHbjc+AY5RshCOjlGL04VMFyDSCOOMrU1CiwiIs0tb79jku62RY4VGzU6DnRsBdD/escEyLNgs9v4Ofdn1hxy3OrZkbfD7fkI/whGdxrNmLgxjOgwglDr2X2/kzEMg6ySLGd4cR2ZKamqe3WKj9mHziGda91aCvELYdJ/JpFdms31Pa7nkRGPHH9RRbHLyEiqy+fVAaXgMBi2Or+fm6BoCI13hJLQmkfc8a8DwuvcqkGalgKLiIin2G2w7xvH8uhdX4C9emWQjz/0udoRXmL6gsnseJgt1Z9bXL4+/oczvzyf9YfXs+bQGtamr+Vo+VG3b9c3si9j48YyptMY+kb1bf629XY72KscocFehWGrJKPoMHvz97E//wB7Cw6yryCVfUVplNrKTnqpzr5hLAkbSmBBpst8krxT12D2hdBOx4NIWHUYcQaTTk2yJFfOngKLiIg3KM6BH9533DI6svO0XmIAu319WRMUyJqAALZZfbG7BJhgu8HIChtjKuyMrjCIoibomOsIPmb3x4nhyGRyBCyXwOH82vk48fgJz5/m8lo7kOFjYZ+vL3v9fJ0fD/j6Umo2Y7XbeScjiz4VlbVfbA09IYTEuY+UBMe0igmobZECSyO5b/V97Dm6h4R2CSSEJBDfLp74dvEkhCQQGxiLpQ1OkBKRM2AYkL7FMeryy8cuE3UdSkwmNgX4szrAnzWBAWT5uPdj6VZRwZiSMsaUljKgrJwWMWPCZHYsvXU+LO5fVz9vN1tIt1iw+gbSPqyz+8hITUg5y1tp4r1O9+9303coauH2HN3D3mN72Xtsb63nfMw+xAXHOQOMM8y0S6BTcKdGXeonIi2cyQRxQxyPK18Au43UghTHip7D69ic/T2V9uMjC/4WK0OjBzA25jxGtx9Mp4AoMOyO4GO3VX9e/dH5tVHHMdfzTvi65mH2cYy41BUqzD6OkYv6gke9r7Oc9nwQM+A9i8HFW2mE5RTSCtJIKUwhtSCVtMI00grTSC1M5VDhIbdfLicym8x0COrgDDAJIQnEtYsjoZ3jo1pMi7RehmFQVFnEsfJj5Jfnk1+ez7HyY86vc0pz+DbzW1IKUtxe1ym4k3NFz3mx5+Hvc/L9i0RaA90SamI2u43skmxSC1NJLawOMwVpzs9LT9FMqH1g+zrDTHy7eNr5tWvy+kXk9JTbyp2B48TgceLXNR8LyguoMqpOeW0fkw+DYgY5J8wmhSa12b2DpO1SYPEgwzDILcsltcARZlILHCMyNeGmvk6SNSL8I5wBpmZEJiHE8XmYNUy/0EQawDAMbIYNm2GjtLLUGTAKKgocn5edEDgq8t3Cx6n+8XEy/hZ/Qq2hhFnDCLOGEWINcX7eO7I3IzqMINgvuBHfrUjLo8DixfLL84+HmerbSzVf55WdfPmexWQhwCcAfx9/AnwC3D+3OL4O8A3A31LH8y6Puo4F+ATga/Ztc4HIbtix2W1UGVXY7I4/bFX2KscfuZMcr/na9Xmb3YZR83+G46PdsGNggIHzc7vh2L3X+bxx/DV2HF/X1OZ2vnH8es7XuDxfcy071bsDGzi/j9tHo+6va7geP/G1ta554rXquGaVvcrxczZsbj8/m2HDbtidz1cZVdjtdZ9X32vczqk+z3ktw+782ZwNi8lCqDWUEL/jgaMmiIRaQ91CievXuqUjcmqadOvFQq2h9I/uT//o/rWeK64sdsyTKXAJM9WjNFklWdgMG0WVRfW2yT5bZpPZEWhqAo+vSxByCTr+Pv6YMNX+Y+n6h/Rkf1wNnH+Y6/rjfuIfcNc/9rXOr/kXdE2IOEXAODGIOP8QS5sQ5BtUO3D4hRLmf0IQ8av+6B9KsG9w8/c2ERE3ZxRY5s+fz7PPPktGRgZ9+/Zl3rx5jBkzpt7zV61axaxZs/jll1/o2LEj9913H9OmTXM7Z+nSpTz88MPs27ePrl278sQTT3DttdeeSXktWpBvEL0ietW5lXpZVRkFFQWUVpVSVlVGaVUpJVUlzs9dj7s+ymxllFa6fF59XklVifPzmgnEdsNOcWVxve222xIfkw8WswWLyYLFbHH72sfs4zzu+rXZZMZkMmHG8dGEyfnRbDK7fW0y1fHcieefcI2TXavWMRwjZSeOmJ14vL6v3V5z4rn1vcb5wf14zc/PbDIf/5m6/Pxcj5lNZnzMjnNrvaau19b1GrO5zvOsFqtW74m0UA0OLEuWLGHmzJnMnz+fUaNG8frrr3PppZeyfft2EhISap1/4MABLrvsMu68807+9a9/sW7dOqZPn050dDTXXXcdABs2bGDy5Mn89a9/5dprr+Xjjz9m0qRJrF27lmHDhp39u2wl/H38m2yIudJeSVlVWd2BxzUQ2Y5/bhiG2x9oTJz2H+qaf63Wer6e1zjPdw0DLn+Yfcw+9YYIZ9ioI3jUF0Rq3peIiHiHBs9hGTZsGIMGDeLVV191HuvduzfXXHMNc+bMqXX+/fffz2effcaOHcf3u5g2bRo//PADGzZsAGDy5MkUFBTwxRdfOM+55JJLCA8PZ/HixadVV0uawyIiIiIOp/v3u0E3ZSsqKtiyZQsTJ050Oz5x4kTWr19f52s2bNhQ6/yLL76Y7777jsrKypOeU981AcrLyykoKHB7iIiISOvUoMCSk5ODzWYjJibG7XhMTAyZmZl1viYzM7PO86uqqsjJyTnpOfVdE2DOnDmEhoY6H/Hx6pMoIiLSWp3RtPcT7+0bhnHS+/11nX/i8YZec/bs2eTn5zsfaWlpp12/iIiItCwNmnQbFRWFxWKpNfKRnZ1da4SkRmxsbJ3n+/j4EBkZedJz6rsmgNVqxWq1NqR8ERERaaEaNMLi5+fH4MGDSU5OdjuenJzMyJEj63zNiBEjap2/fPlyhgwZgq+v70nPqe+aIiIi0rY0eFnzrFmzmDJlCkOGDGHEiBEsWLCA1NRUZ1+V2bNnk56ezjvvvAM4VgS9/PLLzJo1izvvvJMNGzbwz3/+0231zz333MPYsWN5+umnufrqq/n000/56quvWLt2bSO9TREREWnJGhxYJk+eTG5uLo8//jgZGRn069ePZcuWkZiYCEBGRgapqanO85OSkli2bBn33nsvr7zyCh07duTFF1909mABGDlyJO+//z4PPfQQDz/8MF27dmXJkiXqwSIiIiKA9hISERERD2qSPiwiIiIinqDAIiIiIl5PgUVERES8ngKLiIiIeD0FFhEREfF6DV7W7K1qFjtpE0QREZGWo+bv9qkWLbeawFJYWAigTRBFRERaoMLCQkJDQ+t9vtX0YbHb7Rw+fJh27dqddNPEhiooKCA+Pp60tDT1d3Ghn0tt+pnUpp9J3fRzqU0/k9rays/EMAwKCwvp2LEjZnP9M1VazQiL2WwmLi6uya4fEhLSqv8Hc6b0c6lNP5Pa9DOpm34utelnUltb+JmcbGSlhibdioiIiNdTYBERERGvp8ByClarlUceeQSr1erpUryKfi616WdSm34mddPPpTb9TGrTz8Rdq5l0KyIiIq2XRlhERETE6ymwiIiIiNdTYBERERGvp8AiIiIiXk+B5RTmz59PUlIS/v7+DB48mDVr1ni6JI+ZM2cO5513Hu3ataN9+/Zcc8017Nq1y9NleZU5c+ZgMpmYOXOmp0vxuPT0dG655RYiIyMJDAxkwIABbNmyxdNleUxVVRUPPfQQSUlJBAQE0KVLFx5//HHsdrunS2tWq1ev5sorr6Rjx46YTCY++eQTt+cNw+DRRx+lY8eOBAQEMH78eH755RfPFNtMTvYzqays5P7776d///4EBQXRsWNHbr31Vg4fPuy5gj1EgeUklixZwsyZM3nwwQfZunUrY8aM4dJLLyU1NdXTpXnEqlWrmDFjBhs3biQ5OZmqqiomTpxIcXGxp0vzCps3b2bBggWcc845ni7F444ePcqoUaPw9fXliy++YPv27Tz33HOEhYV5ujSPefrpp3nttdd4+eWX2bFjB8888wzPPvssL730kqdLa1bFxcWce+65vPzyy3U+/8wzz/D888/z8ssvs3nzZmJjY7nooouc+8W1Rif7mZSUlPD999/z8MMP8/333/PRRx+xe/durrrqKg9U6mGG1Gvo0KHGtGnT3I716tXLeOCBBzxUkXfJzs42AGPVqlWeLsXjCgsLje7duxvJycnGuHHjjHvuucfTJXnU/fffb4wePdrTZXiVyy+/3Lj99tvdjv3qV78ybrnlFg9V5HmA8fHHHzu/ttvtRmxsrPHUU085j5WVlRmhoaHGa6+95oEKm9+JP5O6fPvttwZgpKSkNE9RXkIjLPWoqKhgy5YtTJw40e34xIkTWb9+vYeq8i75+fkAREREeLgSz5sxYwaXX345F154oadL8QqfffYZQ4YM4de//jXt27dn4MCBvPHGG54uy6NGjx7N119/ze7duwH44YcfWLt2LZdddpmHK/MeBw4cIDMz0+33rtVqZdy4cfq96yI/Px+TydTmRixbzeaHjS0nJwebzUZMTIzb8ZiYGDIzMz1UlfcwDINZs2YxevRo+vXr5+lyPOr999/n+++/Z/PmzZ4uxWvs37+fV199lVmzZvHnP/+Zb7/9lj/+8Y9YrVZuvfVWT5fnEffffz/5+fn06tULi8WCzWbjiSee4MYbb/R0aV6j5ndrXb93U1JSPFGS1ykrK+OBBx7gpptuavUbIp5IgeUUTCaT29eGYdQ61hbdfffd/Pjjj6xdu9bTpXhUWloa99xzD8uXL8ff39/T5XgNu93OkCFDePLJJwEYOHAgv/zyC6+++mqbDSxLlizhX//6F4sWLaJv375s27aNmTNn0rFjR37zm994ujyvot+7dausrOSGG27Abrczf/58T5fT7BRY6hEVFYXFYqk1mpKdnV0r/bc1f/jDH/jss89YvXo1cXFxni7Ho7Zs2UJ2djaDBw92HrPZbKxevZqXX36Z8vJyLBaLByv0jA4dOtCnTx+3Y71792bp0qUeqsjz/t//+3888MAD3HDDDQD079+flJQU5syZo8BSLTY2FnCMtHTo0MF5XL93HWFl0qRJHDhwgG+++abNja6AVgnVy8/Pj8GDB5OcnOx2PDk5mZEjR3qoKs8yDIO7776bjz76iG+++YakpCRPl+RxF1xwAT/99BPbtm1zPoYMGcLNN9/Mtm3b2mRYARg1alStJe+7d+8mMTHRQxV5XklJCWaz+69ci8XS5pY1n0xSUhKxsbFuv3crKipYtWpVm/29C8fDyp49e/jqq6+IjIz0dEkeoRGWk5g1axZTpkxhyJAhjBgxggULFpCamsq0adM8XZpHzJgxg0WLFvHpp5/Srl075+hTaGgoAQEBHq7OM9q1a1drDk9QUBCRkZFtem7Pvffey8iRI3nyySeZNGkS3377LQsWLGDBggWeLs1jrrzySp544gkSEhLo27cvW7du5fnnn+f222/3dGnNqqioiL179zq/PnDgANu2bSMiIoKEhARmzpzJk08+Sffu3enevTtPPvkkgYGB3HTTTR6summd7GfSsWNHrr/+er7//ns+//xzbDab83dvREQEfn5+niq7+Xl2kZL3e+WVV4zExETDz8/PGDRoUJtewgvU+Xjrrbc8XZpX0bJmh//85z9Gv379DKvVavTq1ctYsGCBp0vyqIKCAuOee+4xEhISDH9/f6NLly7Ggw8+aJSXl3u6tGa1YsWKOn+P/OY3vzEMw7G0+ZFHHjFiY2MNq9VqjB071vjpp588W3QTO9nP5MCBA/X+7l2xYoWnS29WJsMwjOYMSCIiIiINpTksIiIi4vUUWERERMTrKbCIiIiI11NgEREREa+nwCIiIiJeT4FFREREvJ4Ci4iIiHg9BRYRERHxegosIiIi4vUUWERERMTrKbCIiIiI11NgEREREa/3/wFeGur1AV/LvQAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
@@ -2416,9 +2173,7 @@
{
"cell_type": "markdown",
"id": "c50c02c2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Summing up\n",
"\n",
@@ -2454,9 +2209,7 @@
{
"cell_type": "markdown",
"id": "16a69276",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Another Example from Scikit-Learn's Repository"
]
@@ -2465,10 +2218,7 @@
"cell_type": "code",
"execution_count": 5,
"id": "06c4dbd1",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"\"\"\"\n",
@@ -2547,9 +2297,7 @@
{
"cell_type": "markdown",
"id": "dd22f4e5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Various steps in cross-validation\n",
"\n",
@@ -2572,9 +2320,7 @@
{
"cell_type": "markdown",
"id": "1d78f931",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Cross-validation in brief\n",
"\n",
@@ -2600,9 +2346,7 @@
{
"cell_type": "markdown",
"id": "a3e7ddb7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Code Example for Cross-validation and $k$-fold Cross-validation\n",
"\n",
@@ -2613,10 +2357,7 @@
"cell_type": "code",
"execution_count": 6,
"id": "d7f6476c",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
@@ -2713,9 +2454,7 @@
{
"cell_type": "markdown",
"id": "29f97a19",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## More examples on bootstrap and cross-validation and errors"
]
@@ -2724,10 +2463,7 @@
"cell_type": "code",
"execution_count": 7,
"id": "ea778a04",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Common imports\n",
@@ -2813,9 +2549,7 @@
{
"cell_type": "markdown",
"id": "f5bc26cd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Note that we kept the intercept column in the fitting here. This means that we need to set the **intercept** in the call to the **Scikit-Learn** function as **False**. Alternatively, we could have set up the design matrix $X$ without the first column of ones."
]
@@ -2823,9 +2557,7 @@
{
"cell_type": "markdown",
"id": "4bbb49da",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The same example but now with cross-validation\n",
"\n",
@@ -2836,10 +2568,7 @@
"cell_type": "code",
"execution_count": 8,
"id": "89f26923",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Common imports\n",
@@ -2914,9 +2643,7 @@
{
"cell_type": "markdown",
"id": "3d39d1c9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Notes on scaling with examples\n",
"\n",
@@ -2941,10 +2668,7 @@
"cell_type": "code",
"execution_count": 9,
"id": "2470e6f1",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
@@ -3014,9 +2738,7 @@
{
"cell_type": "markdown",
"id": "3e229ae7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In this example we do not include the intercept and we scale the data by subtracting the mean values. This follows the discussion in the [lecture material](https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/chapter3.html#more-on-rescaling-data).\n",
"see also the weekly slides [for week 36](https://compphysics.github.io/MachineLearning/doc/pub/week36/html/._week36-bs029.html).\n",
@@ -3034,9 +2756,7 @@
{
"cell_type": "markdown",
"id": "2d43d360",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\beta_0, \\beta_1, ... , \\beta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij}\\beta_j\\right)^2,.\n",
@@ -3046,9 +2766,7 @@
{
"cell_type": "markdown",
"id": "777c97c9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Recall also that we use the squared value. This expression can lead to an\n",
"increased penalty for higher differences between predicted and\n",
@@ -3063,9 +2781,7 @@
{
"cell_type": "markdown",
"id": "f464bb58",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial C}{\\partial \\beta_j} = 0,\n",
@@ -3075,9 +2791,7 @@
{
"cell_type": "markdown",
"id": "624b1d2c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"for all $j$. For $\\beta_0$ we have"
]
@@ -3085,9 +2799,7 @@
{
"cell_type": "markdown",
"id": "b872d2db",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial C}{\\partial \\beta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\beta_0 - \\sum_{j=1}^{p-1} X_{ij} \\beta_j\\right).\n",
@@ -3097,9 +2809,7 @@
{
"cell_type": "markdown",
"id": "d6dd0d07",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Multiplying away the constant $2/n$, we obtain"
]
@@ -3107,9 +2817,7 @@
{
"cell_type": "markdown",
"id": "2f7c34ed",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\sum_{i=0}^{n-1} \\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\beta_j.\n",
@@ -3119,9 +2827,7 @@
{
"cell_type": "markdown",
"id": "7e06d400",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Let us specialize first to the case where we have only two parameters $\\beta_0$ and $\\beta_1$.\n",
"Our result for $\\beta_0$ simplifies then to"
@@ -3130,9 +2836,7 @@
{
"cell_type": "markdown",
"id": "bad7ab31",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"n\\beta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\beta_1.\n",
@@ -3142,9 +2846,7 @@
{
"cell_type": "markdown",
"id": "b11fd1d0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We obtain then"
]
@@ -3152,9 +2854,7 @@
{
"cell_type": "markdown",
"id": "e18f186d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\beta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n",
@@ -3164,9 +2864,7 @@
{
"cell_type": "markdown",
"id": "002f906f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"If we define"
]
@@ -3174,9 +2872,7 @@
{
"cell_type": "markdown",
"id": "4b2ce40a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n",
@@ -3186,9 +2882,7 @@
{
"cell_type": "markdown",
"id": "6f6da2aa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and the mean value of the outputs as"
]
@@ -3196,9 +2890,7 @@
{
"cell_type": "markdown",
"id": "f8705ff4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n",
@@ -3208,9 +2900,7 @@
{
"cell_type": "markdown",
"id": "481c0458",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"we have"
]
@@ -3218,9 +2908,7 @@
{
"cell_type": "markdown",
"id": "de837aab",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_0 = \\mu_y - \\beta_1\\mu_{\\boldsymbol{x}_1}.\n",
@@ -3230,9 +2918,7 @@
{
"cell_type": "markdown",
"id": "2c151900",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In the general case with more parameters than $\\beta_0$ and $\\beta_1$, we have"
]
@@ -3240,9 +2926,7 @@
{
"cell_type": "markdown",
"id": "6df6dc61",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\beta_j.\n",
@@ -3252,9 +2936,7 @@
{
"cell_type": "markdown",
"id": "6d6150d9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We can rewrite the latter equation as"
]
@@ -3262,9 +2944,7 @@
{
"cell_type": "markdown",
"id": "904ad761",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\beta_j,\n",
@@ -3274,9 +2954,7 @@
{
"cell_type": "markdown",
"id": "0e427a81",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where we have defined"
]
@@ -3284,9 +2962,7 @@
{
"cell_type": "markdown",
"id": "d40dc5aa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n",
@@ -3296,9 +2972,7 @@
{
"cell_type": "markdown",
"id": "259f8a07",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n",
"\n",
@@ -3308,9 +2982,7 @@
{
"cell_type": "markdown",
"id": "be7a32a4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\boldsymbol{\\beta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\beta}).\n",
@@ -3320,9 +2992,7 @@
{
"cell_type": "markdown",
"id": "e1723452",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"If we minimize with respect to $\\boldsymbol{\\beta}$ we have then"
]
@@ -3330,9 +3000,7 @@
{
"cell_type": "markdown",
"id": "a8979b1c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n",
@@ -3342,9 +3010,7 @@
{
"cell_type": "markdown",
"id": "7c9b9a4d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n",
"and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n",
@@ -3355,9 +3021,7 @@
{
"cell_type": "markdown",
"id": "6daf72ea",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\hat{\\boldsymbol{\\beta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n",
@@ -3367,9 +3031,7 @@
{
"cell_type": "markdown",
"id": "23161172",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Now we try to implement this."
]
@@ -3378,10 +3040,7 @@
"cell_type": "code",
"execution_count": 10,
"id": "9b9f556f",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"\n",
@@ -3451,9 +3110,7 @@
{
"cell_type": "markdown",
"id": "b0a158df",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Finally, instead of using our own function we repeat the same example\n",
"using the **standardscaler** functionality of the library\n",
@@ -3464,10 +3121,7 @@
"cell_type": "code",
"execution_count": 11,
"id": "911c4d30",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"from sklearn import linear_model\n",
@@ -3532,7 +3186,25 @@
]
}
],
- "metadata": {},
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
"nbformat": 4,
"nbformat_minor": 5
}
diff --git a/doc/pub/week38/ipynb/week38.ipynb b/doc/pub/week38/ipynb/week38.ipynb
index a9e42c232..b4915cba6 100644
--- a/doc/pub/week38/ipynb/week38.ipynb
+++ b/doc/pub/week38/ipynb/week38.ipynb
@@ -3,9 +3,7 @@
{
"cell_type": "markdown",
"id": "92c39815",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"\n",
@@ -15,9 +13,7 @@
{
"cell_type": "markdown",
"id": "c28ac15e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"# Week 38: Logistic Regression and Optimization\n",
"**Morten Hjorth-Jensen**, Department of Physics and Center for Computing in Science Education, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n",
@@ -28,9 +24,7 @@
{
"cell_type": "markdown",
"id": "04bf16b5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Plans for week 38\n",
"\n",
@@ -69,9 +63,7 @@
{
"cell_type": "markdown",
"id": "e47b1a7c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Material from last week and relevant for the first project"
]
@@ -79,9 +71,7 @@
{
"cell_type": "markdown",
"id": "1ad44c38",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Various steps in cross-validation\n",
"\n",
@@ -104,9 +94,7 @@
{
"cell_type": "markdown",
"id": "48181efc",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## How to set up the cross-validation for Ridge and/or Lasso\n",
"\n",
@@ -120,9 +108,7 @@
{
"cell_type": "markdown",
"id": "62106e79",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*}\n",
@@ -136,9 +122,7 @@
{
"cell_type": "markdown",
"id": "7d392fb4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"* Evaluate the prediction performance of these models on the test set by $C[y_i, \\boldsymbol{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]$. Or, by the prediction error $|y_i - \\boldsymbol{X}_{i, \\ast} \\boldsymbol{\\beta}_{-i}(\\lambda)|$, the relative error, the error squared or the R2 score function.\n",
"\n",
@@ -150,9 +134,7 @@
{
"cell_type": "markdown",
"id": "883c8ba9",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Cross-validation in brief\n",
"\n",
@@ -178,9 +160,7 @@
{
"cell_type": "markdown",
"id": "2db06c8a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Code Example for Cross-validation and $k$-fold Cross-validation\n",
"\n",
@@ -189,13 +169,21 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 3,
"id": "7487b649",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAG0CAYAAADacZikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABYLUlEQVR4nO3dd3gUZb/G8e+mF1JIgBQSejP03pQiHaWoCDaKvQCCKCIqihWxwavYC+W1gIoURVqUKiA1SCcgkFBiqGmQunP+4HWPgSSQkGR2s/fnuuYcZuaZ2XuW5d2fzzz7jMUwDAMRERERJ+JidgARERGR0qYCSERERJyOCiARERFxOiqARERExOmoABIRERGnowJIREREnI4KIBEREXE6KoBERETE6agAEhEREaejAkhEREScjqkF0KRJk2jZsiV+fn5UqlSJ/v37s2/fvlxtDMNg4sSJhIeH4+3tTadOndi1a9cVzz137lyioqLw9PQkKiqKefPmldRliIiIiIOxmPkssJ49e3LHHXfQsmVLsrOzee6559ixYwe7d+/G19cXgMmTJ/Paa68xY8YM6tSpw6uvvsrq1avZt28ffn5+eZ53/fr13HDDDbzyyivccsstzJs3jxdeeIG1a9fSunXrK+ayWq0cP34cPz8/LBZLsV6ziIiIlAzDMEhJSSE8PBwXlyv08Rh2JDEx0QCMVatWGYZhGFar1QgNDTXeeOMNW5v09HQjICDA+Pjjj/M9z8CBA42ePXvm2tajRw/jjjvuuKoc8fHxBqBFixYtWrRoccAlPj7+it/1btiRpKQkAIKCggA4dOgQCQkJdO/e3dbG09OTjh07sm7dOh5++OE8z7N+/XqeeOKJXNt69OjB1KlT82yfkZFBRkaGbd34X6dYfHw8/v7+Rb4eERERKT3JyclERkbme4fo3+ymADIMgzFjxnD99dfToEEDABISEgAICQnJ1TYkJIQjR47ke66EhIQ8j/nnfJeaNGkSL7300mXb/f39VQCJiIg4mKsZvmI3vwIbMWIEf/75J99+++1l+y69EMMwrnhxhTlm/PjxJCUl2Zb4+PhCphcRERFHYhc9QCNHjmThwoWsXr2aiIgI2/bQ0FDgYo9OWFiYbXtiYuJlPTz/FhoaellvT0HHeHp64unpeS2XICIiIg7E1B4gwzAYMWIEP/74I7/99hvVq1fPtb969eqEhoayfPly27bMzExWrVpFu3bt8j1v27Ztcx0DsGzZsgKPEREREedhag/Q8OHD+eabb1iwYAF+fn62XpuAgAC8vb2xWCyMHj2a119/ndq1a1O7dm1ef/11fHx8uOuuu2znGTJkCJUrV2bSpEkAjBo1ig4dOjB58mT69evHggULiI6OZu3atcWaPycnh6ysrGI9p8il3N3dcXV1NTuGiEiZYmoB9NFHHwHQqVOnXNunT5/OsGHDAHj66ae5cOECjz32GGfPnqV169YsW7Ys1wjvuLi4XL/3b9euHbNnz+b5559nwoQJ1KxZkzlz5lzVHEBXwzAMEhISOHfuXLGcT+RKAgMDCQ0N1bxUIiLFxNSJEO1VcnIyAQEBJCUl5fkrsBMnTnDu3DkqVaqEj4+PvpSkxBiGwfnz50lMTCQwMDDXWDgREcntSt/f/2YXg6AdSU5Ojq34CQ4ONjuOOAFvb2/g4kD+SpUq6XaYiEgxsJufwTuKf8b8+Pj4mJxEnMk/nzeNORMRKR4qgIpIt72kNOnzJiJSvFQAiYiIiNNRASQiIiJORwWQlEkWi4X58+ebHUNEROyUfgUmIiIipcKanUPamXOknU/HGhhEeKC3aVlUAEmesrKycHd3NztGmZOZmYmHh4fZMUREiswwDFLPJJF85Dhpx05w4fjfZCYkkpOYiHHqFK6nT/PNXWM4gzsp6VkMmPcxXTctxSc9Dd+MC/hhsKlGCz55+j3mPNzWtOvQLbBiYBgG5zOzTVkKM4+l1Wpl8uTJ1KpVC09PT6pUqcJrr73G4cOHsVgsfPfdd3Tq1AkvLy+++uorrFYrL7/8MhEREXh6etKkSROWLFliO19mZiYjRowgLCwMLy8vqlWrZnscCcDEiROpUqUKnp6ehIeH8/jjj18x4/jx42nTps1l2xs1asSLL74IwKZNm+jWrRsVKlQgICCAjh07snXr1qt+H/7tStdw7tw5HnroIUJCQvDy8qJBgwb8/PPPtv1z586lfv36eHp6Uq1aNd55551c569WrRqvvvoqw4YNIyAggAcffBCAdevW0aFDB7y9vYmMjOTxxx8nLS2tSNcgIlJcDKuVUwfj2Dt/GVvf+YwNj09gw633sqVdT+74YDU3vPkbdScsYVnHW6ncvD51+nal8SN303LiE7T5cBJtv/uMVr/+yLotB1m9/yTb4s6RcTaZkKST+GWcx4WL31m+2RmY/eNW9QAVgwtZOUS9sNSU1979cg98PK7ur3H8+PF89tlnTJkyheuvv54TJ06wd+9e2/5x48bxzjvvMH36dDw9PfnPf/7DO++8wyeffELTpk358ssv6du3L7t27aJ27dq89957LFy4kO+++44qVaoQHx9PfHw8AD/88ANTpkxh9uzZ1K9fn4SEBLZv337FjHfffTdvvPEGBw8epGbNmgDs2rWLHTt28MMPPwCQkpLC0KFDee+99wB455136N27N7GxsbkekXI1CroGq9VKr169SElJ4auvvqJmzZrs3r3bNhHhli1bGDhwIBMnTmTQoEGsW7eOxx57jODgYNujXADeeustJkyYwPPPPw/Ajh076NGjB6+88gpffPEFJ0+eZMSIEYwYMYLp06cXKr+ISFGkJJ7m2OqN/BkZxeGzFzh8Oo0bv3ibXr8voELmBSrkccyRJndwwr8iAGe9/Tnv7kmybwCp5QI5H1CezMAgsoOCMYKDeaJPYzwqBlPOy40KPV8g9vwYvIID8akUhG+FIFr5ejPbxdw+GD0KIw8FTaWdnp7OoUOHqF69Ol5eXgCcz8y2+wIoJSWFihUrMm3aNB544IFc+w4fPkz16tWZOnUqo0aNsm2vXLkyw4cP59lnn7Vta9WqFS1btuSDDz7g8ccfZ9euXURHR182T827777LJ598ws6dOwt9K61x48YMGDCACRMmAPDss88SHR3Nxo0b82yfk5ND+fLl+eabb7j55puBi4Og582bR//+/Qt8rYKuYdmyZfTq1Ys9e/ZQp06dy469++67OXnyJMuWLbNte/rpp1m0aBG7du0CLvYANW3alHnz5tnaDBkyBG9vbz755BPbtrVr19KxY0fS0tJsn6t/y+tzJyJyJYbVSsLO/RxftprMLdvw3ruLSkdiCT978eHjbR6dQYL/xXJn7KqZDN/wPVYsJAZW5GxwKOcrhJAVGgYRESQNvJPgKuGE+HtRwdsNb2/7u52vR2GUMm93V3a/3MO0174ae/bsISMjgy5duuTbpkWLFrY/Jycnc/z4cdq3b5+rTfv27W09OcOGDaNbt27UrVuXnj17cvPNN9O9e3cAbr/9dqZOnUqNGjXo2bMnvXv3pk+fPri5Xfkjd/fdd/Pll18yYcIEDMPg22+/ZfTo0bb9iYmJvPDCC/z222/8/fff5OTkcP78eeLi4q7qvfi3gq4hJiaGiIiIPIsfuPie9uvX77L3Z+rUqeTk5Nh6iv79vsLFnqMDBw7w9ddf27YZhoHVauXQoUNcd911hb4OERGAjLTz7DiRytbjKWw9co62n73F0DVzyOspgn8HVKRLJVdcm1alarAvdbs/wxHPcYQ2uY5QXx9CSz196VIBVAwsFstV34Yyyz/PkyqIr6/vZdsu7RUxDMO2rVmzZhw6dIjFixcTHR3NwIED6dq1Kz/88AORkZHs27eP5cuXEx0dzWOPPcZbb73FqlWrrtgjdNddd/HMM8+wdetWLly4QHx8PHfccYdt/7Bhwzh58iRTp06latWqeHp60rZtWzIzM6/mrciloGu40nv27/fi39suden7arVaefjhh/McE1WlSpVCX4OIOC/DauXQig0k/vgzPqtXUHvvNiYNeoUtEVEAeAZEcJeLK4cr1+Js3QYYDRtSrmVTIm5oRUhEKK/lOlt1My7BNPb9rS3Fpnbt2nh7e/Prr79edgssL/7+/oSHh7N27Vo6dOhg275u3TpatWqVq92gQYMYNGgQAwYMoGfPnpw5c4agoCC8vb3p27cvffv2Zfjw4dSrV48dO3bQrFmzAl87IiKCDh068PXXX3PhwgW6du1KSEiIbf+aNWv48MMP6d27NwDx8fGcOnWqsG/JFa+hUaNGHD16lP379+fZCxQVFcXatWtzbVu3bh116tQp8IGlzZo1Y9euXdSqVavImUXEeWWlZ7D3mwWcn/MDNdf9So3UM9T41/62Zw8R3L0zzaqWp/nQJmT/90VqBxZufKQzUAHkJLy8vBg3bhxPP/00Hh4etG/fnpMnT7Jr1658b4uNHTuWF198kZo1a9KkSROmT59OTEyM7dbNlClTCAsLo0mTJri4uPD9998TGhpKYGAgM2bMICcnh9atW+Pj48N///tfvL29qVq16lXlvfvuu5k4cSKZmZlMmTIl175atWrx3//+lxYtWpCcnMzYsWOvqocrLwVdQ8eOHenQoQO33XYb7777LrVq1WLv3r1YLBZ69uzJk08+ScuWLXnllVcYNGgQ69evZ9q0aXz44YcFvua4ceNo06YNw4cP58EHH8TX15c9e/awfPly3n///SJdh4iUbdk5VtYcOMXP20+Q9MtSPp85zrbvvLsnsfWacaHjjYTc1ocnO7TEYvIAY4dgyGWSkpIMwEhKSrps34ULF4zdu3cbFy5cMCHZtcnJyTFeffVVo2rVqoa7u7tRpUoV4/XXXzcOHTpkAMa2bdsua//SSy8ZlStXNtzd3Y3GjRsbixcvtu3/9NNPjSZNmhi+vr6Gv7+/0aVLF2Pr1q2GYRjGvHnzjNatWxv+/v6Gr6+v0aZNGyM6Ovqqs549e9bw9PQ0fHx8jJSUlFz7tm7darRo0cLw9PQ0ateubXz//fdG1apVjSlTptjaAMa8efOu+DoFXYNhGMbp06eNe++91wgODja8vLyMBg0aGD///LNt/w8//GBERUXZ3s+33nor1/kvzfWPjRs3Gt26dTPKlStn+Pr6Go0aNTJee+21fHM68udORIru8NrNxrrbHzDe6/mgUXXcz0bVcT8bNcYuMA5UrGps6DbA2P75bCM9Nc3smHajoO/vS+lXYHko7K/AREqaPnciziMnK5sdn3yD+7T3qL9vCwCJvuXp/eTX3NQskt4Nw2hRtTyururluZR+BSYiIuJg0pNTiXnxbSL++xlNTh8HIMfiwo5G7TCGDWPdI13w8LK/n547KpWPUqrWrFlDuXLl8l2K2+uvv57va/Xq1avYX09EpLDSs3L4cu0hfu52F22mvkTE6eMke5Vj/YD7Obl9D01i1tB09P0qfoqZeoCkVLVo0YKYmJhSe71HHnmEgQMH5rmvqAOnRUSKQ3ZGJnPX7uedDQkkpmRQs1Fv2sZu5uh9j9LwucdpWz7A7IhlmgogKVXe3t6l+vPvoKAggoKCSu31RESuxs5ZP+I77ilcK9Qg8aYnqBzozf239KbiBw9Q2c7nlSsr9C6LiIiUkoSdsZwY+hBNt64EIDD5NK93DGdAt8Z4uGlUSmnSuy0iIlLCDKuVjePfoFzzJjTdupJsiwt/3HQXLvv2cVevpip+TKAeIBERkRJ0fOcBzgy8k1Z7NgOwt3oDvKZ/QeuOra5wpJQklZwiIiIl5JcdJxg4cxshRw5wwc2TDSOeo/a+bVRT8WM69QCJiIgUs4yMTF5fsp+Z64+Aqw/THnyZB+7sQJvWjc2OJv+jHiApNsOGDaN///4FtunUqROjR48ulTwiImY48edejtRpwqkvvwLgkY41mfDOcCJV/NgVFUBOJK8C5YcffsDLy4s333yTiRMnYrFYLluio6PNCSwi4mD2zl+GR7u21Inbw7OrZzD97sY806sebnpshd3R34gT+/zzz7n77ruZNm0aTz/9NAD169fnxIkTuZYOHTqYnFRExP5tnvwR1QfcTHDaOQ5Wro3LqpV0bhhhdizJhwqg4pSWlv+Snn71bS9cuLq21+DNN99kxIgRfPPNNzzwwAO27W5uboSGhuZaPDwuTr++Y8cObrzxRry9vQkODuahhx4iNTW1gLcjjSFDhlCuXDnCwsJ45513rimziIi9Wj9yAi2eeQzPnCximtxA6PaNhDWqZ3YsKYAKoOJUrlz+y2235W5bqVL+bS99RlW1anm3K6JnnnmGV155hZ9//pnbLs2Vj/Pnz9OzZ0/Kly/Ppk2b+P7774mOjmbEiBH5HjN27FhWrFjBvHnzWLZsGStXrmTLli1Fzi0iYm8Mq5X1g0fSdtqrAGzocw8NN/6Gb3CgucHkivQrMCezePFiFixYwK+//sqNN9542f4dO3bkeihpVFQUGzdu5Ouvv+bChQvMmjULX19fAKZNm0afPn2YPHkyISEhuc6TmprKF198waxZs+jWrRsAM2fOJCJC3cEiUjYYhsHLP++myp4jtAXW3/cEbT57G4uL+hYcgQqg4lTA7SBcXXOvJybm3/bSfzyHDxc50qUaNWrEqVOneOGFF2jZsiV+fn659tetW5eFCxfa1j09PQHYs2cPjRs3thU/AO3bt8dqtbJv377LCqCDBw+SmZlJ27ZtbduCgoKoW7dusV2LiIhZDMPg1UV7mL7uCJYuDxIx+Ha6jRpsdiwpBBVAxelfxYFpba+gcuXKzJ07l86dO9OzZ0+WLFmSqwjy8PDI82GlhmFgsVjyPGde2w3DKLbMIiL2xLBa+enZKczMqQWubky6rTHdWlUxO5YUkvrpnFCVKlVYtWoViYmJdO/eneTk5CseExUVRUxMDGn/Gnz9+++/4+LiQp06dS5rX6tWLdzd3dmwYYNt29mzZ9m/f3/xXISIiEn+eGgsfSc/xQcL3uDlPtdxh4ofh2RqAbR69Wr69OlDeHg4FouF+fPn59qf15w0FouFt956K99zzpgxI89j0i/9FZaTi4iIYOXKlZw+fZru3buTlJRUYPu7774bLy8vhg4dys6dO1mxYgUjR45k8ODBl93+AihXrhz3338/Y8eO5ddff2Xnzp0MGzYMF90bFxEHtumlKbT54l0AArt1Zkj7GiYnkqIy9dsoLS2Nxo0bM23atDz3XzofzZdffonFYrniL5f8/f0vO9bLy6skLsGhVa5cmVWrVnHu3Dm6devGuXPn8m3r4+PD0qVLOXPmDC1btmTAgAF06dIl3787gLfeeosOHTrQt29funbtyvXXX0/z5s1L4EpERErejhnf0+TlsQCsv/0BWn/wusmJ5FpYDDsZrGGxWJg3b16Bj1Lo378/KSkp/Prrr/m2mTFjBqNHjy7wy/xKkpOTCQgIICkpCX9//1z70tPTOXToENWrV1dRJaVGnzsRcx1asZ6KPbtSLvM8m9v1pNmqn3Fxc73ygVKqCvr+vpTD3I/4+++/WbRoEffff/8V26amplK1alUiIiK4+eab2bZtW4HtMzIySE5OzrWIiIgAJB1NwHPAbZTLPM+u2k1puGyuip8ywGEKoJkzZ+Ln58ett95aYLt69eoxY8YMFi5cyLfffouXlxft27cnNjY232MmTZpEQECAbYmMjCzu+CIi4oByrAbT/vMjASlnOR4URsRvv+Dp62N2LCkGDlMAffnll7aBuAVp06YN99xzD40bN+aGG27gu+++o06dOrz//vv5HjN+/HiSkpJsS3x8fHHHFxERB/Tm0r185lqVO++dQvp3PxAQEWp2JCkmDjEP0Jo1a9i3bx9z5swp9LEuLi60bNmywB4gT09P24R/V8tOhk6Jk9DnTaT0Ld+VwCer/gLgoeF9qdEo3OREUpwcogfoiy++oHnz5jRu3LjQxxqGQUxMDGFhYcWSxd3dHbj4bCyR0vLP5+2fz5+IlKy/d8dSpWt7Wsft4P7rq3Ozip8yx9QeoNTUVA4cOGBbP3ToEDExMQQFBVGlysWJpZKTk/n+++/zfZL4kCFDqFy5MpMmTQLgpZdeok2bNtSuXZvk5GTee+89YmJi+OCDD4ols6urK4GBgST+71EWPj4++c6QLHKtDMPg/PnzJCYmEhgYiOulj1QRkWKXk5XNqf6DqJ/wFy+vm0X1mWPNjiQlwNQCaPPmzXTu3Nm2PmbMGACGDh3KjBkzAJg9ezaGYXDnnXfmeY64uLhck+udO3eOhx56iISEBAICAmjatCmrV6+mVatWxZY7NPTiPeDEgp7nJVKMAgMDbZ87ESlZGx8aS9vYbaR5eOMz51s83PUfHmWR3cwDZE+udh6BnJwcsrKySjGZOCN3d3f1/IiUkgNLV1O19424W3PY9PJUWk4YZXYkKYTCzAPkEIOg7ZWrq6u+mEREyojM8+m43Hcf7tYctrbqouKnjHOIQdAiIiIlbcvDT1Hj+EHO+gRQdfZ0s+NICVMBJCIiTm/n0XOc3bwdgEMTJxNcXRPilnW6BSYiIk4tx2rw7Pyd/Nl3HI8PGMKYsQ+aHUlKgQogERFxat/8cYQ/jybh5+3OPU8ONjuOlBLdAhMREad16mAc7o+PJOh8EmN71KWSX8GPW5KyQz1AIiLitA7d+xh3bF5E3ZQEGk3Ne745KZvUAyQiIk5pz/e/0HLNIqxY8H3rDVxdNKu/M1EBJCIiTseanYPbU08CsLnrrdTp08XkRFLaVACJiIjT2Tr5Q2rH7SXNw5saH79rdhwxgQogERFxKunJqUS89QoAOwY/SoWaVUxOJGZQASQiIk5l69iXCU06SUJARZq8+5LZccQk+hWYiIg4jTNpmYz1b8n9zfvScEAPQv3LmR1JTKICSEREnMYnqw5yzNWHuUOeYtiI682OIybSLTAREXEKiWdSmLn+MABPdq+Di3727tRUAImIiFP4a+ijfP7f8dxiOUnnupXMjiMm0y0wEREp8xJ2xtJs8Rw8crIJvM4fi0W9P85OPUAiIlLmHXnyOTxystlVpxn17+lvdhyxAyqARESkTDu59yBNf50HgOXll7G46KtPVACJiEgZd3D8K3jkZLOnZiOiBt1kdhyxEyqARESkzDoXd4JGi+YAkDnuGZPTiD1RASQiImXWzpffwScrnYOVa9Po/kFmxxE7ol+BiYhImZSakc2osM7c2CuLAX1aUVNjf+Rf9GkQEZEyac6meE5nweYbb6HFw3eZHUfsjAogEREpc3Kyc5i19gAAD9xQHVfN+iyXUAEkIiJlzp/TZjBz8lCG7FvBrU0jzI4jdkgFkIiIlDle096n2rkT9PJKxdvD1ew4YodUAImISJlyYNFKrju4nUwXN2pNfNrsOGKnVACJiEiZcm7SmwBsb9edinVrmJxG7JUKIBERKTNO7T9E4/XLAAgcr94fyZ8KIBERKTNiJ72HuzWHvTUaUrt3R7PjiB1TASQiImVCTlY21ed9DUDqvQ+YnEbsnQogEREpE37bf4pRvccwv2l3Goy63+w4YudUAImISJnw9cY4/qjSkN2vTsXLz9fsOGLnVACJiIjDiz9znlX7TwJwV6sqJqcRR2BqAbR69Wr69OlDeHg4FouF+fPn59o/bNgwLBZLrqVNmzZXPO/cuXOJiorC09OTqKgo5s2bV0JXICIi9uCvJ59nwvJPuS0gnWoV1PsjV2ZqAZSWlkbjxo2ZNm1avm169uzJiRMnbMsvv/xS4DnXr1/PoEGDGDx4MNu3b2fw4MEMHDiQP/74o7jji4iIHchKz6DB99O5b8tC7vE+Z3YccRAWwzAMs0MAWCwW5s2bR//+/W3bhg0bxrlz5y7rGSrIoEGDSE5OZvHixbZtPXv2pHz58nz77bdXdY7k5GQCAgJISkrC39//ql9bRERKX8wH/6XJiCGc9g3E/1QC7l6eZkcSkxTm+9vuxwCtXLmSSpUqUadOHR588EESExMLbL9+/Xq6d++ea1uPHj1Yt25dvsdkZGSQnJycaxEREcdgnTEdgNhu/VT8yFWz6wKoV69efP311/z222+88847bNq0iRtvvJGMjIx8j0lISCAkJCTXtpCQEBISEvI9ZtKkSQQEBNiWyMjIYrsGEREpOWePHKfB1jUAhDz+sMlpxJHYdQE0aNAgbrrpJho0aECfPn1YvHgx+/fvZ9GiRQUeZ7FYcq0bhnHZtn8bP348SUlJtiU+Pr5Y8ouISMnaN+UTPKzZxEbWpXrntmbHEQfiZnaAwggLC6Nq1arExsbm2yY0NPSy3p7ExMTLeoX+zdPTE09PdZuKiDiaij9cHNt5ZsCdJicRR2PXPUCXOn36NPHx8YSFheXbpm3btixfvjzXtmXLltGuXbuSjiciIqVo9+GT/BZen+P+Fak7+iGz44iDMbUHKDU1lQMHDtjWDx06RExMDEFBQQQFBTFx4kRuu+02wsLCOHz4MM8++ywVKlTglltusR0zZMgQKleuzKRJkwAYNWoUHTp0YPLkyfTr148FCxYQHR3N2rVrS/36RESk5Hy/I5HpNz5AzMhn+aBK/v9hLJIXUwugzZs307lzZ9v6mDFjABg6dCgfffQRO3bsYNasWZw7d46wsDA6d+7MnDlz8PPzsx0TFxeHi8v/d2S1a9eO2bNn8/zzzzNhwgRq1qzJnDlzaN26deldmIiIlKjMbCsLYo4DMKBlVZPTiCOym3mA7InmARIRsW/rZy9h2vwtxNZvybpnu+Lm6lAjOqSEFOb726EGQYuIiAB4Tnmbrzf+yuo7H8PNtfuVDxC5hEpmERFxKKmnzhL1v7l/wocMNDmNOCoVQCIi4lD2fvxfvLIzia8YSc3uN5gdRxyUCiAREXEo7j98B8DRHn2xuOhrTIpGnxwREXEYSfEniNqxAYDwh4eZG0YcmgogERFxGPs+moW7NYeD4bWoen0Ls+OIA1MBJCIiDiPztxUAJN7U39wg4vD0M3gREXEIicnpDO40ggZ1e/LxE33NjiMOTj1AIiLiEBbtOIGBBbfWLal8XQ2z44iDUwEkIiIOYdHWOAD6Ng43OYmUBboFJiIidi9hx34+f/pmltRtx43PzDM7jpQB6gESERG7d/jzrwhMT6XR+UQqBfqYHUfKABVAIiJi9/wX/wRAcs+bTU4iZYUKIBERsWunD8VTL3Y7AFUfuNvkNFJWqAASERG7dvDzb3DBIDayLmGN6pkdR8oIFUAiImLXPH9aAMDp7jeZnETKEhVAIiJit5L/PsV1uzYBEHbvXSankbJEP4MXERG7tXZPArFtBtAiKZ727ZubHUfKEBVAIiJit34+lsEvN9zDiM61aG92GClTdAtMRETsUnpWDiv2ngSgZ4NQk9NIWaMCSERE7FLMj8u5Yddaqvu6UD/c3+w4UsboFpiIiNgl148+5NNVP7HeOhSLpZfZcaSMUQ+QiIjYnZysbGptWg2A/6BbTU4jZZEKIBERsTuxP0VT/nwSyV7lqHtrT7PjSBmkAkhEROzOuTk/ArC/2fW4eXqYnEbKIhVAIiJid0JWR1/8w82a/VlKhgogERGxK8dj9lA94RDZFhdqDx5gdhwpo1QAiYiIXflr7i8A7K/ViIAIzf8jJUMFkIiI2JVPq99Ax4c+Zc+TL5odRcowzQMkIiJ2Iy0jmw0HT5NZPpzGt3Q0O46UYeoBEhERu7Em9hSZOVaqBvtQs6Kv2XGkDFMPkIiI2I1yT4/h44OHOfbASCwWi9lxpAxTASQiInbBmp1DvTVLqJB6lp0hT5kdR8o43QITERG7cGDxSiqkniXF04c6A3qbHUfKOBVAIiJiF07/sBCA2EZt8PDxMjmNlHUqgERExC6UX/MbANnde5icRJyBqQXQ6tWr6dOnD+Hh4VgsFubPn2/bl5WVxbhx42jYsCG+vr6Eh4czZMgQjh8/XuA5Z8yYgcViuWxJT08v4asREZGiSjqaQO3DuwGoetctJqcRZ2BqAZSWlkbjxo2ZNm3aZfvOnz/P1q1bmTBhAlu3buXHH39k//799O3b94rn9ff358SJE7kWLy91p4qI2KsD38zD1bByOLQ6IVG1zY4jTsDUX4H16tWLXr165bkvICCA5cuX59r2/vvv06pVK+Li4qhSpUq+57VYLISGXv306RkZGWRkZNjWk5OTr/pYERG5dttPZ2GtHEV2m7ZUMzuMOAWHGgOUlJSExWIhMDCwwHapqalUrVqViIgIbr75ZrZt21Zg+0mTJhEQEGBbIiMjizG1iIgUxDAMPvaP4vZ73sQ6aZLZccRJOEwBlJ6ezjPPPMNdd92Fv79/vu3q1avHjBkzWLhwId9++y1eXl60b9+e2NjYfI8ZP348SUlJtiU+Pr4kLkFERPKwNyGFxJQMvN1daVE9yOw44iQcYiLErKws7rjjDqxWKx9++GGBbdu0aUObNm1s6+3bt6dZs2a8//77vPfee3ke4+npiaenZ7FmFhGRqxMT/Qf+6am0qFcDTzdXs+OIk7D7HqCsrCwGDhzIoUOHWL58eYG9P3lxcXGhZcuWBfYAiYiIeRq9+gzb3ruLIfF/mB1FnIhdF0D/FD+xsbFER0cTHBxc6HMYhkFMTAxhYWElkFBERK5F6qmz1D7wJ66GlVo9OpgdR5yIqbfAUlNTOXDggG390KFDxMTEEBQURHh4OAMGDGDr1q38/PPP5OTkkJCQAEBQUBAeHh4ADBkyhMqVKzPpfwPnXnrpJdq0aUPt2rVJTk7mvffeIyYmhg8++KD0L1BERAp0YPZPNLFmczQ4nIhWjcyOI07E1AJo8+bNdO7c2bY+ZswYAIYOHcrEiRNZuPDitOhNmjTJddyKFSvo1KkTAHFxcbi4/H9H1rlz53jooYdISEggICCApk2bsnr1alq1alWyFyMiIoWWsWgRAMdadyDC5CziXCyGYRhmh7A3ycnJBAQEkJSUVOgxRyIicvWOB4cTfuYEMR/Oosmjg82OIw6uMN/fdj0GSEREyq7jm3cSfuYEmS5u1B7Yx+w44mRUAImIiCni//f09wM1G+AbHGhuGHE6DjEPkIiIlD3zKzflx54j6dC6DlFmhxGnowJIRERKXY7VYPFZV8417sHAe9uZHUeckG6BiYhIqdt1PIlz57Pw83SjcUSA2XHECakAEhGRUnf8i68YtnkhN/ln4OaqryIpfboFJiIipS5i9gwm7t3CH43Cgb5mxxEnpLJbRERKVXpSCnVitwMQetvNJqcRZ6UCSERESlXsvCV45GST6F+BKm2bmh1HnJQKIBERKVWpi5YCcKRZOywu+hoSc+iTJyIiparihjUAWLp2NTmJODMVQCIiUmrOHjlOjaOxAFQbpMdfiHlUAImISKnZu2wt2S6uHAqrQYVa1cyOI05MBZCIiJSahcH1aDLqWxZP+I/ZUcTJaR4gEREpNWsPnOK8hzfXdW5pdhRxcuoBEhGRUhF3+jzxZy7g7mqhVfUgs+OIk1MBJCIipeLEO++zcOZonjqyGl9P3YAQc+kTKCIipcJtxW80SjhAmut5s6OIqAdIRERKnmG1UnXnJgACenUzOY2ICiARESkFcWu3UCH1LOluHtTs08XsOCIqgEREpOQlLFwMwIHajfD09TE5jYgKIBERKQXuq1cBkNL2BpOTiFykAkhEREqUNTuH6rs2A1D+pu4mpxG5SAWQiIiUqP0HjrOmahOOBIVTs3dns+OIAPoZvIiIlLDfT2XzSt+n6VCnIrO8PM2OIwKoB0hERErY+oOnAWhbI9jkJCL/r8gFUHZ2NtHR0XzyySekpKQAcPz4cVJTU4stnIiIOLacrGxO/bEFDIO2NVUAif0o0i2wI0eO0LNnT+Li4sjIyKBbt274+fnx5ptvkp6ezscff1zcOUVExAH9Ff078z96hMNBlYl47YjZcURsitQDNGrUKFq0aMHZs2fx9va2bb/lllv49ddfiy2ciIg4tlM/LwEgKbIabm6uJqcR+X9F6gFau3Ytv//+Ox4eHrm2V61alWPHjhVLMBERcXzea9cAcKF9R5OTiORWpB4gq9VKTk7OZduPHj2Kn5/fNYcSERHHl52RSa292wCo0LeHyWlEcitSAdStWzemTp1qW7dYLKSmpvLiiy/Su3fv4somIiIO7OCSVZTLPE+SVzmqd2lvdhyRXIp0C2zKlCl07tyZqKgo0tPTueuuu4iNjaVChQp8++23xZ1RREQc0JlFywD4K6oFTTX+R+xMkQqg8PBwYmJimD17Nlu2bMFqtXL//fdz99135xoULSIizst33cXxPxk3dDA5icjlijwTtLe3N/feey/33ntvceYREZEyIDPbyrtN+tOmXATdbu9ndhyRyxRpDNDMmTNZtGiRbf3pp58mMDCQdu3aceTI1c/zsHr1avr06UN4eDgWi4X58+fn2m8YBhMnTiQ8PBxvb286derErl27rnjeuXPnEhUVhaenJ1FRUcybN++qM4mIyLXbfvQcKyMa8ulND1O9bVOz44hcpkgF0Ouvv2671bV+/XqmTZvGm2++SYUKFXjiiSeu+jxpaWk0btyYadOm5bn/zTff5N1332XatGls2rSJ0NBQunXrZpt5Oi/r169n0KBBDB48mO3btzN48GAGDhzIH3/8UbiLFBGRIvvn8RdtagTh4mIxOY3I5SyGYRiFPcjHx4e9e/dSpUoVxo0bx4kTJ5g1axa7du2iU6dOnDx5svBBLBbmzZtH//79gYu9P+Hh4YwePZpx48YBkJGRQUhICJMnT+bhhx/O8zyDBg0iOTmZxYsX27b17NmT8uXLX/UA7eTkZAICAkhKSsLf37/Q1yIi4uy+uPtpVmeVo8djA7mrUz2z44iTKMz3d5F6gMqVK8fp0xer+2XLltG1a1cAvLy8uHDhQlFOeZlDhw6RkJBA9+7dbds8PT3p2LEj69aty/e49evX5zoGoEePHgUek5GRQXJycq5FRESKJj0ljXvmTGXm9y/S3k3PhxT7VOR5gB544AEeeOAB9u/fz0033QTArl27qFq1arEES0hIACAkJCTX9pCQENu+/I4r7DGTJk0iICDAtkRGRl5DchER53ZwYTSeOVmcKhdElXbNzI4jkqciFUAffPABbdu25eTJk8ydO5fg4ItP+N2yZQt33XVXsQa0WHLfOzYM47Jt13rM+PHjSUpKsi3x8fFFDywi4uSSlywH4EjDllhcivQ1I1LiivQz+MDAQN5++23+/PNPEhMTWbhwIQDNmzcvtmChoaHAxR6dsLAw2/bExMTLenguPe7S3p4rHePp6Ymnp+c1JhYREYCAP34HILtjJ3ODiBSgSAXQkiVLGDJkCKdPn+bSMdQWiyXP54QVVvXq1QkNDWX58uU0bXrxJ5SZmZmsWrWKyZMn53tc27ZtWb58ea5foy1btox27dpdcyYRESnYhaRUav21E4Dw/r1MTiOSvyL1TY4YMYLbb7+d48ePY7Vacy2FKX5SU1OJiYkhJiYGuDjwOSYmhri4OCwWC6NHj+b1119n3rx57Ny5k2HDhuHj45PrNtuQIUMYP368bX3UqFEsW7aMyZMns3fvXiZPnkx0dDSjR48uyqWKiEghHPwpGo+cbBL9golo2dDsOCL5KlIPUGJiImPGjCnwttLV2Lx5M507d7atjxkzBoChQ4cyY8YMnn76aS5cuMBjjz3G2bNnad26NcuWLcv1xPm4uDhc/nWPuV27dsyePZvnn3+eCRMmULNmTebMmUPr1q2vKauIiFzZuehVAMQ1akUljf8RO1akeYDuu+8+2rdvz/33318SmUyneYBERIpm4EfrOLNlO090rc1Nd3Q1O444mcJ8fxepB2jatGncfvvtrFmzhoYNG+Lu7p5r/+OPP16U04qIiAO7kJlDzNEkMitUIaprW7PjiBSoSAXQN998w9KlS/H29mblypW5fmJusVhUAImIOKFtcWfJzLES4u9JtWAfs+OIFKhIBdDzzz/Pyy+/zDPPPJNr/I2IiDivjLff5b3VvxN/651YLLr9JfatSNVLZmYmgwYNUvEjIiI2Ib8tpu+e1bS0JpkdReSKilTBDB06lDlz5hR3FhERcVD/nv8nrF9Pk9OIXFmRboHl5OTw5ptvsnTpUho1anTZIOh33323WMKJiIhjOPhTNA00/484kCIVQDt27LDNzrxz585c+670nC4RESl7Uv73/C/N/yOOokgF0IoVK4o7h4iIOLCAjesAyOnY0eQkIldHZbqIiFyTXM//6qfnf4ljKFIPkIiIyD92b9mLW8XqVEpPIqJFA7PjiFwVFUAiInJNVmX78d7QKdxavwLvavyPOAh9UkVE5Jps+OsMAC3rhpmcROTqqQASEZEiu5CWzr6DCQC0qRFschqRq6cCSEREiuzgvCVsfvd2vvx5sp7/JQ5FBZCIiBRZypLluFtz8A8sp3ngxKGoABIRkSLT/D/iqFQAiYhIkWj+H3FkKoBERKRIDv4UjUdONon+FTT/jzgcFUAiIlIktud/NWyJRfP/iIPRJ1ZERIrkn/E/Vo3/EQekmaBFRKTQLmTm8G3N62ln8aFh/95mxxEpNBVAIiJSaNvizvLfxj1ZfkN/1mv8jzgg3QITEZFC2/DXaQDa1AjS/D/ikFQAiYhIoWXNX0iVsydoUz3I7CgiRaJbYCIiUigXzqXwxKfPMi4nm6MP/Gl2HJEiUQ+QiIgUyr/n/6ncvL7ZcUSKRAWQiIgUSuqSaEDz/4hj0ydXREQKxX/j74Dm/xHHpgJIRESu2oVzKdQ6tAvQ87/EsakAEhGRq5Zr/I/m/xEHpgJIRESumsb/SFmhn8GLiMhV+6RZX75ID2Jg72ZmRxG5JirfRUTkqpzPzGbtmRyW125D7X7dzY4jck1UAImIyFXZfPgsWTkGlQO9qRrsY3YckWuiAkhERK5K2rQPeWLNV/T3PKfnf4nDs/sCqFq1algslsuW4cOH59l+5cqVebbfu3dvKScXESlbai2Yzah1s+mcdtTsKCLXzO4HQW/atImcnBzb+s6dO+nWrRu33357gcft27cPf39/23rFihVLLKOISFmXdCyRGvH7Aahy200mpxG5dnZfAF1auLzxxhvUrFmTjleYgbRSpUoEBgaWYDIREefx14+/0NSwElcxkirX1TQ7jsg1s/tbYP+WmZnJV199xX333XfF+89NmzYlLCyMLl26sGLFigLbZmRkkJycnGsREZH/l7Hs4vw/x5u3MzmJSPFwqAJo/vz5nDt3jmHDhuXbJiwsjE8//ZS5c+fy448/UrduXbp06cLq1avzPWbSpEkEBATYlsjIyBJILyLiuEI2rwPAo1tXk5OIFA+LYRiG2SGuVo8ePfDw8OCnn34q1HF9+vTBYrGwcOHCPPdnZGSQkZFhW09OTiYyMpKkpKRc44hERJzRqYNxVKhVFYBzR44TWCXM5EQieUtOTiYgIOCqvr/tfgzQP44cOUJ0dDQ//vhjoY9t06YNX331Vb77PT098fT0vJZ4IiJl1t7VW2jo6cupCuHUVPEjZYTD3AKbPn06lSpV4qabCv/rg23bthEWpn+0IiJF8bN/TZo+/g0/vfap2VFEio1D9ABZrVamT5/O0KFDcXPLHXn8+PEcO3aMWbNmATB16lSqVatG/fr1bYOm586dy9y5c82ILiLi8NYdPI3VxZVGra8zO4pIsXGIAig6Opq4uDjuu+++y/adOHGCuLg423pmZiZPPfUUx44dw9vbm/r167No0SJ69+5dmpFFRMqE+JMpxJ1Ow9XVhVbVg82OI1JsHGoQdGkpzCAqEZGybOML7xD+n8ms7HQr9yz42Ow4IgUqzPe3w4wBEhGR0ufy269EJJ+kpo+e/SVliwogERHJk2G1UvXPPwDwu6mHyWlEipcKIBERyVPc+m1UTDlDhqs7tfp1MzuOSLFSASQiInlKmPszALG1G+Hl52tyGpHipQJIRETy5LniVwBSbuhschKR4qcCSERELpOVnkGt3ZsBqHjrzSanESl+DjEPkIiIlK4/9x1ne8NuND15kMZdrzc7jkixUwEkIiKXWfV3Ju91fYibG4Uxzc3V7DgixU63wERE5DKrY08B0KF2RZOTiJQMFUAiIpJLUsJpfH5fjUd2FtfXrmB2HJESoVtgIiKSy1+z5/PNt88SW7k24W/3NzuOSIlQD5CIiOSSuWQpAKebtjI5iUjJUQEkIiK5RGxaC4BXbz3+QsouFUAiImJzbPMOKp85QZaLK7Vu1/w/UnapABIREZuj3y0EILZmQ8pVKG9yGpGSowJIRERs3P/3+Iuk9p3MDSJSwlQAiYgIANnpGdTasRGAoP69TU4jUrL0M3gREQFg6/FUXrxrEt2P/snjvfUAVCnbVACJiAgAK/afZE+lGtTpfj2u7vp6kLJNt8BERASAFXsTAehct5LJSURKngogEREhcc9BHvp8IjfvXU2HOnr+l5R96uMUEREOf/UDt+5aQf20vwnynWx2HJESpx4gERHBfekSAM506GpyEpHSoQJIRMTJZZ5Pp86OPwCoMLC/uWFESokKIBERJ7f/xyX4Zl7gtG8gNXt0MDuOSKlQASQi4uRS5118/MVfza/Hxc3V5DQipUMFkIiIkwv9fQUALjdp9mdxHiqARESc2LH4RM4bFrItLtS6+zaz44iUGv0MXkTEia04doHn732fzsEWplfWBIjiPNQDJCLixH7d8zcALVrUMTmJSOlSASQi4qRSk9OI2RUPQPeoEJPTiJQuFUAiIk5q/2ffsGHqHfxn1SfUqlTO7DgipUoFkIiIk8qZPx/PnGwqhVfAYrGYHUekVKkAEhFxQlnpGdTdvBqAwDsGmJxGpPSpABIRcUL7vv8F//RUzvgGUKd/d7PjiJQ6uy6AJk6ciMViybWEhoYWeMyqVato3rw5Xl5e1KhRg48//riU0oqIOI60OT8AcKBVZ1zdNSOKOB+7/9TXr1+f6Oho27qra/7TtB86dIjevXvz4IMP8tVXX/H777/z2GOPUbFiRW67TRN8iYgAGFYrVX+/+L+rHrfeYnIaEXPYfQHk5uZ2xV6ff3z88cdUqVKFqVOnAnDdddexefNm3n77bRVAIiL/81f0WmqeS+SCmyd171EBJM7Jrm+BAcTGxhIeHk716tW54447+Ouvv/Jtu379erp3z30vu0ePHmzevJmsrKx8j8vIyCA5OTnXIiJSVi1P8mBil4eIvmkw3oF+ZscRMYVdF0CtW7dm1qxZLF26lM8++4yEhATatWvH6dOn82yfkJBASEjuybxCQkLIzs7m1KlT+b7OpEmTCAgIsC2RkZHFeh0iIvZk4fEsZrToS/rzL5gdRcQ0dl0A9erVi9tuu42GDRvStWtXFi1aBMDMmTPzPebSuSwMw8hz+7+NHz+epKQk2xIfH18M6UVE7E/8mfPsPpGMiwW6XKfZn8V52f0YoH/z9fWlYcOGxMbG5rk/NDSUhISEXNsSExNxc3MjODg43/N6enri6elZrFlFROzR3g+mc0fMXk53702Qr4fZcURMY9c9QJfKyMhgz549hIWF5bm/bdu2LF++PNe2ZcuW0aJFC9zd3UsjooiIXav6+Qe8sXQaDyRsMTuKiKnsugB66qmnWLVqFYcOHeKPP/5gwIABJCcnM3ToUODirashQ4bY2j/yyCMcOXKEMWPGsGfPHr788ku++OILnnrqKbMuQUTEbhyP2UOduD3kWFyo+fCQKx8gUobZ9S2wo0ePcuedd3Lq1CkqVqxImzZt2LBhA1WrVgXgxIkTxMXF2dpXr16dX375hSeeeIIPPviA8PBw3nvvPf0EXkQEOPLJTMKBPXWa0qBmFbPjiJjKYvwzSlhskpOTCQgIICkpCX9/f7PjiIgUi9iq11E7bi9/jH2V1m8+Z3YckWJXmO9vu74FJiIixePYll3UjttLjsWFWo/q9peICiAREScQ9+nF6UP21GtGcHXNdSaiAkhExAkk7jtEjsWFtL569IUI2PkgaBERuXYHElMY1WYYkxr145fHe5gdR8QuqAdIRKSMm7ftGAD1m9UlKLyiyWlE7IMKIBGRMsyancPvK7cDcEuzyianEbEfugUmIlKG7f3hF358405W1G1L+1dWmx1HxG6oB0hEpAxL/WImLhj4hVfCy93V7DgidkMFkIhIGZWekka9NUsA8LlvqMlpROyLCiARkTJq92ff4p+Rxt8BFYm6o4/ZcUTsigogEZEyynXWDAD+6t4PFzfd/hL5NxVAIiJl0N+7Y2nw53oAIsY8ZnIaEfujAkhEpAw6+PZHuBpWdtduQmSbpmbHEbE7+hm8iEgZY7UaTKjWhWa9rPS/uaXZcUTsknqARETKmLUHTnHwPCxt1YvmD95hdhwRu6QCSESkjJm9KQ6AW5pW1tw/IvlQASQiUoacPnSU4U8O4r5NC7ijRYTZcUTslsYAiYiUIbGvTaHN3wdx9/KgTuVAs+OI2C31AImIlBFZ6RnU+H4WAEn3P2xyGhH7pgJIRKSM+PO9L6mUfIpT5crTaMyDZscRsWsqgEREygjfTz4CILb/3Xj6+picRsS+qQASESkDYhetoN5fO8h0caP2i0+aHUfE7qkAEhEpA85NfgeAP9v1oEKtauaGEXEA+hWYiIiDO5F0gSlVOzDk7zPUGDfG7DgiDkEFkIiIg/ts9SHWVa5P9pgb+O7mtmbHEXEIugUmIuLATqdm8O3GizM/j+hcy+Q0Io5DBZCIiAPbPepZnlryMR19M7mhdgWz44g4DN0CExFxUCmJp2n8zafckJ5Ks7v7YLFYzI4k4jDUAyQi4qB2TpiMf3oqRypVofGIYWbHEXEoKoBERBxQ8t+nqPfVJwAkPvYELm566rtIYagAEhFxQLvHvED588kcCalK02ceMzuOiMNRASQi4mBOHYyj0fdfAnB6/Au4eXqYnEjE8agAEhFxMLFPPI9PVgb7q15H05HDzI4j4pD0KzAREQcSf+Y8o2v05OHmSbQafS8WF/13rEhR6F+OiIgDeW3RHv72CuC3R56lwT39zY4j4rBUAImIOIj1G/awZOcJXF0sTLg5yuw4Ig7NrgugSZMm0bJlS/z8/KhUqRL9+/dn3759BR6zcuVKLBbLZcvevXtLKbWISPHLSDtPRN8ezPruBUbU9qJuqJ/ZkUQcml2PAVq1ahXDhw+nZcuWZGdn89xzz9G9e3d2796Nr69vgcfu27cPf39/23rFihVLOq6ISInZOmYibU/G430hlSa9G5kdR8Th2XUBtGTJklzr06dPp1KlSmzZsoUOHToUeGylSpUIDAy8qtfJyMggIyPDtp6cnFzorCIiJeV4zB4aT38fgENPv0jLED3zS+Ra2fUtsEslJSUBEBQUdMW2TZs2JSwsjC5durBixYoC206aNImAgADbEhkZWSx5RUSulTU7h7OD7sEnK53dtZvQ4rmRZkcSKRMshmEYZoe4GoZh0K9fP86ePcuaNWvybbdv3z5Wr15N8+bNycjI4L///S8ff/wxK1euzLfXKK8eoMjISJKSknLdRhMRKW1/jHmJ1lMmct7dk7PrNlG5RUOzI4nYreTkZAICAq7q+9thCqDhw4ezaNEi1q5dS0RERKGO7dPn4lOSFy5ceFXtC/MGioiUlKMb/yTo+lb4ZGXwx5Mv0frtF8yOJGLXCvP97RC3wEaOHMnChQtZsWJFoYsfgDZt2hAbG1sCyURESkZWjpV352/luF9FdtZtTss3njM7kkiZYtcFkGEYjBgxgh9//JHffvuN6tWrF+k827ZtIywsrJjTiYiUnDcW7+VHa0XuePgDys//Xk97Fylmdv0rsOHDh/PNN9+wYMEC/Pz8SEhIACAgIABvb28Axo8fz7Fjx5g1axYAU6dOpVq1atSvX5/MzEy++uor5s6dy9y5c027DhGRwli+bh9frD0EwGt3t6JyvVCTE4mUPXZdAH300UcAdOrUKdf26dOnM2zYMABOnDhBXFycbV9mZiZPPfUUx44dw9vbm/r167No0SJ69+5dWrFFRIosfsM2WnbtyANtbsf1ySfpUV/Fj0hJcJhB0KVJg6BFxAxnDh3lfMs2RJw+xu5ajamzcyNunh5mxxJxGGVuELSISFl34VwKJ7v0JOL0MY6XD6XS4vkqfkRKkAogERGT5WRls7dLX+oe2kWSVzmyfvqZCrWqmR1LpExTASQiYqLsjEy2depD060ryXB15/isOVRt39zsWCJlngogERGTZOVY+WL8NFqsW0KWiyu73/2E627XDzZESoNd/wpMRKSsSs/KYdTsbSz1qEtyp6F0u60zTUcMNTuWiNNQASQiUspOHYzjye//ZNU5Cx6uLjT7cDJNrgsxO5aIU9EtMBGRUvTXr+vIatGKx98fSyV3g1n3t6KLih+RUqcCSESkFBhWK388/SrhPTsTdu5vKmWm8sNttWhTI9jsaCJOSbfARERK2JlDRzk8cAitN68AYHvDtlT9eS6BVfSMQhGzqAdIRKSEWLNz2Pj8m7hGXUezzSvIdHFjw/Bnabh1jYofEZOpABIRKQFbjpxh0CfrcZv+JQHpqRwMr8WRn6JpM+01PdldxA7oFpiISDE6sHQ17x/IYEF8JgAv3TyKZ93iaf7uRD3aQsSOqAASEblGOVnZ7Pj8W9z/8x/q79tCVKtb+bnL/QxsEcHjXW4kLMDb7IgicgkVQCIiRRT/x3aOTv2Y6ovn0iTpJADZFhca+VpZPvoGalTyMzmhiORHBZCIyFWyWg3+PJbE8t0JdHjiXlrv2UDk//Yle5Vjd+/bqfbSeNo2qG1qThG5MhVAIiL5yEg7z5EVGziz9DfcN6znkV5PcDLTAkAFn4o0t7iwu34rsgYPof6jQ2jj52tyYhG5WiqARMTppWflcPTsBU5v3IaxaBFuO/4k+OBeIv8+Qh3DamtXtW4PztdoRKe6lQi9cSJp9T+nUeVK5gUXkSJTASQiZVJ2jpXUjGxSTpwkI/YAFxISST9yjJwTJ+DECdxPJuJ9MoHJPR9hdblIDAOGbPmJl6M/yXWec95+HK7TmPS27Zk4tBd1W9bH3VUziIg4OhVApSgjO4eTp1NwTThx2T7DuPj/rf7+GIHlL65kZeF6/Fi+5zP8/Mgp/79p9LOzcYuP+/99GLnaWsuVw1rx4n+pGjlW3A4fuvzFAQMwfMuRExpq2+f214HL2tnO6+NLdli4bd39wP7c5/vXIYa3D9kRkf/f9uB+yPn//7rOdV4PT7KrVvtX21gsWdlcngAMTw+yqte0vZb7wVgsmZmXtrr4f909yKz1/+Mz3P86iOXChbwzuLqRWaeebd3jr4NYzqflOp/t7C4uZEQ1uPhnAzwOH8IlNflfDXK3T2/Y5P/Pe/gQlqSkPDMAXGjQEFwufuF6HDmM69kzl1zVv9rWbwRuF/9Zu8fH4Xb65GXnM/6X5cJ1DTA8PS+2PRqPe2JCnu8vwIWoBlg9L/6Syf3YUTwS8v9cXqgXhdW3HABux4/hHheHNSsLIzsHIycbIysLI9uKNSebU01bkx5QnhyrFd/YfZT/cyvWnBzIzsbIzoacHMjJwcjMZNcNvUisEE5GVg6Vt66n2a/zcMlIxzUjHbfMTNwy0/G8kIbP+VSe6fskv1VuCMDA7ct4c8l7+eYNPHwAo34kvh6upEU1YmtyVzIaNMS7RVPCOrahUr2aNHFRwSNS1qgAKkW7jicz/qVvWPrliHzbfNR6AJM7DQOgytkTrP70wXzbzmh2MxO7PQJAxdSzbPpgcL5t5zTsxrjeowDwzTjPrqkD8237U70bGNlvHAAu1hz+eqtfvm2ja7bkgQEv2tb3vX0LnjlZebb9vWoj7r7jddv6tv/cSfn0lDzbbguryy1D3rGtr/twGOEpp/Jsu7dCVXre/4Ft/dfPHqHGmaN5tj0SGErPhz+3rf80YxQN/z6YZ9uTvoG0HPGVbf27r5+m1dHdebZN9fCmwRPf29ZnfvcCHQ9tzbNtjsWFmk8vtK1/NO91eu1fl2dbgLpP/kiG28X5Y979+R1u3bUi37ZNHv+Gc97+ALy2dBp3xyzJt23bR6dzwr8iAM/99jkPbpqfb9suD3zEweCLxeuY1f/l8fVz8m1789Cp7AytBcCjG75n3KqZ+ba9/a432BR5sXC8d/MCXvz1s3zbfpTkz+oazQEYtH0HD/2+ON+23snnoPLFP5/3DyTRL5g0X39SgyqSHlyR7JBQLKGhuEVU5oGO1/NCgzoE+XpgsViAx/I9r4iUHSqASpGLxYKHmwsX3Dzzb+Tujrf7xVliPdzdSHP3AsCSV1sPD3w8XLEA3h6upHnknmvE+NdRVk9Pynle/Ov2wY0UT598I+R4eeHndbGti9VCsle5fNtmefng7+X2vy8OSPEqR/qlBdD/YmR6+RDo427bnOLjBy55XhkXfMoR5Pv/k8al+Ppzxpp3YZXm60/w/9paLJDq68/pjMA826b6BlCh3P+//2nlAjiVFpRn23O+AVTy+/+2F/wCOemXu+0/73G6hxch/v/fNt0/kET/Cnme12qxEBbgZVvPCAgkIaBinm0Bwvy9yPK4eO7sgPIkBOY/5iQ00Adf74ufg+zAII6XD823bUiQD27+F9saQUEcC8r/0QwVg8qRFeSDxQIuwUEcDQ6/rM0/f5MVg/2oFnzx8+UWHERcxUisLq4YLi7kuLpiuLhidXHB6upG9WohuFcLxtXFgv/5umw/1R6rqxuGiwuGqxuGqyu4uGL18KB1+/rUq10DTzcXwutY2FDLH4u3FxZvH1x8vHH19sI9MADPCsGMu642r1SqgJ+XG+6uNwEv5XttIuKcLIaRx30NJ5ecnExAQABJSUn4+/ubHUdERESuQmG+v3VjW0RERJyOCiARERFxOiqARERExOmoABIRERGnowJIREREnI4KIBEREXE6KoBERETE6agAEhEREaejAkhEREScjgogERERcToqgERERMTpqAASERERp6MCSERERJyOCiARERFxOm5mB7BHhmEAkJycbHISERERuVr/fG//8z1eEBVAeUhJSQEgMjLS5CQiIiJSWCkpKQQEBBTYxmJcTZnkZKxWK8ePH8fPzw+LxVKs505OTiYyMpL4+Hj8/f2L9dz2oKxfH5T9a9T1Ob6yfo26PsdXUtdoGAYpKSmEh4fj4lLwKB/1AOXBxcWFiIiIEn0Nf3//MvvBhrJ/fVD2r1HX5/jK+jXq+hxfSVzjlXp+/qFB0CIiIuJ0VACJiIiI01EBVMo8PT158cUX8fT0NDtKiSjr1wdl/xp1fY6vrF+jrs/x2cM1ahC0iIiIOB31AImIiIjTUQEkIiIiTkcFkIiIiDgdFUAiIiLidFQAmahv375UqVIFLy8vwsLCGDx4MMePHzc7VrE4fPgw999/P9WrV8fb25uaNWvy4osvkpmZaXa0YvXaa6/Rrl07fHx8CAwMNDvONfvwww+pXr06Xl5eNG/enDVr1pgdqdisXr2aPn36EB4ejsViYf78+WZHKlaTJk2iZcuW+Pn5UalSJfr378++ffvMjlWsPvroIxo1amSbPK9t27YsXrzY7FglZtKkSVgsFkaPHm12lGIxceJELBZLriU0NNS0PCqATNS5c2e+++479u3bx9y5czl48CADBgwwO1ax2Lt3L1arlU8++YRdu3YxZcoUPv74Y5599lmzoxWrzMxMbr/9dh599FGzo1yzOXPmMHr0aJ577jm2bdvGDTfcQK9evYiLizM7WrFIS0ujcePGTJs2zewoJWLVqlUMHz6cDRs2sHz5crKzs+nevTtpaWlmRys2ERERvPHGG2zevJnNmzdz44030q9fP3bt2mV2tGK3adMmPv30Uxo1amR2lGJVv359Tpw4YVt27NhhXhhD7MaCBQsMi8ViZGZmmh2lRLz55ptG9erVzY5RIqZPn24EBASYHeOatGrVynjkkUdybatXr57xzDPPmJSo5ADGvHnzzI5RohITEw3AWLVqldlRSlT58uWNzz//3OwYxSolJcWoXbu2sXz5cqNjx47GqFGjzI5ULF588UWjcePGZsewUQ+QnThz5gxff/017dq1w93d3ew4JSIpKYmgoCCzY0geMjMz2bJlC927d8+1vXv37qxbt86kVHItkpKSAMrsv7mcnBxmz55NWloabdu2NTtOsRo+fDg33XQTXbt2NTtKsYuNjSU8PJzq1atzxx138Ndff5mWRQWQycaNG4evry/BwcHExcWxYMECsyOViIMHD/L+++/zyCOPmB1F8nDq1ClycnIICQnJtT0kJISEhASTUklRGYbBmDFjuP7662nQoIHZcYrVjh07KFeuHJ6enjzyyCPMmzePqKgos2MVm9mzZ7N161YmTZpkdpRi17p1a2bNmsXSpUv57LPPSEhIoF27dpw+fdqUPCqAilleg7wuXTZv3mxrP3bsWLZt28ayZctwdXVlyJAhGHY8OXdhrw/g+PHj9OzZk9tvv50HHnjApORXryjXWFZYLJZc64ZhXLZN7N+IESP4888/+fbbb82OUuzq1q1LTEwMGzZs4NFHH2Xo0KHs3r3b7FjFIj4+nlGjRvHVV1/h5eVldpxi16tXL2677TYaNmxI165dWbRoEQAzZ840JY+bKa9aho0YMYI77rijwDbVqlWz/blChQpUqFCBOnXqcN111xEZGcmGDRvstku3sNd3/PhxOnfuTNu2bfn0009LOF3xKOw1lgUVKlTA1dX1st6exMTEy3qFxL6NHDmShQsXsnr1aiIiIsyOU+w8PDyoVasWAC1atGDTpk385z//4ZNPPjE52bXbsmULiYmJNG/e3LYtJyeH1atXM23aNDIyMnB1dTUxYfHy9fWlYcOGxMbGmvL6KoCK2T8FTVH80/OTkZFRnJGKVWGu79ixY3Tu3JnmzZszffp0XFwco8PxWv4OHZWHhwfNmzdn+fLl3HLLLbbty5cvp1+/fiYmk6tlGAYjR45k3rx5rFy5kurVq5sdqVQYhmHX/5tZGF26dLnsV1H33nsv9erVY9y4cWWq+IGL33V79uzhhhtuMOX1VQCZZOPGjWzcuJHrr7+e8uXL89dff/HCCy9Qs2ZNu+39KYzjx4/TqVMnqlSpwttvv83Jkydt+8yc96G4xcXFcebMGeLi4sjJySEmJgaAWrVqUa5cOXPDFdKYMWMYPHgwLVq0sPXYxcXFlZlxW6mpqRw4cMC2fujQIWJiYggKCqJKlSomJisew4cP55tvvmHBggX4+fnZevMCAgLw9vY2OV3xePbZZ+nVqxeRkZGkpKQwe/ZsVq5cyZIlS8yOViz8/PwuG7P1zxjRsjCW66mnnqJPnz5UqVKFxMREXn31VZKTkxk6dKg5gcz8CZoz+/PPP43OnTsbQUFBhqenp1GtWjXjkUceMY4ePWp2tGIxffp0A8hzKUuGDh2a5zWuWLHC7GhF8sEHHxhVq1Y1PDw8jGbNmpWpn1CvWLEiz7+roUOHmh2tWOT372369OlmRys29913n+3zWbFiRaNLly7GsmXLzI5VosrSz+AHDRpkhIWFGe7u7kZ4eLhx6623Grt27TItj8Uw7HjErYiIiEgJcIxBGSIiIiLFSAWQiIiIOB0VQCIiIuJ0VACJiIiI01EBJCIiIk5HBZCIiIg4HRVAIiIi4nRUAImIiIjTUQEkIletU6dOjB492uwYeTp9+jSVKlXi8OHDAKxcuRKLxcK5c+dK9HWL+jozZswgMDCwUMe0bNmSH3/8sVDHiEjeVACJiGlOnDjBXXfdRd26dXFxccm3uJo7dy5RUVF4enoSFRXFvHnzLmszadIk+vTpQ7Vq1Uo2tIkmTJjAM888g9VqNTuKiMNTASQipsnIyKBixYo899xzNG7cOM8269evZ9CgQQwePJjt27czePBgBg4cyB9//GFrc+HCBb744gseeOCB0opuiptuuomkpCSWLl1qdhQRh6cCSESK5OzZswwZMoTy5cvj4+NDr169iI2NzdXms88+IzIyEh8fH2655RbefffdXLd9qlWrxn/+8x+GDBlCQEBAnq8zdepUunXrxvjx46lXrx7jx4+nS5cuTJ061dZm8eLFuLm50bZt23zznj59mjvvvJOIiAh8fHxo2LAh3377ba42nTp1YuTIkYwePZry5csTEhLCp59+SlpaGvfeey9+fn7UrFmTxYsXX3b+33//ncaNG+Pl5UXr1q3ZsWNHrv0zZsygSpUqtvfi9OnTufYfPHiQfv36ERISQrly5WjZsiXR0dG52ri6utK7d+/LcotI4akAEpEiGTZsGJs3b2bhwoWsX78ewzDo3bs3WVlZwMWC4JFHHmHUqFHExMTQrVs3XnvttUK/zvr16+nevXuubT169GDdunW29dWrV9OiRYsCz5Oenk7z5s35+eef2blzJw899BCDBw/O1ZMEMHPmTCpUqMDGjRsZOXIkjz76KLfffjvt2rVj69at9OjRg8GDB3P+/Plcx40dO5a3336bTZs2UalSJfr27Wt7L/744w/uu+8+HnvsMWJiYujcuTOvvvpqruNTU1Pp3bs30dHRbNu2jR49etCnTx/i4uJytWvVqhVr1qy5ujdPRPJn2nPoRcThdOzY0Rg1apSxf/9+AzB+//13275Tp04Z3t7exnfffWcYhmEMGjTIuOmmm3Idf/fddxsBAQEFnvtS7u7uxtdff51r29dff214eHjY1vv162fcd999udqsWLHCAIyzZ8/mez29e/c2nnzyyVwZrr/+ett6dna24evrawwePNi27cSJEwZgrF+/PtfrzJ4929bm9OnThre3tzFnzhzDMAzjzjvvNHr27JnrtQcNGpTve/GPqKgo4/3338+1bcGCBYaLi4uRk5NT4LEiUjD1AIlIoe3Zswc3Nzdat25t2xYcHEzdunXZs2cPAPv27aNVq1a5jrt0/WpZLJZc64Zh5Np24cIFvLy8CjxHTk4Or732Go0aNSI4OJhy5cqxbNmyy3pYGjVqZPuzq6srwcHBNGzY0LYtJCQEgMTExFzH/fv2W1BQUK73Ys+ePZfdnrt0PS0tjaeffpqoqCgCAwMpV64ce/fuvSyft7c3VquVjIyMAq9XRArmZnYAEXE8hmHku/2fwuTSIqWg4woSGhpKQkJCrm2JiYm2QgSgQoUKnD17tsDzvPPOO0yZMoWpU6fSsGFDfH19GT16NJmZmbnaubu751q3WCy5tv1zTVfzS6x/vxdXMnbsWJYuXcrbb79NrVq18Pb2ZsCAAZflO3PmDD4+Pnh7e1/xnCKSP/UAiUihRUVFkZ2dnWv8zOnTp9m/fz/XXXcdAPXq1WPjxo25jtu8eXOhX6tt27YsX74817Zly5bRrl0723rTpk3ZvXt3gedZs2YN/fr145577qFx48bUqFHjskHb12LDhg22P589e5b9+/dTr1494OL79e/9l7b/J9+wYcO45ZZbaNiwIaGhobY5jf5t586dNGvWrNhyizgrFUAiUmi1a9emX79+PPjgg6xdu5bt27dzzz33ULlyZfr16wfAyJEj+eWXX3j33XeJjY3lk08+YfHixZf1CsXExBATE0NqaionT54kJiYmVzEzatQoli1bxuTJk9m7dy+TJ08mOjo615xBPXr0YNeuXQX2AtWqVYvly5ezbt069uzZw8MPP3xZz9K1ePnll/n111/ZuXMnw4YNo0KFCvTv3x+Axx9/nCVLlvDmm2+yf/9+pk2bxpIlSy7L9+OPPxITE8P27du566678uxlWrNmzWWDwkWk8FQAiUiRTJ8+nebNm3PzzTfTtm1bDMPgl19+sd0uat++PR9//DHvvvsujRs3ZsmSJTzxxBOXjdVp2rQpTZs2ZcuWLXzzzTc0bdqU3r172/a3a9eO2bNnM336dBo1asSMGTOYM2dOrvFHDRs2pEWLFnz33Xf55p0wYQLNmjWjR48edOrUidDQUFuBUhzeeOMNRo0aRfPmzTlx4gQLFy7Ew8MDgDZt2vD555/z/vvv06RJE5YtW8bzzz+f6/gpU6ZQvnx52rVrR58+fejRo8dlPT3Hjh1j3bp13HvvvcWWW8RZWYyi3JQXESmCBx98kL1795bIz7h/+eUXnnrqKXbu3ImLS9n8b7uxY8eSlJTEp59+anYUEYenQdAiUmLefvttunXrhq+vL4sXL2bmzJl8+OGHJfJavXv3JjY2lmPHjhEZGVkir2G2SpUq8dRTT5kdQ6RMUA+QiJSYgQMHsnLlSlJSUqhRowYjR47kkUceMTuWiIgKIBEREXE+ZfNGuYiIiEgBVACJiIiI01EBJCIiIk5HBZCIiIg4HRVAIiIi4nRUAImIiIjTUQEkIiIiTkcFkIiIiDid/wMJyAelV+9DGgAAAABJRU5ErkJggg==\n",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"%matplotlib inline\n",
"\n",
@@ -218,14 +206,14 @@
"## Cross-validation on Ridge regression using KFold only\n",
"\n",
"# Decide degree on polynomial to fit\n",
- "poly = PolynomialFeatures(degree = 6)\n",
+ "poly = PolynomialFeatures(degree = 2)\n",
"\n",
"# Decide which values of lambda to use\n",
"nlambdas = 500\n",
"lambdas = np.logspace(-3, 5, nlambdas)\n",
"\n",
"# Initialize a KFold instance\n",
- "k = 5\n",
+ "k = 10\n",
"kfold = KFold(n_splits = k)\n",
"\n",
"# Perform the cross-validation to estimate MSE\n",
@@ -293,9 +281,7 @@
{
"cell_type": "markdown",
"id": "3d38a274",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Material for lecture Thursday September 21"
]
@@ -303,9 +289,7 @@
{
"cell_type": "markdown",
"id": "53cb96b0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Logistic Regression\n",
"\n",
@@ -325,9 +309,7 @@
{
"cell_type": "markdown",
"id": "6120b1c7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Classification problems\n",
"\n",
@@ -351,9 +333,7 @@
{
"cell_type": "markdown",
"id": "936083c0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Optimization and Deep learning\n",
"\n",
@@ -375,9 +355,7 @@
{
"cell_type": "markdown",
"id": "32780c62",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Basics\n",
"\n",
@@ -400,9 +378,7 @@
{
"cell_type": "markdown",
"id": "20d08abd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"y_i = \\begin{bmatrix} 0 & \\mathrm{no}\\\\ 1 & \\mathrm{yes} \\end{bmatrix}.\n",
@@ -412,9 +388,7 @@
{
"cell_type": "markdown",
"id": "58b3452f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Linear classifier\n",
"\n",
@@ -430,9 +404,7 @@
{
"cell_type": "markdown",
"id": "7f605f89",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"\n",
"
\n",
@@ -448,9 +420,7 @@
{
"cell_type": "markdown",
"id": "40905342",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $\\boldsymbol{y}$ is a vector representing the possible outcomes, $\\boldsymbol{X}$ is our\n",
"$n\\times p$ design matrix and $\\boldsymbol{\\beta}$ represents our estimators/predictors."
@@ -459,9 +429,7 @@
{
"cell_type": "markdown",
"id": "77f09c4f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Some selected properties\n",
"\n",
@@ -486,9 +454,7 @@
{
"cell_type": "markdown",
"id": "4eea9d16",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Simple example\n",
"\n",
@@ -499,10 +465,7 @@
"cell_type": "code",
"execution_count": 2,
"id": "fdccf16e",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Common imports\n",
@@ -564,9 +527,7 @@
{
"cell_type": "markdown",
"id": "3db91831",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Plotting the mean value for each group\n",
"\n",
@@ -577,10 +538,7 @@
"cell_type": "code",
"execution_count": 3,
"id": "b570a592",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])\n",
@@ -596,9 +554,7 @@
{
"cell_type": "markdown",
"id": "45e5a12a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We are now trying to find a function $f(y\\vert x)$, that is a function which gives us an expected value for the output $y$ with a given input $x$.\n",
"In standard linear regression with a linear dependence on $x$, we would write this in terms of our model"
@@ -607,9 +563,7 @@
{
"cell_type": "markdown",
"id": "21d22b13",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"f(y_i\\vert x_i)=\\beta_0+\\beta_1 x_i.\n",
@@ -619,9 +573,7 @@
{
"cell_type": "markdown",
"id": "e6b2b13a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This expression implies however that $f(y_i\\vert x_i)$ could take any\n",
"value from minus infinity to plus infinity. If we however let\n",
@@ -638,9 +590,7 @@
{
"cell_type": "markdown",
"id": "038a694a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The logistic function\n",
"\n",
@@ -660,9 +610,7 @@
{
"cell_type": "markdown",
"id": "581a37c8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(t) = \\frac{1}{1+\\mathrm \\exp{-t}}=\\frac{\\exp{t}}{1+\\mathrm \\exp{t}}.\n",
@@ -672,9 +620,7 @@
{
"cell_type": "markdown",
"id": "8e6ff605",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Note that $1-p(t)= p(-t)$."
]
@@ -682,9 +628,7 @@
{
"cell_type": "markdown",
"id": "07e8c6c7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Examples of likelihood functions used in logistic regression and nueral networks\n",
"\n",
@@ -695,10 +639,7 @@
"cell_type": "code",
"execution_count": 4,
"id": "82e381c1",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"\"\"\"The sigmoid function (or the logistic curve) is a\n",
@@ -760,9 +701,7 @@
{
"cell_type": "markdown",
"id": "cffc710d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Two parameters\n",
"\n",
@@ -772,9 +711,7 @@
{
"cell_type": "markdown",
"id": "077523d2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*}\n",
@@ -787,9 +724,7 @@
{
"cell_type": "markdown",
"id": "378c1ba1",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $\\boldsymbol{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$. \n",
"\n",
@@ -799,9 +734,7 @@
{
"cell_type": "markdown",
"id": "12641f70",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(y_i=0\\vert x_i, \\boldsymbol{\\beta}) = 1-p(y_i=1\\vert x_i, \\boldsymbol{\\beta}).\n",
@@ -811,9 +744,7 @@
{
"cell_type": "markdown",
"id": "72d4d322",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Maximum likelihood\n",
"\n",
@@ -828,9 +759,7 @@
{
"cell_type": "markdown",
"id": "218bad85",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*}\n",
@@ -842,9 +771,7 @@
{
"cell_type": "markdown",
"id": "7f1151b5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"from which we obtain the log-likelihood and our **cost/loss** function"
]
@@ -852,9 +779,7 @@
{
"cell_type": "markdown",
"id": "c31ffe4e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathcal{C}(\\boldsymbol{\\beta}) = \\sum_{i=1}^n \\left( y_i\\log{p(y_i=1|x_i,\\boldsymbol{\\beta})} + (1-y_i)\\log\\left[1-p(y_i=1|x_i,\\boldsymbol{\\beta}))\\right]\\right).\n",
@@ -864,9 +789,7 @@
{
"cell_type": "markdown",
"id": "f653cea3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The cost function rewritten\n",
"\n",
@@ -876,9 +799,7 @@
{
"cell_type": "markdown",
"id": "d117384a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathcal{C}(\\boldsymbol{\\beta}) = \\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{(1+\\exp{(\\beta_0+\\beta_1x_i)})}\\right).\n",
@@ -888,9 +809,7 @@
{
"cell_type": "markdown",
"id": "3f7763f2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\\beta$.\n",
"Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that"
@@ -899,9 +818,7 @@
{
"cell_type": "markdown",
"id": "0c5d4782",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathcal{C}(\\boldsymbol{\\beta})=-\\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{(1+\\exp{(\\beta_0+\\beta_1x_i)})}\\right).\n",
@@ -911,9 +828,7 @@
{
"cell_type": "markdown",
"id": "e47ac025",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This equation is known in statistics as the **cross entropy**. Finally, we note that just as in linear regression, \n",
"in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression."
@@ -922,9 +837,7 @@
{
"cell_type": "markdown",
"id": "0775be1d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Minimizing the cross entropy\n",
"\n",
@@ -938,9 +851,7 @@
{
"cell_type": "markdown",
"id": "2c6d8021",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\beta_0} = -\\sum_{i=1}^n \\left(y_i -\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right),\n",
@@ -950,9 +861,7 @@
{
"cell_type": "markdown",
"id": "934b3029",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and"
]
@@ -960,9 +869,7 @@
{
"cell_type": "markdown",
"id": "5736ce62",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\beta_1} = -\\sum_{i=1}^n \\left(y_ix_i -x_i\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right).\n",
@@ -972,9 +879,7 @@
{
"cell_type": "markdown",
"id": "24448336",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## A more compact expression\n",
"\n",
@@ -987,9 +892,7 @@
{
"cell_type": "markdown",
"id": "62af3134",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = -\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{p}\\right).\n",
@@ -999,9 +902,7 @@
{
"cell_type": "markdown",
"id": "e75f8b6b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"If we in addition define a diagonal matrix $\\boldsymbol{W}$ with elements \n",
"$p(y_i\\vert x_i,\\boldsymbol{\\beta})(1-p(y_i\\vert x_i,\\boldsymbol{\\beta})$, we can obtain a compact expression of the second derivative as"
@@ -1010,9 +911,7 @@
{
"cell_type": "markdown",
"id": "afbcdd5a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} = \\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X}.\n",
@@ -1022,9 +921,7 @@
{
"cell_type": "markdown",
"id": "6a4b6b7f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Extending to more predictors\n",
"\n",
@@ -1034,9 +931,7 @@
{
"cell_type": "markdown",
"id": "0487a05f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\log{ \\frac{p(\\boldsymbol{\\beta}\\boldsymbol{x})}{1-p(\\boldsymbol{\\beta}\\boldsymbol{x})}} = \\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p.\n",
@@ -1046,9 +941,7 @@
{
"cell_type": "markdown",
"id": "55d00e90",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Here we defined $\\boldsymbol{x}=[1,x_1,x_2,\\dots,x_p]$ and $\\boldsymbol{\\beta}=[\\beta_0, \\beta_1, \\dots, \\beta_p]$ leading to"
]
@@ -1056,9 +949,7 @@
{
"cell_type": "markdown",
"id": "13f16948",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(\\boldsymbol{\\beta}\\boldsymbol{x})=\\frac{ \\exp{(\\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p)}}{1+\\exp{(\\beta_0+\\beta_1x_1+\\beta_2x_2+\\dots+\\beta_px_p)}}.\n",
@@ -1068,9 +959,7 @@
{
"cell_type": "markdown",
"id": "99af8c4d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Including more classes\n",
"\n",
@@ -1082,9 +971,7 @@
{
"cell_type": "markdown",
"id": "8ba91bb6",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\log{\\frac{p(C=1\\vert x)}{p(K\\vert x)}} = \\beta_{10}+\\beta_{11}x_1,\n",
@@ -1094,9 +981,7 @@
{
"cell_type": "markdown",
"id": "f5833039",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and"
]
@@ -1104,9 +989,7 @@
{
"cell_type": "markdown",
"id": "451f2890",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\log{\\frac{p(C=2\\vert x)}{p(K\\vert x)}} = \\beta_{20}+\\beta_{21}x_1,\n",
@@ -1116,9 +999,7 @@
{
"cell_type": "markdown",
"id": "1ae365e3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and so on till the class $C=K-1$ class"
]
@@ -1126,9 +1007,7 @@
{
"cell_type": "markdown",
"id": "b3187ffb",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\log{\\frac{p(C=K-1\\vert x)}{p(K\\vert x)}} = \\beta_{(K-1)0}+\\beta_{(K-1)1}x_1,\n",
@@ -1138,9 +1017,7 @@
{
"cell_type": "markdown",
"id": "edc72487",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and the model is specified in term of $K-1$ so-called log-odds or\n",
"**logit** transformations."
@@ -1149,9 +1026,7 @@
{
"cell_type": "markdown",
"id": "1e550b7a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## More classes\n",
"\n",
@@ -1172,9 +1047,7 @@
{
"cell_type": "markdown",
"id": "dc2781ed",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(C=k\\vert \\mathbf {x} )=\\frac{\\exp{(\\beta_{k0}+\\beta_{k1}x_1)}}{1+\\sum_{l=1}^{K-1}\\exp{(\\beta_{l0}+\\beta_{l1}x_1)}}.\n",
@@ -1184,9 +1057,7 @@
{
"cell_type": "markdown",
"id": "720ee440",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"It is easy to extend to more predictors. The final class is"
]
@@ -1194,9 +1065,7 @@
{
"cell_type": "markdown",
"id": "eaa0254a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"p(C=K\\vert \\mathbf {x} )=\\frac{1}{1+\\sum_{l=1}^{K-1}\\exp{(\\beta_{l0}+\\beta_{l1}x_1)}},\n",
@@ -1206,9 +1075,7 @@
{
"cell_type": "markdown",
"id": "f6bc3445",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and they sum to one. Our earlier discussions were all specialized to\n",
"the case with two classes only. It is easy to see from the above that\n",
@@ -1223,9 +1090,7 @@
{
"cell_type": "markdown",
"id": "b25b0241",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Friday September 23"
]
@@ -1233,9 +1098,7 @@
{
"cell_type": "markdown",
"id": "380d3c17",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Searching for Optimal Regularization Parameters $\\lambda$\n",
"\n",
@@ -1252,10 +1115,7 @@
"cell_type": "code",
"execution_count": 5,
"id": "b1e69471",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
@@ -1307,9 +1167,7 @@
{
"cell_type": "markdown",
"id": "e82def1d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Here we have performed a rather data greedy calculation as function of the regularization parameter $\\lambda$. There is no resampling here. The latter can easily be added by employing the function **RidgeCV** instead of just calling the **Ridge** function. For **RidgeCV** we need to pass the array of $\\lambda$ values.\n",
"By inspecting the figure we can in turn determine which is the optimal regularization parameter.\n",
@@ -1319,9 +1177,7 @@
{
"cell_type": "markdown",
"id": "325e0956",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Grid Search\n",
"\n",
@@ -1334,10 +1190,7 @@
"cell_type": "code",
"execution_count": 6,
"id": "557868b5",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
@@ -1387,9 +1240,7 @@
{
"cell_type": "markdown",
"id": "72f0779e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"By default the grid search function includes cross validation with\n",
"five folds. The [Scikit-Learn\n",
@@ -1402,9 +1253,7 @@
{
"cell_type": "markdown",
"id": "e6f83d4a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Randomized Grid Search\n",
"\n",
@@ -1422,10 +1271,7 @@
"cell_type": "code",
"execution_count": 7,
"id": "fadead70",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
@@ -1476,9 +1322,7 @@
{
"cell_type": "markdown",
"id": "42658f49",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Wisconsin Cancer Data\n",
"\n",
@@ -1491,10 +1335,7 @@
"cell_type": "code",
"execution_count": 8,
"id": "01c1d986",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
@@ -1518,9 +1359,7 @@
{
"cell_type": "markdown",
"id": "a86570ee",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Using the correlation matrix\n",
"\n",
@@ -1532,10 +1371,7 @@
"cell_type": "code",
"execution_count": 9,
"id": "e241e87e",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
@@ -1577,9 +1413,7 @@
{
"cell_type": "markdown",
"id": "31db566e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Discussing the correlation data\n",
"\n",
@@ -1602,10 +1436,7 @@
"cell_type": "code",
"execution_count": 10,
"id": "5ddb180b",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)"
@@ -1614,9 +1445,7 @@
{
"cell_type": "markdown",
"id": "95c6a55b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and then"
]
@@ -1625,10 +1454,7 @@
"cell_type": "code",
"execution_count": 11,
"id": "f3347712",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"correlation_matrix = cancerpd.corr().round(1)"
@@ -1637,9 +1463,7 @@
{
"cell_type": "markdown",
"id": "f60362a3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Diagonalizing this matrix we can in turn say something about which\n",
"features are of relevance and which are not. This leads us to\n",
@@ -1650,9 +1474,7 @@
{
"cell_type": "markdown",
"id": "232f7e0d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Other measures in classification studies: Cancer Data again"
]
@@ -1661,10 +1483,7 @@
"cell_type": "code",
"execution_count": 12,
"id": "552632a5",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
@@ -1704,9 +1523,7 @@
{
"cell_type": "markdown",
"id": "7752c5ea",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Optimization, the central part of any Machine Learning algortithm\n",
"\n",
@@ -1725,9 +1542,7 @@
{
"cell_type": "markdown",
"id": "f307c73e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Revisiting our Logistic Regression case\n",
"\n",
@@ -1742,9 +1557,7 @@
{
"cell_type": "markdown",
"id": "921fcab7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{align*}\n",
@@ -1757,9 +1570,7 @@
{
"cell_type": "markdown",
"id": "9863a96d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $\\boldsymbol{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$."
]
@@ -1767,9 +1578,7 @@
{
"cell_type": "markdown",
"id": "69a4b9a3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The equations to solve\n",
"\n",
@@ -1783,9 +1592,7 @@
{
"cell_type": "markdown",
"id": "f3f454ef",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = -\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{p}\\right).\n",
@@ -1795,9 +1602,7 @@
{
"cell_type": "markdown",
"id": "8ba81e87",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"If we in addition define a diagonal matrix $\\boldsymbol{W}$ with elements \n",
"$p(y_i\\vert x_i,\\boldsymbol{\\beta})(1-p(y_i\\vert x_i,\\boldsymbol{\\beta})$, we can obtain a compact expression of the second derivative as"
@@ -1806,9 +1611,7 @@
{
"cell_type": "markdown",
"id": "0b87735e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} = \\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X}.\n",
@@ -1818,9 +1621,7 @@
{
"cell_type": "markdown",
"id": "3de8fc00",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This defines what is called the Hessian matrix."
]
@@ -1828,9 +1629,7 @@
{
"cell_type": "markdown",
"id": "a582cfba",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Solving using Newton-Raphson's method\n",
"\n",
@@ -1842,9 +1641,7 @@
{
"cell_type": "markdown",
"id": "7cb1055f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T}\\right)^{-1}_{\\boldsymbol{\\beta}^{\\mathrm{old}}}\\times \\left(\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}\\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}},\n",
@@ -1854,9 +1651,7 @@
{
"cell_type": "markdown",
"id": "ecea7f51",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"or in matrix form as"
]
@@ -1864,9 +1659,7 @@
{
"cell_type": "markdown",
"id": "f98861e3",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X} \\right)^{-1}\\times \\left(-\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{p}) \\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}}.\n",
@@ -1876,9 +1669,7 @@
{
"cell_type": "markdown",
"id": "4fd11bf8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The right-hand side is computed with the old values of $\\beta$. \n",
"\n",
@@ -1888,9 +1679,7 @@
{
"cell_type": "markdown",
"id": "9b7f27a4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Brief reminder on Newton-Raphson's method\n",
"\n",
@@ -1908,9 +1697,7 @@
{
"cell_type": "markdown",
"id": "50e2e1f0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The equations\n",
"\n",
@@ -1924,9 +1711,7 @@
{
"cell_type": "markdown",
"id": "5605583d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"\n",
"
\n",
@@ -1940,9 +1725,7 @@
{
"cell_type": "markdown",
"id": "d6ce1a0c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"For small enough values of the function and for well-behaved\n",
"functions, the terms beyond linear are unimportant, hence we obtain"
@@ -1951,9 +1734,7 @@
{
"cell_type": "markdown",
"id": "7462cf59",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"f(x)+(s-x)f'(x)\\approx 0,\n",
@@ -1963,9 +1744,7 @@
{
"cell_type": "markdown",
"id": "b3609230",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"yielding"
]
@@ -1973,9 +1752,7 @@
{
"cell_type": "markdown",
"id": "63c5804e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"s\\approx x-\\frac{f(x)}{f'(x)}.\n",
@@ -1985,9 +1762,7 @@
{
"cell_type": "markdown",
"id": "2f6643d0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Having in mind an iterative procedure, it is natural to start iterating with"
]
@@ -1995,9 +1770,7 @@
{
"cell_type": "markdown",
"id": "58afbcf0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"x_{n+1}=x_n-\\frac{f(x_n)}{f'(x_n)}.\n",
@@ -2007,9 +1780,7 @@
{
"cell_type": "markdown",
"id": "61a12296",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Simple geometric interpretation\n",
"\n",
@@ -2029,9 +1800,7 @@
{
"cell_type": "markdown",
"id": "39144130",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Extending to more than one variable\n",
"\n",
@@ -2042,9 +1811,7 @@
{
"cell_type": "markdown",
"id": "b98db024",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{array}{cc} f_1(x_1,x_2) &=0\\\\\n",
@@ -2055,9 +1822,7 @@
{
"cell_type": "markdown",
"id": "79154f84",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"which we Taylor expand to obtain"
]
@@ -2065,9 +1830,7 @@
{
"cell_type": "markdown",
"id": "2b225307",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\begin{array}{cc} 0=f_1(x_1+h_1,x_2+h_2)=&f_1(x_1,x_2)+h_1\n",
@@ -2083,9 +1846,7 @@
{
"cell_type": "markdown",
"id": "72363dfd",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"Defining the Jacobian matrix ${\\bf \\boldsymbol{J}}$ we have"
]
@@ -2093,9 +1854,7 @@
{
"cell_type": "markdown",
"id": "7608f604",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"{\\bf \\boldsymbol{J}}=\\left( \\begin{array}{cc}\n",
@@ -2108,9 +1867,7 @@
{
"cell_type": "markdown",
"id": "d10f9e88",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"we can rephrase Newton's method as"
]
@@ -2118,9 +1875,7 @@
{
"cell_type": "markdown",
"id": "e8a79127",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\left(\\begin{array}{c} x_1^{n+1} \\\\ x_2^{n+1} \\end{array} \\right)=\n",
@@ -2132,9 +1887,7 @@
{
"cell_type": "markdown",
"id": "d8451a71",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where we have defined"
]
@@ -2142,9 +1895,7 @@
{
"cell_type": "markdown",
"id": "8c2c4387",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\left(\\begin{array}{c} h_1^{n} \\\\ h_2^{n} \\end{array} \\right)=\n",
@@ -2156,9 +1907,7 @@
{
"cell_type": "markdown",
"id": "3dc3a90d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We need thus to compute the inverse of the Jacobian matrix and it\n",
"is to understand that difficulties may\n",
@@ -2171,9 +1920,7 @@
{
"cell_type": "markdown",
"id": "fee016c8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Steepest descent\n",
"\n",
@@ -2188,9 +1935,7 @@
{
"cell_type": "markdown",
"id": "e4340278",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k),\n",
@@ -2200,9 +1945,7 @@
{
"cell_type": "markdown",
"id": "d2af5812",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"with $\\gamma_k > 0$.\n",
"\n",
@@ -2214,9 +1957,7 @@
{
"cell_type": "markdown",
"id": "a8237bc8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## More on Steepest descent\n",
"\n",
@@ -2229,9 +1970,7 @@
{
"cell_type": "markdown",
"id": "079c64d8",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k), \\ \\ k \\geq 0.\n",
@@ -2241,9 +1980,7 @@
{
"cell_type": "markdown",
"id": "08da6b25",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The parameter $\\gamma_k$ is often referred to as the step length or\n",
"the learning rate within the context of Machine Learning."
@@ -2252,9 +1989,7 @@
{
"cell_type": "markdown",
"id": "7c1a4917",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The ideal\n",
"\n",
@@ -2280,9 +2015,7 @@
{
"cell_type": "markdown",
"id": "4d2aabf0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The sensitiveness of the gradient descent\n",
"\n",
@@ -2302,9 +2035,7 @@
{
"cell_type": "markdown",
"id": "7377154c",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Convex functions\n",
"\n",
@@ -2324,9 +2055,7 @@
{
"cell_type": "markdown",
"id": "697326eb",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Convex function\n",
"\n",
@@ -2336,9 +2065,7 @@
{
"cell_type": "markdown",
"id": "a532b777",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Conditions on convex functions\n",
"\n",
@@ -2373,9 +2100,7 @@
{
"cell_type": "markdown",
"id": "ad4152f5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## More on convex functions\n",
"\n",
@@ -2401,9 +2126,7 @@
{
"cell_type": "markdown",
"id": "e48d339b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Some simple problems\n",
"\n",
@@ -2431,9 +2154,7 @@
{
"cell_type": "markdown",
"id": "9ae5c509",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Revisiting our first homework\n",
"\n",
@@ -2455,10 +2176,7 @@
"cell_type": "code",
"execution_count": 13,
"id": "dd182f20",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"x = 2*np.random.rand(m,1)\n",
@@ -2468,9 +2186,7 @@
{
"cell_type": "markdown",
"id": "eef5bc40",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n",
"The linear regression model is given by"
@@ -2479,9 +2195,7 @@
{
"cell_type": "markdown",
"id": "2af1247d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"h_\\beta(x) = \\boldsymbol{y} = \\beta_0 + \\beta_1 x,\n",
@@ -2491,9 +2205,7 @@
{
"cell_type": "markdown",
"id": "20fcf8f2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"such that"
]
@@ -2501,9 +2213,7 @@
{
"cell_type": "markdown",
"id": "2b6947aa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{y}_i = \\beta_0 + \\beta_1 x_i.\n",
@@ -2513,9 +2223,7 @@
{
"cell_type": "markdown",
"id": "c5838344",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Gradient descent example\n",
"\n",
@@ -2527,9 +2235,7 @@
{
"cell_type": "markdown",
"id": "c5af1dca",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"X \\equiv \\begin{bmatrix}\n",
@@ -2543,9 +2249,7 @@
{
"cell_type": "markdown",
"id": "f380504f",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"The cost/loss/risk function is given by ("
]
@@ -2553,9 +2257,7 @@
{
"cell_type": "markdown",
"id": "c4e7530e",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C(\\beta) = \\frac{1}{n}||X\\beta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\beta_0 + \\beta_1 x_i)^2 - 2 y_i (\\beta_0 + \\beta_1 x_i) + y_i^2\\right]\n",
@@ -2565,9 +2267,7 @@
{
"cell_type": "markdown",
"id": "babaeeee",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"and we want to find $\\beta$ such that $C(\\beta)$ is minimized."
]
@@ -2575,9 +2275,7 @@
{
"cell_type": "markdown",
"id": "1f25dc02",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The derivative of the cost/loss function\n",
"\n",
@@ -2587,9 +2285,7 @@
{
"cell_type": "markdown",
"id": "58eb4735",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\nabla_{\\beta} C(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
@@ -2601,9 +2297,7 @@
{
"cell_type": "markdown",
"id": "32f33b17",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"where $X$ is the design matrix defined above."
]
@@ -2611,9 +2305,7 @@
{
"cell_type": "markdown",
"id": "02ffa8c7",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The Hessian matrix\n",
"The Hessian matrix of $C(\\beta)$ is given by"
@@ -2622,9 +2314,7 @@
{
"cell_type": "markdown",
"id": "7009c819",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{H} \\equiv \\begin{bmatrix}\n",
@@ -2637,9 +2327,7 @@
{
"cell_type": "markdown",
"id": "71cf8211",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This result implies that $C(\\beta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite."
]
@@ -2647,9 +2335,7 @@
{
"cell_type": "markdown",
"id": "b41b50aa",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Simple program\n",
"\n",
@@ -2659,9 +2345,7 @@
{
"cell_type": "markdown",
"id": "1b52d696",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_{k+1} = \\beta_k - \\gamma \\nabla_\\beta C(\\beta_k), \\ k=0,1,\\cdots\n",
@@ -2671,9 +2355,7 @@
{
"cell_type": "markdown",
"id": "ef629c8b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We can use the expression we computed for the gradient and let use a\n",
"$\\beta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n",
@@ -2686,9 +2368,7 @@
{
"cell_type": "markdown",
"id": "0c30718a",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Gradient Descent Example\n",
"\n",
@@ -2699,10 +2379,7 @@
"cell_type": "code",
"execution_count": 14,
"id": "84f33bde",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"\n",
@@ -2756,9 +2433,7 @@
{
"cell_type": "markdown",
"id": "d332552b",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## And a corresponding example using **scikit-learn**"
]
@@ -2767,10 +2442,7 @@
"cell_type": "code",
"execution_count": 15,
"id": "c46612a1",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Importing various packages\n",
@@ -2794,9 +2466,7 @@
{
"cell_type": "markdown",
"id": "2aa00fc2",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Gradient descent and Ridge\n",
"\n",
@@ -2806,9 +2476,7 @@
{
"cell_type": "markdown",
"id": "c2d248a4",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"C_{\\text{ridge}}(\\beta) = \\frac{1}{n}||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n",
@@ -2818,9 +2486,7 @@
{
"cell_type": "markdown",
"id": "fa969e77",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"In order to minimize $C_{\\text{ridge}}(\\beta)$ using GD we adjust the gradient as follows"
]
@@ -2828,9 +2494,7 @@
{
"cell_type": "markdown",
"id": "60d7d114",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\nabla_\\beta C_{\\text{ridge}}(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
@@ -2842,9 +2506,7 @@
{
"cell_type": "markdown",
"id": "a281f2c0",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"We can easily extend our program to minimize $C_{\\text{ridge}}(\\beta)$ using gradient descent and compare with the analytical solution given by"
]
@@ -2852,9 +2514,7 @@
{
"cell_type": "markdown",
"id": "5c40a890",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\beta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n",
@@ -2864,9 +2524,7 @@
{
"cell_type": "markdown",
"id": "5e848da6",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## The Hessian matrix for Ridge Regression\n",
"The Hessian matrix of Ridge Regression for our simple example is given by"
@@ -2875,9 +2533,7 @@
{
"cell_type": "markdown",
"id": "54b17645",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"$$\n",
"\\boldsymbol{H} \\equiv \\begin{bmatrix}\n",
@@ -2890,9 +2546,7 @@
{
"cell_type": "markdown",
"id": "a8bb3901",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"This implies that the Hessian matrix is positive definite, hence the stationary point is a\n",
"minimum.\n",
@@ -2904,9 +2558,7 @@
{
"cell_type": "markdown",
"id": "61ab0a41",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Program example for gradient descent with Ridge Regression"
]
@@ -2915,10 +2567,7 @@
"cell_type": "code",
"execution_count": 16,
"id": "630a15e8",
- "metadata": {
- "collapsed": false,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"from random import random, seed\n",
@@ -2976,9 +2625,7 @@
{
"cell_type": "markdown",
"id": "21abaca5",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Using gradient descent methods, limitations\n",
"\n",
@@ -2998,9 +2645,7 @@
{
"cell_type": "markdown",
"id": "5753b51d",
- "metadata": {
- "editable": true
- },
+ "metadata": {},
"source": [
"## Challenge yourself the coming weekend\n",
"\n",
@@ -3008,7 +2653,25 @@
]
}
],
- "metadata": {},
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.10"
+ }
+ },
"nbformat": 4,
"nbformat_minor": 5
}
diff --git a/doc/src/Projects/2023/Project1/Project1.do.txt b/doc/src/Projects/2023/Project1/Project1.do.txt
index 3e7f5b128..8e58048f9 100644
--- a/doc/src/Projects/2023/Project1/Project1.do.txt
+++ b/doc/src/Projects/2023/Project1/Project1.do.txt
@@ -324,19 +324,19 @@ term which measures the deviation from the true data and the mean value of the m
That is, show that
!bt
\[
-\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=(\mathrm{Bias}[\tilde{y}])^2+\mathrm{var}[\tilde{f}]+\sigma^2,
+\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathrm{Bias}[y]+\mathrm{var}[\tilde{y}]+\sigma^2,
\]
!et
with
!bt
\[
-(\mathrm{Bias}[\tilde{y}])^2=\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2,
+\mathrm{Bias}[y]=\mathbb{E}\left[\left(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right]\right)^2\right],
\]
!et
and
!bt
\[
-\mathrm{var}[\tilde{f}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2.
+\mathrm{var}[\tilde{y}]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2.
\]
!et
The answer to this exercise should be included in the theory part of the report. This exercise is also part of the weekly exercises of week 37.