diff --git a/doc/LectureNotes/exercisesweek36.ipynb b/doc/LectureNotes/exercisesweek36.ipynb index 04624a409..cb2d7d2ce 100644 --- a/doc/LectureNotes/exercisesweek36.ipynb +++ b/doc/LectureNotes/exercisesweek36.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "6bb78ffe", + "id": "50b2fbbe", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "a756b88a", + "id": "da488c0c", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "4c64515c", + "id": "b84d6b13", "metadata": { "editable": true }, @@ -44,7 +44,7 @@ }, { "cell_type": "markdown", - "id": "6d83ecbd", + "id": "96c9c28e", "metadata": { "editable": true }, @@ -62,7 +62,7 @@ }, { "cell_type": "markdown", - "id": "96444623", + "id": "439f1456", "metadata": { "editable": true }, @@ -74,7 +74,7 @@ }, { "cell_type": "markdown", - "id": "66ea8e0c", + "id": "b51e09f7", "metadata": { "editable": true }, @@ -84,7 +84,7 @@ }, { "cell_type": "markdown", - "id": "90823663", + "id": "02c45981", "metadata": { "editable": true }, @@ -97,7 +97,7 @@ }, { "cell_type": "markdown", - "id": "812b9485", + "id": "cc0e91ea", "metadata": { "editable": true }, @@ -107,7 +107,7 @@ }, { "cell_type": "markdown", - "id": "b0bf6d67", + "id": "b5805f35", "metadata": { "editable": true }, @@ -119,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "737ebbd9", + "id": "6e3095bf", "metadata": { "editable": true }, @@ -134,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "d0089b18", + "id": "da90fe04", "metadata": { "editable": true }, @@ -147,7 +147,7 @@ }, { "cell_type": "markdown", - "id": "14388b10", + "id": "1a106e07", "metadata": { "editable": true }, @@ -159,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "a8bc91a0", + "id": "3917877b", "metadata": { "editable": true }, @@ -171,7 +171,7 @@ }, { "cell_type": "markdown", - "id": "9652f49f", + "id": "78226f28", "metadata": { "editable": true }, @@ -183,7 +183,7 @@ }, { "cell_type": "markdown", - "id": "80519c5d", + "id": "951dfffa", "metadata": { "editable": true }, @@ -193,7 +193,7 @@ }, { "cell_type": "markdown", - "id": "43f58011", + "id": "21d2770e", "metadata": { "editable": true }, @@ -205,7 +205,7 @@ }, { "cell_type": "markdown", - "id": "4f336993", + "id": "ec212498", "metadata": { "editable": true }, @@ -217,7 +217,7 @@ }, { "cell_type": "markdown", - "id": "b9dc87be", + "id": "4ffabf6c", "metadata": { "editable": true }, @@ -229,7 +229,7 @@ }, { "cell_type": "markdown", - "id": "7dc4483d", + "id": "f97a6f45", "metadata": { "editable": true }, @@ -241,7 +241,7 @@ }, { "cell_type": "markdown", - "id": "e45e9fa3", + "id": "8761ed23", "metadata": { "editable": true }, @@ -253,7 +253,7 @@ }, { "cell_type": "markdown", - "id": "acb74b76", + "id": "92f8479e", "metadata": { "editable": true }, @@ -268,7 +268,7 @@ }, { "cell_type": "markdown", - "id": "67537508", + "id": "9df91bda", "metadata": { "editable": true }, @@ -280,7 +280,7 @@ }, { "cell_type": "markdown", - "id": "c2de46e7", + "id": "e6de0312", "metadata": { "editable": true }, @@ -290,7 +290,7 @@ }, { "cell_type": "markdown", - "id": "0ead662e", + "id": "8e09d132", "metadata": { "editable": true }, @@ -302,7 +302,7 @@ }, { "cell_type": "markdown", - "id": "d8b43f14", + "id": "a9c924ab", "metadata": { "editable": true }, @@ -314,7 +314,7 @@ }, { "cell_type": "markdown", - "id": "d90dde5d", + "id": "3b9328a1", "metadata": { "editable": true }, @@ -338,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "e303dad2", + "id": "5b54b7b4", "metadata": { "editable": true }, @@ -350,7 +350,7 @@ }, { "cell_type": "markdown", - "id": "4fa7dde4", + "id": "a7ebe31b", "metadata": { "editable": true }, @@ -360,7 +360,7 @@ }, { "cell_type": "markdown", - "id": "1e1b12fc", + "id": "874e0dd3", "metadata": { "editable": true }, @@ -372,7 +372,7 @@ }, { "cell_type": "markdown", - "id": "a6ea7503", + "id": "9adcc39f", "metadata": { "editable": true }, diff --git a/doc/pub/week36/ipynb/week36.ipynb b/doc/pub/week36/ipynb/week36.ipynb index 9137437f1..24add6994 100644 --- a/doc/pub/week36/ipynb/week36.ipynb +++ b/doc/pub/week36/ipynb/week36.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "aa4367c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", @@ -15,9 +13,7 @@ { "cell_type": "markdown", "id": "e6758e15", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "# Week 36: Statistical interpretation of Linear Regression and Resampling techniques\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", @@ -28,9 +24,7 @@ { "cell_type": "markdown", "id": "d223ba4c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Plans for week 36\n", "\n", @@ -52,9 +46,7 @@ { "cell_type": "markdown", "id": "92af6100", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Material for the active learning sessions Tuesday and Wednesday" ] @@ -62,9 +54,7 @@ { "cell_type": "markdown", "id": "289087d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Summary from last Week and discussion of SVD, Ridge and Lasso regression with examples" ] @@ -72,9 +62,7 @@ { "cell_type": "markdown", "id": "dec4f46f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linear Regression and the SVD\n", "\n", @@ -84,9 +72,7 @@ { "cell_type": "markdown", "id": "dc52e1c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -96,9 +82,7 @@ { "cell_type": "markdown", "id": "778e061e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $p\\times p$, with $p$ corresponding to the singular values, we defined last week the matrix" ] @@ -106,9 +90,7 @@ { "cell_type": "markdown", "id": "e0b29417", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma} = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\end{bmatrix},\n", @@ -118,9 +100,7 @@ { "cell_type": "markdown", "id": "b9a47c36", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where the tilde-matrix $\\tilde{\\boldsymbol{\\Sigma}}$ is a matrix of dimension $p\\times p$ containing only the singular values $\\sigma_i$, that is" ] @@ -128,9 +108,7 @@ { "cell_type": "markdown", "id": "0e7ba13f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{\\Sigma}}=\\begin{bmatrix} \\sigma_0 & 0 & 0 & \\dots & 0 & 0 \\\\\n", @@ -145,9 +123,7 @@ { "cell_type": "markdown", "id": "440f1b7e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning we can write" ] @@ -155,9 +131,7 @@ { "cell_type": "markdown", "id": "c82e2760", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2\\boldsymbol{V}^T.\n", @@ -167,9 +141,7 @@ { "cell_type": "markdown", "id": "822670fb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" ] @@ -177,9 +149,7 @@ { "cell_type": "markdown", "id": "91ad3b47", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2.\n", @@ -189,9 +159,7 @@ { "cell_type": "markdown", "id": "39f3629f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## What does it mean?\n", "\n", @@ -203,9 +171,7 @@ { "cell_type": "markdown", "id": "3656f362", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", @@ -215,9 +181,7 @@ { "cell_type": "markdown", "id": "607c77e0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In other words, each non-zero singular value of $\\boldsymbol{X}$ is a positive\n", "square root of an eigenvalue of $\\boldsymbol{X}^T\\boldsymbol{X}$. It means also that\n", @@ -237,9 +201,7 @@ { "cell_type": "markdown", "id": "3c860ef3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{X}]=\\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X},\n", @@ -249,9 +211,7 @@ { "cell_type": "markdown", "id": "5e180c8a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that every squared non-singular value of $\\boldsymbol{X}$ divided by $n$ (\n", "the number of samples) are the eigenvalues of the covariance\n", @@ -264,9 +224,7 @@ { "cell_type": "markdown", "id": "898d8de3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## And finally $\\boldsymbol{X}\\boldsymbol{X}^T$\n", "\n", @@ -276,9 +234,7 @@ { "cell_type": "markdown", "id": "22cb9718", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{U}^T.\n", @@ -288,9 +244,7 @@ { "cell_type": "markdown", "id": "5c2265fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $n\\times n$, we have" ] @@ -298,9 +252,7 @@ { "cell_type": "markdown", "id": "83081288", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\boldsymbol{0}\\\\ \\end{bmatrix}=\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix},\n", @@ -310,9 +262,7 @@ { "cell_type": "markdown", "id": "626bcbf8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "leading to" ] @@ -320,9 +270,7 @@ { "cell_type": "markdown", "id": "f9ad9b1a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\boldsymbol{U}^T.\n", @@ -332,9 +280,7 @@ { "cell_type": "markdown", "id": "87035899", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying with $\\boldsymbol{U}$ from the right gives us the eigenvalue problem" ] @@ -342,9 +288,7 @@ { "cell_type": "markdown", "id": "e6e317a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U}=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}.\n", @@ -354,9 +298,7 @@ { "cell_type": "markdown", "id": "ee0cd565", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that the eigenvalues of $\\boldsymbol{X}\\boldsymbol{X}^T$ are again given by\n", "the non-zero singular values plus now a series of zeros. The column\n", @@ -371,9 +313,7 @@ { "cell_type": "markdown", "id": "6eb10b0f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Code for SVD and Inversion of Matrices\n", "\n", @@ -385,10 +325,7 @@ "cell_type": "code", "execution_count": 1, "id": "2a4acd2d", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "Ainv = np.linlag.pinv(A)" @@ -397,22 +334,38 @@ { "cell_type": "markdown", "id": "faa8f5ce", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "b49b7993", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2 3]\n", + " [2 4 5]\n", + " [3 5 6]]\n", + "test U\n", + "[[-2.22044605e-16 -7.49932427e-16 -8.23408962e-16]\n", + " [-7.49932427e-16 0.00000000e+00 4.77954956e-17]\n", + " [-8.23408962e-16 4.77954956e-17 2.22044605e-16]]\n", + "test VT\n", + "[[ 3.33066907e-16 -7.32066545e-17 3.32714903e-16]\n", + " [-7.32066545e-17 0.00000000e+00 -1.82997013e-16]\n", + " [ 3.32714903e-16 -1.82997013e-16 -3.33066907e-16]]\n", + "[[2.62367905e-12 2.18314256e-12 5.79092330e-13]\n", + " [2.58815192e-12 2.13873363e-12 5.63105118e-13]\n", + " [9.12159237e-13 7.49622586e-13 1.96287431e-13]]\n" + ] + } + ], "source": [ "import numpy as np\n", "# SVD inversion\n", @@ -448,9 +401,7 @@ { "cell_type": "markdown", "id": "8f63e253", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Inverse of Rectangular Matrix\n", "\n", @@ -468,9 +419,7 @@ { "cell_type": "markdown", "id": "e1120966", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{A}_{\\mathrm{PI}}= \\boldsymbol{V}\\boldsymbol{D}_{\\mathrm{PI}}\\boldsymbol{U}^T,\n", @@ -480,22 +429,32 @@ { "cell_type": "markdown", "id": "f3352999", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{D}_{\\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\\boldsymbol{\\Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "9f34d2f0", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0.3 0.4]\n", + " [0.5 0.6]\n", + " [0.7 0.8]\n", + " [0.9 1. ]]\n", + "[[-13. -6. 1. 8. ]\n", + " [ 11.5 5.5 -0.5 -6.5]]\n", + "[[0. 0. 0. 0.]\n", + " [0. 0. 0. 0.]]\n" + ] + } + ], "source": [ "import numpy as np\n", "# SVD inversion\n", @@ -525,9 +484,7 @@ { "cell_type": "markdown", "id": "9d7a538e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**." ] @@ -535,9 +492,7 @@ { "cell_type": "markdown", "id": "47bdc416", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ridge and LASSO Regression\n", "\n", @@ -548,9 +503,7 @@ { "cell_type": "markdown", "id": "9ecebdc2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", @@ -560,9 +513,7 @@ { "cell_type": "markdown", "id": "19f490d5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or we can state it as" ] @@ -570,9 +521,7 @@ { "cell_type": "markdown", "id": "5fd6fca5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -583,9 +532,7 @@ { "cell_type": "markdown", "id": "7e4a2d95", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have used the definition of a norm-2 vector, that is" ] @@ -593,9 +540,7 @@ { "cell_type": "markdown", "id": "4f2374a9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", @@ -605,9 +550,7 @@ { "cell_type": "markdown", "id": "43a71fde", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## From OLS to Ridge and Lasso\n", "\n", @@ -620,9 +563,7 @@ { "cell_type": "markdown", "id": "beeb89f0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -633,9 +574,7 @@ { "cell_type": "markdown", "id": "0e01d29e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to the Ridge regression minimization problem where we\n", "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n", @@ -645,9 +584,7 @@ { "cell_type": "markdown", "id": "0610be33", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -657,9 +594,7 @@ { "cell_type": "markdown", "id": "2c612cc1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have a new optimization equation" ] @@ -667,9 +602,7 @@ { "cell_type": "markdown", "id": "cdc4949b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", @@ -680,9 +613,7 @@ { "cell_type": "markdown", "id": "61fe2430", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", "\n", @@ -692,9 +623,7 @@ { "cell_type": "markdown", "id": "2ec8974f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", @@ -704,9 +633,7 @@ { "cell_type": "markdown", "id": "ce2f3bc8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Ridge Regression Equations\n", "\n", @@ -716,9 +643,7 @@ { "cell_type": "markdown", "id": "640c5179", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n", @@ -728,9 +653,7 @@ { "cell_type": "markdown", "id": "7eaa7fc6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and \n", "taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n", @@ -742,9 +665,7 @@ { "cell_type": "markdown", "id": "96cb4b51", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -754,9 +675,7 @@ { "cell_type": "markdown", "id": "887e2b8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that" ] @@ -764,9 +683,7 @@ { "cell_type": "markdown", "id": "d2d1b062", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n", @@ -776,9 +693,7 @@ { "cell_type": "markdown", "id": "c2d61418", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $t$ a finite positive number." ] @@ -786,9 +701,7 @@ { "cell_type": "markdown", "id": "30191661", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Note on Scikit-Learn\n", "\n", @@ -798,9 +711,7 @@ { "cell_type": "markdown", "id": "9893fc88", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+n\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -810,9 +721,7 @@ { "cell_type": "markdown", "id": "3a99a8e0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In our codes where we compare our own codes with **Scikit-Learn**, we do thus not include the $1/n$ factor in the cost function." ] @@ -820,9 +729,7 @@ { "cell_type": "markdown", "id": "4860197d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Comparison with OLS\n", "When we compare this with the ordinary least squares result we have" @@ -831,9 +738,7 @@ { "cell_type": "markdown", "id": "8fc608c8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -843,9 +748,7 @@ { "cell_type": "markdown", "id": "6980cf65", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n", "\n", @@ -859,9 +762,7 @@ { "cell_type": "markdown", "id": "97582105", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## SVD analysis\n", "\n", @@ -872,9 +773,7 @@ { "cell_type": "markdown", "id": "5a2d1f31", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\beta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n", @@ -884,9 +783,7 @@ { "cell_type": "markdown", "id": "0fd6c833", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For Ridge regression this becomes" ] @@ -894,9 +791,7 @@ { "cell_type": "markdown", "id": "42fc47a0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\beta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n", @@ -906,9 +801,7 @@ { "cell_type": "markdown", "id": "ddf89486", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$." ] @@ -916,9 +809,7 @@ { "cell_type": "markdown", "id": "d8405084", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpreting the Ridge results\n", "\n", @@ -928,9 +819,7 @@ { "cell_type": "markdown", "id": "e18c3873", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n", @@ -940,9 +829,7 @@ { "cell_type": "markdown", "id": "619a3e0b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n", "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n", @@ -956,9 +843,7 @@ { "cell_type": "markdown", "id": "3d710ec4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More interpretations\n", "\n", @@ -968,9 +853,7 @@ { "cell_type": "markdown", "id": "4b93515b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n", @@ -980,9 +863,7 @@ { "cell_type": "markdown", "id": "13f9965f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the standard OLS results in" ] @@ -990,9 +871,7 @@ { "cell_type": "markdown", "id": "3716830b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{n-1}\\boldsymbol{u}_i\\boldsymbol{u}_i^T\\boldsymbol{y},\n", @@ -1002,9 +881,7 @@ { "cell_type": "markdown", "id": "8560f487", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -1012,9 +889,7 @@ { "cell_type": "markdown", "id": "a5ed8586", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n", @@ -1024,9 +899,7 @@ { "cell_type": "markdown", "id": "2166ee0e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n", "the Ridge estimator converges to zero when the hyperparameter goes to\n", @@ -1041,9 +914,7 @@ { "cell_type": "markdown", "id": "1bf4e8ba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Lasso Regression Equations\n", "\n", @@ -1053,9 +924,7 @@ { "cell_type": "markdown", "id": "d4ac9b61", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -1065,9 +934,7 @@ { "cell_type": "markdown", "id": "ca7eb968", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Taking the derivative with respect to $\\boldsymbol{\\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)" ] @@ -1075,9 +942,7 @@ { "cell_type": "markdown", "id": "f7e1c7fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{d \\vert \\beta\\vert}{d \\boldsymbol{\\beta}}=\\mathrm{sgn}(\\boldsymbol{\\beta})=\\left\\{\\begin{array}{cc} 1 & \\beta > 0 \\\\-1 & \\beta < 0, \\end{array}\\right.\n", @@ -1087,9 +952,7 @@ { "cell_type": "markdown", "id": "a916713b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have that the derivative of the cost function is" ] @@ -1097,9 +960,7 @@ { "cell_type": "markdown", "id": "f0e82317", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-2\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=0,\n", @@ -1109,9 +970,7 @@ { "cell_type": "markdown", "id": "027d977b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and reordering we have" ] @@ -1119,9 +978,7 @@ { "cell_type": "markdown", "id": "ee77fb17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta}+\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -1131,9 +988,7 @@ { "cell_type": "markdown", "id": "45e187c5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This equation does not lead to a nice analytical equation as in Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later." ] @@ -1141,9 +996,7 @@ { "cell_type": "markdown", "id": "dbcb7400", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression\n", "\n", @@ -1156,9 +1009,7 @@ { "cell_type": "markdown", "id": "c452f2c4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2,\n", @@ -1168,9 +1019,7 @@ { "cell_type": "markdown", "id": "f437485c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and minimizing we have that" ] @@ -1178,9 +1027,7 @@ { "cell_type": "markdown", "id": "ff46e688", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\beta}_i^{\\mathrm{OLS}} = y_i.\n", @@ -1190,9 +1037,7 @@ { "cell_type": "markdown", "id": "ec2c3a4e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ridge Regression\n", "\n", @@ -1202,9 +1047,7 @@ { "cell_type": "markdown", "id": "8a5a2307", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\beta_i^2,\n", @@ -1214,9 +1057,7 @@ { "cell_type": "markdown", "id": "61b266e4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and minimizing we have that" ] @@ -1224,9 +1065,7 @@ { "cell_type": "markdown", "id": "35e97bfb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\beta}_i^{\\mathrm{Ridge}} = \\frac{y_i}{1+\\lambda}.\n", @@ -1236,9 +1075,7 @@ { "cell_type": "markdown", "id": "6824e494", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Lasso Regression\n", "\n", @@ -1248,9 +1085,7 @@ { "cell_type": "markdown", "id": "3ed3d14b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\vert\\beta_i\\vert=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\sqrt{\\beta_i^2},\n", @@ -1260,9 +1095,7 @@ { "cell_type": "markdown", "id": "9022e9da", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and minimizing we have that" ] @@ -1270,9 +1103,7 @@ { "cell_type": "markdown", "id": "66057e65", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "-2\\sum_{i=0}^{p-1}(y_i-\\beta_i)+\\lambda \\sum_{i=0}^{p-1}\\frac{(\\beta_i)}{\\vert\\beta_i\\vert}=0,\n", @@ -1282,9 +1113,7 @@ { "cell_type": "markdown", "id": "2ac529a0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to" ] @@ -1292,9 +1121,7 @@ { "cell_type": "markdown", "id": "edf0ed42", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}_i^{\\mathrm{Lasso}} = \\left\\{\\begin{array}{ccc}y_i-\\frac{\\lambda}{2} &\\mathrm{if} & y_i> \\frac{\\lambda}{2}\\\\\n", @@ -1306,9 +1133,7 @@ { "cell_type": "markdown", "id": "bef83eda", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Plotting these results shows clearly that Lasso regression suppresses (sets to zero) values of $\\beta_i$ for specific values of $\\lambda$. Ridge regression reduces on the other hand the values of $\\beta_i$ as function of $\\lambda$." ] @@ -1316,9 +1141,7 @@ { "cell_type": "markdown", "id": "10ba1ab6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Yet another Example\n", "\n", @@ -1328,9 +1151,7 @@ { "cell_type": "markdown", "id": "0a913bdd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y}=\\begin{bmatrix}4 \\\\ 2 \\\\3\\end{bmatrix},\n", @@ -1340,9 +1161,7 @@ { "cell_type": "markdown", "id": "0074bec1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and our inputs as a $3\\times 2$ design matrix" ] @@ -1350,9 +1169,7 @@ { "cell_type": "markdown", "id": "d908894e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}2 & 0\\\\ 0 & 1 \\\\ 0 & 0\\end{bmatrix},\n", @@ -1362,9 +1179,7 @@ { "cell_type": "markdown", "id": "acb436e8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that we have two features and two unknown parameters $\\beta_0$ and $\\beta_1$ to be determined either by ordinary least squares, Ridge or Lasso regression." ] @@ -1372,9 +1187,7 @@ { "cell_type": "markdown", "id": "bfa2a7c0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The OLS case\n", "\n", @@ -1384,9 +1197,7 @@ { "cell_type": "markdown", "id": "ca8eb48f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left( \\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -1396,9 +1207,7 @@ { "cell_type": "markdown", "id": "40fb5682", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Inserting the above values we obtain that" ] @@ -1406,9 +1215,7 @@ { "cell_type": "markdown", "id": "fa0b61f7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\begin{bmatrix}2 \\\\ 2\\end{bmatrix},\n", @@ -1418,9 +1225,7 @@ { "cell_type": "markdown", "id": "17950676", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The code which implements this simpler case is presented after the discussion of Ridge and Lasso." ] @@ -1428,9 +1233,7 @@ { "cell_type": "markdown", "id": "4bd9e760", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Ridge case\n", "\n", @@ -1440,9 +1243,7 @@ { "cell_type": "markdown", "id": "53b7ac04", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}=\\left( \\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -1452,9 +1253,7 @@ { "cell_type": "markdown", "id": "5946fdaa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Inserting the above values we obtain that" ] @@ -1462,9 +1261,7 @@ { "cell_type": "markdown", "id": "5a665fa8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}=\\begin{bmatrix}\\frac{8}{4+\\lambda} \\\\ \\frac{2}{1+\\lambda}\\end{bmatrix},\n", @@ -1474,9 +1271,7 @@ { "cell_type": "markdown", "id": "6bfb4703", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "There is normally a constraint on the value of $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2$ via the parameter $\\lambda$.\n", "Let us for simplicity assume that $\\beta_0^2+\\beta_1^2=1$ as constraint. This will allow us to find an expression for the optimal values of $\\beta$ and $\\lambda$.\n", @@ -1487,9 +1282,7 @@ { "cell_type": "markdown", "id": "0d65e044", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Writing the Cost Function\n", "\n", @@ -1499,9 +1292,7 @@ { "cell_type": "markdown", "id": "a329c11c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{\\beta}=\\begin{bmatrix} 2\\beta_0 \\\\ \\beta_1 \\\\0 \\end{bmatrix},\n", @@ -1511,9 +1302,7 @@ { "cell_type": "markdown", "id": "db902ab8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=(4-2\\beta_0)^2+(2-\\beta_1)^2+\\lambda(\\beta_0^2+\\beta_1^2),\n", @@ -1523,9 +1312,7 @@ { "cell_type": "markdown", "id": "8ccb1f96", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and taking the derivative with respect to $\\beta_0$ we get" ] @@ -1533,9 +1320,7 @@ { "cell_type": "markdown", "id": "518818da", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_0=\\frac{8}{4+\\lambda},\n", @@ -1545,9 +1330,7 @@ { "cell_type": "markdown", "id": "091ccc73", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and for $\\beta_1$ we obtain" ] @@ -1555,9 +1338,7 @@ { "cell_type": "markdown", "id": "a4f7e799", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_1=\\frac{2}{1+\\lambda},\n", @@ -1567,9 +1348,7 @@ { "cell_type": "markdown", "id": "0a80d417", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Using the constraint for $\\beta_0^2+\\beta_1^2=1$ we can constrain $\\lambda$ by solving" ] @@ -1577,9 +1356,7 @@ { "cell_type": "markdown", "id": "ac53f555", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\frac{8}{4+\\lambda}\\right)^2+\\left(\\frac{2}{1+\\lambda}\\right)^2=1,\n", @@ -1589,9 +1366,7 @@ { "cell_type": "markdown", "id": "961a8142", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which gives $\\lambda=4.571$ and $\\beta_0=0.933$ and $\\beta_1=0.359$." ] @@ -1599,9 +1374,7 @@ { "cell_type": "markdown", "id": "f16a9bf8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Lasso case\n", "\n", @@ -1612,9 +1385,7 @@ { "cell_type": "markdown", "id": "88e73934", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=(4-2\\beta_0)^2+(2-\\beta_1)^2+\\lambda(\\vert\\beta_0\\vert+\\vert\\beta_1\\vert),\n", @@ -1624,9 +1395,7 @@ { "cell_type": "markdown", "id": "12ef013d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_0}=-4(4-2\\beta_0)+\\lambda\\mathrm{sgn}(\\beta_0)=0,\n", @@ -1636,9 +1405,7 @@ { "cell_type": "markdown", "id": "b1a05c80", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -1646,9 +1413,7 @@ { "cell_type": "markdown", "id": "dde74227", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_1}=-2(2-\\beta_1)+\\lambda\\mathrm{sgn}(\\beta_1)=0.\n", @@ -1658,9 +1423,7 @@ { "cell_type": "markdown", "id": "a7e8c2f4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We have now four cases to solve besides the trivial cases $\\beta_0$ and/or $\\beta_1$ are zero, namely\n", "1. $\\beta_0 > 0$ and $\\beta_1 > 0$,\n", @@ -1675,9 +1438,7 @@ { "cell_type": "markdown", "id": "0481eb81", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The first Case\n", "\n", @@ -1687,9 +1448,7 @@ { "cell_type": "markdown", "id": "bd95432d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "-4(4-2\\beta_0)+\\lambda=0,\n", @@ -1699,9 +1458,7 @@ { "cell_type": "markdown", "id": "3ecbb638", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -1709,9 +1466,7 @@ { "cell_type": "markdown", "id": "315dc174", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "-2(2-\\beta_1)+\\lambda=0.\n", @@ -1721,9 +1476,7 @@ { "cell_type": "markdown", "id": "4a23b51f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which yields" ] @@ -1731,9 +1484,7 @@ { "cell_type": "markdown", "id": "991181e5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_0=\\frac{16+\\lambda}{8},\n", @@ -1743,9 +1494,7 @@ { "cell_type": "markdown", "id": "92ad54a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -1753,9 +1502,7 @@ { "cell_type": "markdown", "id": "c67d239d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_1=\\frac{4+\\lambda}{2}.\n", @@ -1765,9 +1512,7 @@ { "cell_type": "markdown", "id": "02130691", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Using the constraint on $\\beta_0$ and $\\beta_1$ we can then find the optimal value of $\\lambda$ for the different cases. We leave this as an exercise to you." ] @@ -1775,9 +1520,7 @@ { "cell_type": "markdown", "id": "a7c79c97", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple code for solving the above problem\n", "\n", @@ -1788,13 +1531,30 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "fa197615", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2. 2.]\n", + "Training MSE for OLS\n", + "3.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAGwCAYAAACHJU4LAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABB20lEQVR4nO3deZzN9eLH8feZMftmGWsGQ91EWUO2K9m3pFJXspYSiaSkbpKrO3FLXXWLUihl0qLSL2shRbYQspTIFCLLzNhmzMz398e3mXHMYmacmc9ZXs/H4/s4y3zOOe8zxpz3fJfP12FZliUAAAA35Gc6AAAAQF4oKgAAwG1RVAAAgNuiqAAAALdFUQEAAG6LogIAANwWRQUAALitUqYDXI6MjAwdPHhQERERcjgcpuMAAIACsCxLycnJqlKlivz88l9n4tFF5eDBg4qJiTEdAwAAFEFCQoKqVq2a7xiPLioRERGS7DcaGRlpOA0AACiIpKQkxcTEZH2O58eji0rm5p7IyEiKCgAAHqYgu22wMy0AAHBbFBUAAOC2KCoAAMBtefQ+KgWVnp6u8+fPm44BHxMQECB/f3/TMQDAo3l1UbEsS4cPH9bJkydNR4GPKl26tCpVqsQ8PwBQRF5dVDJLSoUKFRQaGsqHBUqMZVk6c+aMjhw5IkmqXLmy4UQA4Jm8tqikp6dnlZRy5cqZjgMfFBISIkk6cuSIKlSowGYgACgCr92ZNnOflNDQUMNJ4Msyf/7YRwoAisZri0omNvfAJH7+AODyeH1RAQAAnouiAgAA3BZFBZdl//79cjgc2rJlS55jVq5cKYfD4ZGHiQ8cOFC33HKL6RgA4LMoKm5o4MCBcjgcGjp0aI6vDRs2TA6HQwMHDsy678iRI7r//vtVrVo1BQUFqVKlSurUqZPWrl2bNaZGjRpyOBw5lueeey7PHDfeeGPWuMDAQNWqVUvjxo1TSkpK1piYmBgdOnRI1157rWvefBHl9f4ylxtvvLFIz/vf//5Xs2fPdmlWADDOsqTz56Vz56TTp6XEROn4cenoUSk11XmcYV57eLKni4mJUXx8vF588cWsw1zPnTunefPmqVq1ak5jb7vtNp0/f15z5sxRzZo19ccff+jLL7/U8ePHncZNnDhRQ4YMcbrvUqfYHjJkiCZOnKjU1FRt2LBBgwYNkiTFxcVJkvz9/VWpUqXLeq+usGHDBqWnp0uS1qxZo9tuu027d+/OOqt2YGCg0/jz588rICDgks8bFRXl+rAAfNv581JSkr2cOmUXhVOn7KVTJ+mv3/lavlxavdouExcvKSnSK69IVavaY2fMkKZPt0vG+fM5ly+/lBo2tMdOmSKNHZt3vhUrpMw/7tLSpAL8rixOvllUTp/O+2v+/lJwcMHG+vll/0DlNTYsrPD5JDVq1Ei//PKLPv74Y/Xt21eS9PHHHysmJkY1a9bMGnfy5El98803Wrlypdq0aSNJql69upo2bZrjOSMiIgpdKkJDQ7MeU61aNb333ntaunRpVlHZv3+/YmNjtXnzZjVo0ECS9MUXX2jUqFFKSEjQDTfcoAEDBuR43jfeeEMTJ07UsWPH1KlTJ7Vu3VoTJ0502jy0cOFCTZgwQTt27FCVKlU0YMAAPfnkkypVKuePbfny5bOuly1bVpJUoUIFlS5dWpJ99M1rr72mRYsWafny5RozZozGjx+v++67T1999ZUOHz6satWqadiwYRo5cmTWcw0cOFAnT57UJ598Isley1SvXj0FBwdr5syZCgwM1NChQzVhwoRCfV8BeJG9e+3l6FHpzz+zlxMn7OWDD6S//mjSiBF2wcjLL79IsbH29eXLpcmT8x47cWJ2UTlyRMpnE7zOncu+fqk5nf76o0+S5AZHLvpmUQkPz/trXbtK//d/2bcrVJDOnMl9bJs20sqV2bdr1LB/OC90GavNBg0apFmzZmUVlbfeekuDBw/WygteMzw8XOHh4frkk090ww03KCgoqMivdylbt27Vt99+qxo1auQ5JiEhQbfeequGDh2qBx54QBs3btQjjzziNObbb7/V0KFDNXnyZN18881avny5nnrqKacxS5Ys0d13361p06apdevW2rt3r+677z5J0tNPP12k/E8//bTi4uL04osvyt/fXxkZGapatarmz5+v6OhorVmzRvfdd58qV66sO+64I8/nmTNnjkaPHq1169Zp7dq1GjhwoFq2bKkOHToUKRcAN5SRIR06ZBeQX3+VEhKyl99+k775JvuzJC5OevPNvJ/r2LHsonLh3F4hIVJEhP0HbXh4zj9smzeXHnjAHhccnL0EBdlLlSrZY/v0kZo2lQID7TUgFy6lStmfT5mGDpUGDLALS26L3wV7heTyh2GJszxYYmKiJclKTEzM8bWzZ89aP/74o3X27NmcD7TrQ+5L167OY0ND8x7bpo3z2OjonGOKYMCAAVbPnj2to0ePWkFBQda+ffus/fv3W8HBwdbRo0etnj17WgMGDMga/+GHH1plypSxgoODrRYtWljjxo2ztm7d6vSc1atXtwIDA62wsDCnZcWKFXnmaNOmjRUQEGCFhYVZgYGBliTLz8/P+vDDD7PG7Nu3z5Jkbd682bIsyxo3bpx1zTXXWBkZGVljxo4da0myTpw4YVmWZd15551Wt27dnF6rb9++VlRUVNbt1q1bW//+97+dxrzzzjtW5cqVL/n9W7FihdPrWZZlSbJGjRp1yccOGzbMuu2227JuZ/5bZGrTpo3VqlUrp8c0adLEGjt2bK7Pl+/PIQDzEhMt67vvLOuttyzr/Pns+wcPzv+z4uefs8dOmWJZ9epZVrt2lnXnnZY1fLhljR9vWS+9ZFmzZ1vWyZPOr3f8uPNr+aD8Pr8v5gZVyYBTp/L+2sWrxP46V0uu/C7aF3n//iJHyk10dLS6deumOXPmyLIsdevWTdHR0TnG3XbbberWrZtWr16ttWvXavHixZoyZYpmzpzptNPto48+6nRbkq644op8M/Tt21dPPvmkkpKSNHnyZEVGRuq2227Lc/zOnTt1ww03OE101rx5c6cxu3fvVq9evZzua9q0qT7//POs25s2bdKGDRv07LPPZt2Xnp6uc+fO6cyZM0Wacfj666/Pcd/06dM1c+ZM/frrrzp79qxSU1OzNmHlpV69ek63K1eunHVOHwBu7MgRaf16adMm6fvvpc2b7TUkmVq1kq66yr5evbr9eVC9ur02olo1KSbGXqpWlSpWzH7co4/aS0FkrllBgflmUSnMfiPFNbaABg8erAcffFCS9L///S/PccHBwerQoYM6dOig8ePH695779XTTz/tVEyio6N15ZVXFur1o6Kish4zd+5c1a1bV2+++abuueeeXMdbBdjUZVlWjhlbL35cRkaGnnnmGd166605Hh984T5EhRB20b/P/Pnz9fDDD+uFF15Q8+bNFRERof/85z9at25dvs9z8U64DodDGRkZRcoEoJhkZEjbt0u1amX/bp42Tbrgj58sVapIdeo478cxZoz0xBPusenDx/Ev4OY6d+6s1L8OFevUqVOBH1enTp2sHUBdJSAgQE888YTGjRunPn365LpWI7fX/e6775xu165dW+vXr3e6b+PGjU63GzVqpN27dxe6WBXG6tWr1aJFCw0bNizrvr179xbb6wEoZr//Li1ZIi1ebO+IeuKEtGiR1Lmz/fWmTaW6daVGjaTGje3L666T/trp3gnniXMbFBU35+/vr507d2Zdv9ixY8fUu3dvDR48WPXq1VNERIQ2btyoKVOmqGfPnk5jk5OTdfjwYaf7QkNDsw7hLYi77rpLTzzxhF599VWNGTMmx9eHDh2qF154QaNHj9b999+vTZs25ZiHZMSIEfr73/+uqVOnqkePHvrqq6+0aNEip7Us48ePV/fu3RUTE6PevXvLz89PP/zwg7Zt26ZJkyYVOG9+rrzySr399ttasmSJYmNj9c4772jDhg2KzdzjHoD7S0iQXntN+vxzads256+FhUkHD2bfvvlme4FHYcI3DxAZGZlnmQgPD1ezZs304osv6u9//7uuvfZaPfXUUxoyZIheuegQuPHjx6ty5cpOy2OPPVaoLIGBgXrwwQc1ZcoUncplX59q1arpo48+0sKFC1W/fn1Nnz5d//73v53GtGzZUtOnT9fUqVNVv359LV68WA8//LDTJp1OnTrp888/17Jly9SkSRPdcMMNmjp1qqpXr16ovPkZOnSobr31Vt15551q1qyZjh075rR2BYCbOns2+/qxY/ZRN9u22YfSNmsmTZggrV0rnTwpDR5sKiVcxGEVZKcCN5WUlKSoqCglJibm+CA/d+6c9u3bp9jY2CLv04CSM2TIEO3atUurV682HcWl+DkEXOTUKSk+3p7UrE4d6e237fstS3rwQallS3uytHLlzOZEgeT3+X0xNv3AiOeff14dOnRQWFiYFi1apDlz5ujVV181HQuAu9m2zd60M3eulJxs33fggD0pmb+/vRYlnwMN4PkoKjBi/fr1mjJlipKTk1WzZk1NmzZN9957r+lYANzFunXSv/7lPAHnVVdJ990nDRx46dlV4TUoKjBi/vz5piMAcGcLF9olxc9PuvVWe4bWtm3dYkp3lCyvLyoevAsOvAA/f0ABbd1qryXJPBP7qFH2uXPGjMmehA0+yWuP+smclOtMXufpAUpA5s9fQc7UDPikI0fszTkNG9rnoMks99HR9hmBKSk+z2vXqPj7+6t06dJZU5uHhobmmA0VKC6WZenMmTM6cuSISpcunescOIBPS0uTXnrJPgNw5k6yV1xhn4U+vxPHwud4bVGRpEqVKkkS52GBMaVLl876OQTwl5077bP3bthg327c2C4trVoZjQX35NVFxeFwqHLlyqpQoYLOnz9vOg58TEBAAGtSgIutWye1aSOlpEhRUdLUqfZRPBef5BX4i1cXlUz+/v58YACAO2jc2D6/TnS0NHOmvbkHyIdPFBUAgEFr10pNmthnIi5Vyj5xYJkyHGqMAmFdGwCgeFiW9Nxz9vT2jz+efX/ZspQUFBhrVAAArnf6tH1CwMzJHRMTpYwM9kVBoVFUAACu9dtvUrdu0g8/SAEB0rRp0v33sxYFRUJRAQC4zs8/S+3bS7/+KlWsKH30kb3pBygiigoAwDVSUqQOHeySctVV0rJlUvXqplPBw7GxEADgGkFB9sRtjRpJq1dTUuASFBUAwOW58OSbPXtK69fbm30AF6CoAACK7vvv7Unc9u7Nvo8JNuFCFBUAQNHs3St16SJt3iw9+aTpNPBSFBUAQOEdOSJ17mxf1q8vvf666UTwUhQVAEDhJCdLXbvahyLXqCEtWiRFRppOBS9ltKgkJydr1KhRql69ukJCQtSiRQttyDztNwDA/aSmSrffLm3aZJ9YcMkSqXJl06ngxYwWlXvvvVfLli3TO++8o23btqljx45q3769fv/9d5OxAAB5mTBBWrpUCg2VvvhC+tvfTCeCl3NY1oXHlZWcs2fPKiIiQp9++qm6deuWdX+DBg3UvXt3TZo0KcdjUlJSlJKSknU7KSlJMTExSkxMVCSrHQGg+B0+LN11l/Tww1KPHqbTwEMlJSUpKiqqQJ/fxtaopKWlKT09XcHBwU73h4SE6Jtvvsn1MXFxcYqKispaYmJiSiIqACBTpUrSl19SUlBijBWViIgINW/eXP/617908OBBpaena+7cuVq3bp0OHTqU62PGjRunxMTErCUhIaGEUwOAD0pOlj75JPs2JxdECTK6j8o777wjy7J0xRVXKCgoSNOmTdNdd90l/zwmCwoKClJkZKTTAgAoRpYl3XOP1KuXNHGi6TTwQUaLSq1atbRq1SqdOnVKCQkJWr9+vc6fP6/Y2FiTsQAAmaZNkz74QCpVyj7hIFDC3GIelbCwMFWuXFknTpzQkiVL1LNnT9ORAADbt0uPPmpff+EFqXlzs3ngk0qZfPElS5bIsixdffXV+vnnn/Xoo4/q6quv1qBBg0zGAgCkpUkDB0rnz9s7zo4YYToRfJTRNSqJiYkaPny4ateurf79+6tVq1ZaunSpAgICTMYCAPznP/akbqVLSzNmsAMtjDE2j4orFOY4bABAAf3+u1Szpj0L7dtvS/36mU4EL1OYz2+jm34AAG7oiiukDz+UPvtMuvtu02ng4ygqAICcevRgUje4Bbc46gcA4AZ++UXKY8JNwBSKCgDAnthtyBCpTh17inzATbDpBwAgffqp9NVXUlCQVKuW6TRAFtaoAICvS0mRHnnEvj5mjFSjhtE4wIUoKgDg6156yd4/pXJl6fHHTacBnFBUAMCXHTokTZpkX3/uOSk83Gwe4CIUFQDwZU8+KZ06JTVtypwpcEsUFQDwVZYlhYZK/v725h8/PhLgfvipBABf5XBIr7wi7d/PmZHhtigqAODrqlY1nQDIE0UFAHyNZUljx0rbtplOAlwSRQUAfM3nn0tTpkgtW0rJyabTAPmiqACAL7EsacIE+/qwYVJEhNE4wKVQVADAlyxcKH3/vRQWZs9CC7g5igoA+IoL16aMGCFFRxuNAxQERQUAfMWnn0qbN9uzz7I2BR6CogIAvsCypGeesa8/9JBUrpzZPEABlTIdAABQAtLS7CnyT53KPlMy4AFYowIAviAgwC4ou3dLZcuaTgMUGEUFAHwJ5/OBh+EnFgC83fjx0gcf2Jt/AA/DPioA4M327ZOefVbKyJB27JDq1DGdCCgU1qgAgDebNs0uKZ06UVLgkSgqAOCtEhOlmTPt6w8/bDYLUEQUFQDwVm++aR+OXKeO1LGj6TRAkVBUAMAbpaVJ//2vff3hhyWHw2weoIgoKgDgjRYskA4ckMqXl/r2NZ0GKDKO+gEAbxQdLbVoIbVvL4WEmE4DFBlFBQC8Udu20rffMncKPB6bfgDAm5Xi71F4NooKAHiTU6ekqVOlo0dNJwFcgqICAN4kPt4++eBNN5lOArgERQUAvMkbb9iX/fubzQG4CEUFALzF1q3S+vVSQIA0YIDpNIBLUFQAwFtkrk255RapQgWjUQBXoagAgDc4c0aaO9e+PmSI2SyAC1FUAMAbfPihfRLC2FipXTvTaQCXMVpU0tLS9M9//lOxsbEKCQlRzZo1NXHiRGVkZJiMBQCe59df7X1T7rlH8uNvUHgPozMBTZ48WdOnT9ecOXNUt25dbdy4UYMGDVJUVJRGjhxpMhoAeJannpIeeEDy9zedBHApo0Vl7dq16tmzp7p16yZJqlGjhubNm6eNGzeajAUAnik62nQCwOWMrh9s1aqVvvzyS+3Zs0eStHXrVn3zzTfq2rVrruNTUlKUlJTktACAT0tLk37+2XQKoNgYLSpjx45Vnz59VLt2bQUEBKhhw4YaNWqU+vTpk+v4uLg4RUVFZS0xMTElnBgA3MyXX0pXXSX16GE6CVAsjBaV999/X3PnztV7772n77//XnPmzNHzzz+vOXPm5Dp+3LhxSkxMzFoSEhJKODEAuJl337Uvq1UzmwMoJg7LsixTLx4TE6PHH39cw4cPz7pv0qRJmjt3rnbt2nXJxyclJSkqKkqJiYmKjIwszqgA4H5On5YqVbJPRPjtt1KLFqYTAQVSmM9vo2tUzpw5I7+LDqPz9/fn8GQAKIjPPrNLSmys1Ly56TRAsTB61E+PHj307LPPqlq1aqpbt642b96sqVOnavDgwSZjAYBnyNzs07ev5HCYzQIUE6ObfpKTk/XUU09pwYIFOnLkiKpUqaI+ffpo/PjxCgwMvOTj2fQDwGf9+adUubJ91M/OnVLt2qYTAQVWmM9vo0XlclFUAPis116Thg2TGjWSNm0ynQYolMJ8fhvd9AMAKKKBA6Vy5aTgYNNJgGJFUQEATxQSIt1xh+kUQLHjzFUAAMBtUVQAwJNYltSzpzRpknTypOk0QLGjqACAJ9m82Z4/5dlnOSQZPoGiAgCe5IMP7Mtu3aSoKLNZgBJAUQEAT2FZ0kcf2ddvv91sFqCEUFQAwFNs3y799JMUFGSvUQF8AEUFADzFhx/al506SRERZrMAJYSiAgCeInOzz223mc0BlCCKCgB4gnPnpIYNpehoqUcP02mAEsPMtADgCYKDpXfekdLTJX9/02mAEsMaFQDwJJQU+BiKCgC4uyNHpK1b7cOTAR9DUQEAd/f221KDBlL//qaTACWOogIA7i7zsOTmzc3mAAygqACAO/vtN2ndOvu8Pr16mU4DlDiKCgC4s48/ti9btpQqVzabBTCAogIA7oxJ3uDjKCoA4K6OHZO++ca+zmYf+CiKCgC4q0WLpIwMqV49qXp102kAI5iZFgDcVe/eUoUKUmqq6SSAMRQVAHBXQUFSx46mUwBGsekHAAC4LYoKALijyZOlMWOknTtNJwGMYtMPALgby5KmT5f275dat5auucZ0IsAY1qgAgLvZscMuKUFBUvv2ptMARlFUAMDdfPaZfdmunRQWZjYLYBhFBQDczcKF9uXNN5vNAbgBigoAuJMjR+yTEEpS9+5mswBugKICAO7k//7P3pm2USPpiitMpwGMo6gAgDtJSbHPktyjh+kkgFtwWJZlmQ5RVElJSYqKilJiYqIiIyNNxwEA18jIsAtLSIjpJECxKMznN2tUAMDd+PlRUoC/UFQAwF389pu9NgVAFooKALiLG2+UKlaUNm0ynQRwG0yhDwDu4KefpL17pYAA6W9/M50GcBusUQEAd7BokX3ZurUUEWE2C+BGKCoA4A4yi0rnzmZzAG6GogIApp09K61caV/v0sVoFMDdGC0qNWrUkMPhyLEMHz7cZCwAKFmrVknnzklVq0p165pOA7gVozvTbtiwQenp6Vm3t2/frg4dOqh3794GUwFACcvc7NOli+RwmM0CuBmjRaV8+fJOt5977jnVqlVLbdq0yXV8SkqKUlJSsm4nJSUVaz4AKBH9+klhYVKHDqaTAG7HbQ5PTk1N1dy5czV69Gg58viLIi4uTs8880wJJwOAYnb99fYCIAe3OdfP/Pnzddddd+nAgQOqUqVKrmNyW6MSExPDuX4AAPAghTnXj9usUXnzzTfVpUuXPEuKJAUFBSkoKKgEUwFAMXvtNSkmRmrXjvP7ALlwi6Ly66+/avny5fr4449NRwGAknPunPTII/bhyVu2SPXrm04EuB23mEdl1qxZqlChgrp162Y6CgCUnK+/tktKlSpSvXqm0wBuyXhRycjI0KxZszRgwACVKuUWK3gAoGRcOBsthyUDuTJeVJYvX64DBw5o8ODBpqMAQMm6cP4UALkyvgqjY8eOcpMDjwCg5Pz6q7R7t+TvL7VvbzoN4LaMr1EBAJ+0bJl92ayZVLq00SiAO6OoAIAJGzfalx07ms0BuDnjm34AwCe99pp9aHJYmOkkgFujqACACQ6HdNVVplMAbo9NPwAAwG1RVACgpPXsKd1+u/Tjj6aTAG6PogIAJSk5WfriC+mjj6TgYNNpALdHUQGAkrRypZSWJtWqJdWsaToN4PYoKgBQkpYutS85LBkoEIoKAJQkigpQKBQVACgp+/dLe/bY0+a3bWs6DeARKCoAUFIyp82/4QYpKspsFsBDUFQAoKSULi01b87ZkoFCYGZaACgpvXvbC2eMBwqMNSoAUNIcDtMJAI9BUQGAkrBnj3TypOkUgMcpVFGZMmWKzp49m3X766+/VkpKStbt5ORkDRs2zHXpAMBbDBkilStnz0gLoMAcllXwjaX+/v46dOiQKlSoIEmKjIzUli1bVPOv2RX/+OMPValSRenp6cWT9iJJSUmKiopSYmKiIiMjS+Q1AaDQTp2SypaVzp+Xfv7ZnpUW8GGF+fwu1BqViztNIToOAPiu1avtklKjBtPmA4XEPioAUNwy50/p0IEdaYFCoqgAQHFbvty+bN/ebA7AAxV6HpWZM2cqPDxckpSWlqbZs2crOjpakr0zLQDgAocPS9u22WtSbrrJdBrA4xSqqFSrVk1vvPFG1u1KlSrpnXfeyTEGAPCXL7+0Lxs2lP76ow5AwRWqqOzfv7+YYgCAl2rbVpo+XfprTTSAwmEKfQAoTlWqSPffbzoF4LEKtTPtunXrtGjRIqf73n77bcXGxqpChQq67777nCaAAwAAuByFKioTJkzQDz/8kHV727Ztuueee9S+fXs9/vjjWrhwoeLi4lweEgA80hdfSP/7n8Rmc6DIClVUtmzZonbt2mXdjo+PV7NmzfTGG29o9OjRmjZtmubPn+/ykADgkWbMkB58UIqPN50E8FiFKionTpxQxYoVs26vWrVKnTt3zrrdpEkTJSQkuC4dAHiq8+elFSvs6x06mM0CeLBCFZWKFStq3759kqTU1FR9//33at68edbXk5OTFRAQ4NqEAOCJNmyQkpPtc/w0aGA6DeCxClVUOnfurMcff1yrV6/WuHHjFBoaqtatW2d9/YcfflAtTrYFANmz0d50k+TvbzYL4MEKdXjypEmTdOutt6pNmzYKDw/X7NmzFRgYmPX1t956Sx07dnR5SADwOBee3wdAkTmsIpwCOTExUeHh4fK/6K+E48ePKyIiosQ2/xTmNNEAUGIyN/mkpUl793LGZOAihfn8LtQalcGDBxdo3FtvvVWYpwUA75I5jUNsLCUFuEyFKiqzZ89W9erV1bBhQxVhRQwA+IaWLaXjx5k/BXCBQhWVoUOHKj4+Xr/88osGDx6su+++W2XLli2ubADguSIipOuuM50C8HiFOurn1Vdf1aFDhzR27FgtXLhQMTExuuOOO7RkyRLWsAAAAJcrVFGRpKCgIPXp00fLli3Tjz/+qLp162rYsGGqXr26Tp06VRwZAcBzzJsnNW1qnzEZwGUrdFG5kMPhkMPhkGVZysjIcFUmAPBcS5bYk72xfwrgEoUuKikpKZo3b546dOigq6++Wtu2bdMrr7yiAwcOKDw8vNABfv/9d919990qV66cQkND1aBBA23atKnQzwMAxllW9kRv7dubzQJ4iULtTDts2DDFx8erWrVqGjRokOLj41WuXLkiv/iJEyfUsmVLtW3bVosWLVKFChW0d+9elS5dusjPCQDG7N4t/f67FBRkH/kD4LIVqqhMnz5d1apVU2xsrFatWqVVq1blOu7jjz8u0PNNnjxZMTExmjVrVtZ9NWrUKEwkAHAfmWtTWrWSQkLMZgG8RKGKSv/+/eVwOFz24p999pk6deqk3r17a9WqVbriiis0bNgwDRkyJNfxKSkpSklJybqdlJTksiwAcNkyiwrT5gMuU6Qp9F0lODhYkjR69Gj17t1b69ev16hRozRjxgz1798/x/gJEybomWeeyXE/U+gDMC4tTSpXTkpKkjZulBo3Np0IcFuFmULfaFEJDAzU9ddfrzVr1mTd99BDD2nDhg1au3ZtjvG5rVGJiYmhqAAw788/pQcflLZskXbs4IzJQD6K7Vw/rla5cmXVqVPH6b5rrrlGH330Ua7jg4KCFBQUVBLRAKBwoqOl+Hj7yB8XbiIHfN1lzaNyuVq2bKndu3c73bdnzx5Vr17dUCIAuEyUFMCljBaVhx9+WN99953+/e9/6+eff9Z7772n119/XcOHDzcZCwAK5+xZ6ccf7bUpAFzKaFFp0qSJFixYoHnz5unaa6/Vv/71L7300kvq27evyVgAUDgrVkh16zJ3ClAMjO6jIkndu3dX9+7dTccAgKJbtsy+rFvXbA7ACxldowIAXiGzqDBtPuByFBUAuBwHD9qHIzscFBWgGFBUAOByZK5NadzYnvANgEtRVADgcmQWFabNB4oFRQUAiiojI/v8Ph07ms0CeCnjR/0AgMeyLGn2bLusNG9uOg3glSgqAFBU/v5S5872AqBYsOkHAAC4LYoKABTFuXPS44/bO9NmZJhOA3gtigoAFMU330iTJ0sDB3IiQqAYUVQAoCguPCyZogIUG4oKABTF0qX2JYclA8WKogIAhXXkiLRli32dafOBYkVRAYDCypzkrUEDqUIFo1EAb0dRAYDCYtp8oMRQVACgsLZtsy/ZPwUodsxMCwCFtWGDXVb+9jfTSQCvR1EBgMJyOKR69UynAHwCm34AoDAsy3QCwKdQVACgoE6flqpXl/r1s68DKHYUFQAoqJUrpYQEafVqKTTUdBrAJ1BUAKCgFi+2Lzt3Ztp8oIRQVACgoC4sKgBKBEUFAAri55/tpVQp6aabTKcBfAZFBQAKYskS+7JlSyky0mwWwIdQVACgIDI3+3TpYjYH4GOY8A0ACqJpU/uIH/ZPAUqUw7I8d/aipKQkRUVFKTExUZGsigUAwCMU5vObTT8AAMBtUVQA4FK++oqZaAFDKCoAkJ+EBKldO6lCBenUKdNpAJ9DUQGA/GQellyvnhQebjYL4IMoKgCQH2ajBYyiqABAXlJTpaVL7evMnwIYQVEBgLx8/bWUnCxVqiRdf73pNIBPoqgAQF4+/9y+7NZN8uPXJWAC//MAIDeWJS1caF/v3t1sFsCHMYU+AOTG4ZAWLbLXqrRvbzoN4LMoKgCQl7/9TRo92nQKwKcZ3fQzYcIEORwOp6VSpUomIwEAADdifB+VunXr6tChQ1nLtm3bTEcC4OuOH5duv12aPdveVwWAMcY3/ZQqVYq1KADcy6JF0kcfSXv2SAMHmk4D+DTja1R++uknValSRbGxsfrHP/6hX375Jc+xKSkpSkpKcloAwOUyD0vmaB/AOKNFpVmzZnr77be1ZMkSvfHGGzp8+LBatGihY8eO5To+Li5OUVFRWUtMTEwJJwbg9c6fz542v0cPs1kAyGFZ7rMB9vTp06pVq5Yee+wxjc5lT/uUlBSlpKRk3U5KSlJMTIwSExMVGRlZklEBeKuVK6W2baXoaOnwYcnf33QiwOskJSUpKiqqQJ/fxvdRuVBYWJiuu+46/fTTT7l+PSgoSEFBQSWcCoBPydzs07UrJQVwA8b3UblQSkqKdu7cqcqVK5uOAsBXZRYVNvsAbsFoURkzZoxWrVqlffv2ad26dbr99tuVlJSkAQMGmIwFwFclJ0tly0pBQVLHjqbTAJDhTT+//fab+vTpoz///FPly5fXDTfcoO+++07Vq1c3GQuAr4qIkNaskZKSJPZ7A9yC0aISHx9v8uUBIHeUFMBtuNU+KgBgTFKSlMfUCADMoagAgCTNmSNVrMhJCAE3Q1EBAMmeMj89XWIiScCtUFQA4MgRafVq+3qvXmazAHBCUQGATz+VMjKkxo2lGjVMpwFwAYoKAHz8sX15661mcwDIgaICwLedPCl9+aV9/bbbjEYBkBNFBYBv+/xz+4zJdepIV19tOg2Ai7jVSQkBoMR16iTNmCGFhppOAiAXFBUAvq18eem++0ynAJAHNv0AAAC3xRoVAL4rLs4+EeE//iFFR5tOAyAXFBUAvun0aenZZ+3Lxo0pKoCbYtMPAN+0cKFdUmJjpRtuMJ0GQB4oKgB807vv2pd33SU5HGazAMgTRQWA7zl2TFq82L7et6/ZLADyRVEB4Hs++EBKS5MaNJCuucZ0GgD5oKgA8D2Zm31YmwK4PYoKAN9y/rxUpowUGGgflgzArXF4MgDfEhAgffaZlJQkRUaaTgPgElijAsA3UVIAj0BRAeA7Dh6U9u0znQJAIVBUAPiOadOkmjWlceNMJwFQQBQVAL4hI0N67z37euPGZrMAKDCKCgDf8M03UkKCvW9K9+6m0wAoIIoKAN/w1lv25e23S8HBZrMAKDCKCgDvd/KkNH++ff3ee41GAVA4FBUA3u/dd6WzZ6Vrr+VMyYCHoagA8H6Za1OGDOFMyYCHYWZaAN5v0SLpww/ZiRbwQBQVAN4vNFTq3990CgBFwKYfAN4rLU2yLNMpAFwGigoA7/XWW1KdOtI775hOAqCIKCoAvNcbb0i7dklHjphOAqCIKCoAvNP330sbN0oBAeyfAngwigoA7/TGG/blrbdK5cubzQKgyCgqALxPYqI9yZtkz50CwGNRVAB4n5kzpeRk6ZprpLZtTacBcBkoKgC8S1qa9N//2tdHj5b8+DUHeDL+BwPwLv7+9maffv2ku+82nQbAZXKbohIXFyeHw6FRo0aZjgLAkzkcUuvW0ttvS8HBptMAuExuUVQ2bNig119/XfXq1TMdBQAAuBHjReXUqVPq27ev3njjDZUpUybfsSkpKUpKSnJaACBL377Sww9Lhw6ZTgLARYwXleHDh6tbt25q3779JcfGxcUpKioqa4mJiSmBhAA8wq5d0nvv2TvS8kcM4DWMFpX4+Hh9//33iouLK9D4cePGKTExMWtJSEgo5oQAPMaLL9qXPXpIV19tNgsAlyll6oUTEhI0cuRILV26VMEF3OEtKChIQUFBxZwMgMc5etTeeVaSHnnEbBYALmWsqGzatElHjhxR48aNs+5LT0/X119/rVdeeUUpKSny9/c3FQ+AJ5k2TTp3TmrSxD7iB4DXMFZU2rVrp23btjndN2jQINWuXVtjx46lpAAomKNHpZdesq8//rh9eDIAr2GsqEREROjaa691ui8sLEzlypXLcT8A5On556VTp6TGjaVevUynAeBixooKALjEww/bm326dWNtCuCFHJZlWaZDFFVSUpKioqKUmJioyMhI03EAAEABFObz2/g8KgBQJKmpphMAKAEUFQCe6f77pc6dpe3bTScBUIwoKgA8z65d9rwpS5ZIZ86YTgOgGFFUAHiep5+WMjKkm2+WmjY1nQZAMaKoAPAsa9ZI8+fb1ydONJsFQLGjqADwHGlp0gMP2NcHDZLq1zebB0Cxo6gA8Bz/+5/0ww9SmTLS5Mmm0wAoARQVAJ7BsqT337evx8VJ5cubzQOgRDAzLQDP4HBIK1dKc+dKAwaYTgOghFBUAHiOwEBp8GDTKQCUIDb9AHBvqanSa68xEy3goygqANzb889Lw4ZJHTva+6kA8CkUFQDua8MGe3I3SbrnHs6ODPggigoA95ScLN11lz13Su/e0t13m04EwACKCgD39NBD0s8/SzEx0owZrE0BfBRFBYD7iY+XZs+W/Pykd9+1J3gD4JMoKgDcS2qq9Nhj9vUnn5RatzabB4BRFBUA7iUwUFqxwj6nz/jxptMAMIwJ3wC4n1q1pFdfNZ0CgBtgjQoA9/DCC9Lnn5tOAcDNsEYFgHnz5kljxtg7z27dKl17relEANwEa1QAmPXNN9LAgfb1UaMoKQCcUFQAmPPTT1LPnvaRPr16Sf/5j+lEANwMRQWAGbt2SW3bSsePS02bSnPn2pt+AOAC7KMCoOT9/rv0979LR49K11wjffaZFBpqOhUAN8SfLwBKXpUq0u23S40aSV9/LVWsaDoRADfFGhUAJc/hkF55RTpzRgoPN50GgBtjjQqAkvHWW/ZZkM+ft2/7+VFSAFwSa1QAFK8zZ6Thw+2TDEr2uXseeshoJACeg6ICoPjs2WPvi7Jtm70GZdIk6cEHTacC4EEoKgBcz7Kk996zTyyYnCxVqCDFx9uHIwNAIbCPCgDXe/JJ6e677ZLSurW0eTMlBUCRUFQAuF7//lJkpDRxovTVV/bhyABQBGz6AXB5UlPtWWUPHpT++U/7vtq1pYQEu6wAwGWgqAAomjNnpJkz7fPz/PabPTdKly5S48b21ykpAFyAogKgcH75RZozR3r1VenPP+37KleWxoyx16QAgAtRVAAU3P/9n9S9e/bt2Fhp7FhpwAApONhcLgBei6ICICfLsudAWbxYKlPG3jlWktq0kSIipBtukAYNsmeaLcWvEQDFh98wAKS0NGnHDmn9emndOunLL6X9++2vNWqUXVTCw+39Udj/BEAJMXp48muvvaZ69eopMjJSkZGRat68uRYtWmQyEuDdLMs+Gue775zvr19fatBAuu8+6c037ZISGCi1by/17Ws/LhMlBUAJMrpGpWrVqnruued05ZVXSpLmzJmjnj17avPmzapbt67JaIBnSUuzJ1crUyb7vk8/tSdaO3DAXhIS7Mtz5+zNN4mJ9pE6knTttfaakiZNpKZNpZYtpRtvlMLCjLwdAMjksKwL/1Qyr2zZsvrPf/6je+6555Jjk5KSFBUVpcTEREW6+q+8M2eko0fz/nrZsvYve0k6e1Y6ciTvsWXKZP8VmpIiHT6c99ioKKl0aft6aqp06FDeYyMjsz+Yzp+357HIS0SEnVmS0tPtD6W8hIVJ0dH29YwM+wMuL6GhUvny9nXLsj8I8xISYk+lnunXX/MeGxwsVayYfTshwc6dm6Ag+6iTC8empWVnulBAgBQTk337wAH7+5w5NnO8Zdlja9bMHvvLL/a/dea4zCUjw95P47rrssdu3y6dPGl/LT09+zI93T7nTadO2WOXL8/OkZpq/1umpNiFIj1devbZ7LETJkjffCOdPm0Xk5Mn7cJx6pSdNyUlu3zcequ0YEHO71epUtKVV0pr1mT//Jw8af88+TEHJIDiV6jPb8tNpKWlWfPmzbMCAwOtHTt25Drm3LlzVmJiYtaSkJBgSbISExNdH+izzy7+OHJeXnste+zy5fmPff757LFr1+Y/duLE7LE//JD/2LFjs8f+/HP+Yx98MHvsoUP5jx08OHtsUlL+Y++8M3tsWlr+Y7t3d/4eBwXlPfamm5zHli2b99hmzZzHVq2a99hrr3Uee/XVeY+tWdN5bMOGeY+tWNF5bMuWeY+NjHQe27Fj3mP9/Z3H9uqV//f45Mnssa+9Zln33WdZkyZZ1ttvW9aKFZa1d69lpaZaAGBSYmJigT+/je9Mu23bNjVv3lznzp1TeHi4FixYoDp16uQ6Ni4uTs8880zJBPPzy/9wS3//go+98KgIh8PM2IAA59smxgYGOt8OCcn7L/igoJxjQ0JyjsvtfYeGOm+yyFzDkPm1C4WF2WubMsdcOPbill+mjL2myeHIXvz87MvMtUqZYmKkq66yv+bv77xcvDmlaVP7exMQYF9mLsHB9pKRkf19GjFCuu02+znCw+1MmWvhoqKc/z2GDs35/QIAD2N8009qaqoOHDigkydP6qOPPtLMmTO1atWqXMtKSkqKUlJSsm4nJSUpJiameDb9AACAYlGYTT/Gi8rF2rdvr1q1amnGjBmXHFus+6gAAIBiUZjPb7fbc86yLKe1JgAAwHcZ3UfliSeeUJcuXRQTE6Pk5GTFx8dr5cqVWrx4sclYAADATRgtKn/88Yf69eunQ4cOKSoqSvXq1dPixYvVoUMHk7EAAICbMFpU3nzzTZMvDwAA3Jzb7aMCAACQiaICAADcFkUFAAC4LYoKAABwWxQVAADgtigqAADAbVFUAACA26KoAAAAt0VRAQAAbsvozLSXK/PEz0lJSYaTAACAgsr83M78HM+PRxeV5ORkSVJMTIzhJAAAoLCSk5MVFRWV7xiHVZA646YyMjJ08OBBRUREyOFwuPS5k5KSFBMTo4SEBEVGRrr0ud0B78/zeft79Pb3J3n/e+T9eb7ieo+WZSk5OVlVqlSRn1/+e6F49BoVPz8/Va1atVhfIzIy0mt/ACXenzfw9vfo7e9P8v73yPvzfMXxHi+1JiUTO9MCAAC3RVEBAABui6KSh6CgID399NMKCgoyHaVY8P48n7e/R29/f5L3v0fen+dzh/fo0TvTAgAA78YaFQAA4LYoKgAAwG1RVAAAgNuiqAAAALdFUSmElJQUNWjQQA6HQ1u2bDEdx2VuvvlmVatWTcHBwapcubL69eungwcPmo7lMvv379c999yj2NhYhYSEqFatWnr66aeVmppqOprLPPvss2rRooVCQ0NVunRp03Fc4tVXX1VsbKyCg4PVuHFjrV692nQkl/n666/Vo0cPValSRQ6HQ5988onpSC4TFxenJk2aKCIiQhUqVNAtt9yi3bt3m47lUq+99prq1auXNQla8+bNtWjRItOxik1cXJwcDodGjRpl5PUpKoXw2GOPqUqVKqZjuFzbtm01f/587d69Wx999JH27t2r22+/3XQsl9m1a5cyMjI0Y8YM7dixQy+++KKmT5+uJ554wnQ0l0lNTVXv3r31wAMPmI7iEu+//75GjRqlJ598Ups3b1br1q3VpUsXHThwwHQ0lzh9+rTq16+vV155xXQUl1u1apWGDx+u7777TsuWLVNaWpo6duyo06dPm47mMlWrVtVzzz2njRs3auPGjbrpppvUs2dP7dixw3Q0l9uwYYNef/111atXz1wICwXyxRdfWLVr17Z27NhhSbI2b95sOlKx+fTTTy2Hw2GlpqaajlJspkyZYsXGxpqO4XKzZs2yoqKiTMe4bE2bNrWGDh3qdF/t2rWtxx9/3FCi4iPJWrBggekYxebIkSOWJGvVqlWmoxSrMmXKWDNnzjQdw6WSk5Otq666ylq2bJnVpk0ba+TIkUZysEalAP744w8NGTJE77zzjkJDQ03HKVbHjx/Xu+++qxYtWiggIMB0nGKTmJiosmXLmo6BXKSmpmrTpk3q2LGj0/0dO3bUmjVrDKVCUSUmJkqS1/5/S09PV3x8vE6fPq3mzZubjuNSw4cPV7du3dS+fXujOSgql2BZlgYOHKihQ4fq+uuvNx2n2IwdO1ZhYWEqV66cDhw4oE8//dR0pGKzd+9evfzyyxo6dKjpKMjFn3/+qfT0dFWsWNHp/ooVK+rw4cOGUqEoLMvS6NGj1apVK1177bWm47jUtm3bFB4erqCgIA0dOlQLFixQnTp1TMdymfj4eH3//feKi4szHcV3i8qECRPkcDjyXTZu3KiXX35ZSUlJGjdunOnIhVLQ95fp0Ucf1ebNm7V06VL5+/urf//+stx80uLCvkdJOnjwoDp37qzevXvr3nvvNZS8YIry/ryJw+Fwum1ZVo774N4efPBB/fDDD5o3b57pKC539dVXa8uWLfruu+/0wAMPaMCAAfrxxx9Nx3KJhIQEjRw5UnPnzlVwcLDpOL47hf6ff/6pP//8M98xNWrU0D/+8Q8tXLjQ6Rdkenq6/P391bdvX82ZM6e4oxZJQd9fbj+Ev/32m2JiYrRmzRq3XpVZ2Pd48OBBtW3bVs2aNdPs2bPl5+fePb0o/4azZ8/WqFGjdPLkyWJOV3xSU1MVGhqqDz74QL169cq6f+TIkdqyZYtWrVplMJ3rORwOLViwQLfccovpKC41YsQIffLJJ/r6668VGxtrOk6xa9++vWrVqqUZM2aYjnLZPvnkE/Xq1Uv+/v5Z96Wnp8vhcMjPz08pKSlOXytupUrsldxMdHS0oqOjLzlu2rRpmjRpUtbtgwcPqlOnTnr//ffVrFmz4ox4WQr6/nKT2V1TUlJcGcnlCvMef//9d7Vt21aNGzfWrFmz3L6kSJf3b+jJAgMD1bhxYy1btsypqCxbtkw9e/Y0mAwFYVmWRowYoQULFmjlypU+UVIk+327++/MgmrXrp22bdvmdN+gQYNUu3ZtjR07tkRLiuTDRaWgqlWr5nQ7PDxcklSrVi1VrVrVRCSXWr9+vdavX69WrVqpTJky+uWXXzR+/HjVqlXLrdemFMbBgwd14403qlq1anr++ed19OjRrK9VqlTJYDLXOXDggI4fP64DBw4oPT09a56fK6+8Mutn1pOMHj1a/fr10/XXX6/mzZvr9ddf14EDB7xmv6JTp07p559/zrq9b98+bdmyRWXLls3xO8fTDB8+XO+9954+/fRTRUREZO1XFBUVpZCQEMPpXOOJJ55Qly5dFBMTo+TkZMXHx2vlypVavHix6WguERERkWOfosx9GI3sa2TkWCMPtm/fPq86PPmHH36w2rZta5UtW9YKCgqyatSoYQ0dOtT67bffTEdzmVmzZlmScl28xYABA3J9fytWrDAdrcj+97//WdWrV7cCAwOtRo0aedXhrStWrMj132vAgAGmo122vP6vzZo1y3Q0lxk8eHDWz2b58uWtdu3aWUuXLjUdq1iZPDzZZ/dRAQAA7s/9N9QDAACfRVEBAABui6ICAADcFkUFAAC4LYoKAABwWxQVAADgtigqAADAbVFUAACA26KoAF7oxhtv1KhRo0zHyNWxY8dUoUIF7d+/X5K0cuVKORyOYj+RYlFfZ/bs2SpdunShHtOkSRN9/PHHhXoMgNxRVABc0qFDh3TXXXfp6quvlp+fX54l6KOPPlKdOnUUFBSkOnXqaMGCBTnGxMXFqUePHqpRo0bxhjboqaee0uOPP66MjAzTUQCPR1EBcEkpKSkqX768nnzySdWvXz/XMWvXrtWdd96pfv36aevWrerXr5/uuOMOrVu3LmvM2bNn9eabb+ree+8tqehGdOvWTYmJiVqyZInpKIDHo6gAXu7EiRPq37+/ypQpo9DQUHXp0kU//fST05g33nhDMTExCg0NVa9evTR16lSnzR01atTQf//7X/Xv319RUVG5vs5LL72kDh06aNy4capdu7bGjRundu3a6aWXXsoas2jRIpUqVSrfM3MfO3ZMffr0UdWqVRUaGqrrrrtO8+bNcxpz4403asSIERo1apTKlCmjihUr6vXXX9fp06c1aNAgRUREqFatWlq0aFGO5//2229Vv359BQcHq1mzZjlOZz979mxVq1Yt63tx7Ngxp6/v3btXPXv2VMWKFRUeHq4mTZpo+fLlTmP8/f3VtWvXHLkBFB5FBfByAwcO1MaNG/XZZ59p7dq1sixLXbt21fnz5yXZH9xDhw7VyJEjtWXLFnXo0EHPPvtsoV9n7dq16tixo9N9nTp10po1a7Juf/3117r++uvzfZ5z586pcePG+vzzz7V9+3bdd9996tevn9OaGUmaM2eOoqOjtX79eo0YMUIPPPCAevfurRYtWuj7779Xp06d1K9fP505c8bpcY8++qief/55bdiwQRUqVNDNN9+c9b1Yt26dBg8erGHDhmnLli1q27atJk2a5PT4U6dOqWvXrlq+fLk2b96sTp06qUePHjpw4IDTuKZNm2r16tUF++YByJuRczYDKFaZp2Tfs2ePJcn69ttvs772559/WiEhIdb8+fMty7KsO++80+rWrZvT4/v27WtFRUXl+9wXCwgIsN59912n+959910rMDAw63bPnj2twYMHO41ZsWKFJck6ceJEnu+na9eu1iOPPOKUoVWrVlm309LSrLCwMKtfv35Z9x06dMiSZK1du9bpdeLj47PGHDt2zAoJCbHef/99y7Isq0+fPlbnzp2dXvvOO+/M83uRqU6dOtbLL7/sdN+nn35q+fn5Wenp6fk+FkD+WKMCeLGdO3eqVKlSatasWdZ95cqV09VXX62dO3dKknbv3q2mTZs6Pe7i2wXlcDicbluW5XTf2bNnFRwcnO9zpKen69lnn1W9evVUrlw5hYeHa+nSpTnWWNSrVy/rur+/v8qVK6frrrsu676KFStKko4cOeL0uAs3O5UtW9bpe7Fz584cm6Uuvn369Gk99thjqlOnjkqXLq3w8HDt2rUrR76QkBBlZGQoJSUl3/cLIH+lTAcAUHwsy8rz/swCcXGZyO9x+alUqZIOHz7sdN+RI0eyCoMkRUdH68SJE/k+zwsvvKAXX3xRL730kq677jqFhYVp1KhRSk1NdRoXEBDgdNvhcDjdl/meCnLkzYXfi0t59NFHtWTJEj3//PO68sorFRISottvvz1HvuPHjys0NFQhISGXfE4AeWONCuDF6tSpo7S0NKf9O44dO6Y9e/bommuukSTVrl1b69evd3rcxo0bC/1azZs317Jly5zuW7p0qVq0aJF1u2HDhvrxxx/zfZ7Vq1erZ8+euvvuu1W/fn3VrFkzx86/l+O7777Lun7ixAnt2bNHtWvXlmR/vy78+sXjM/MNHDhQvXr10nXXXadKlSplzQlzoe3bt6tRo0Yuyw34KooK4MWuuuoq9ezZU0OGDNE333yjrVu36u6779YVV1yhnj17SpJGjBihL774QlOnTtVPP/2kGTNmaNGiRTnWsmzZskVbtmzRqVOndPToUW3ZssWpdIwcOVJLly7V5MmTtWvXLk2ePFnLly93mnOlU6dO2rFjR75rVa688kotW7ZMa9as0c6dO3X//ffnWFNzOSZOnKgvv/xS27dv18CBAxUdHa1bbrlFkvTQQw9p8eLFmjJlivbs2aNXXnlFixcvzpHv448/1pYtW7R161bdddddua61Wb16dY6diwEUHkUF8HKzZs1S48aN1b17dzVv3lyWZemLL77I2kzSsmVLTZ8+XVOnTlX9+vW1ePFiPfzwwzn2JWnYsKEaNmyoTZs26b333lPDhg3VtWvXrK+3aNFC8fHxmjVrlurVq6fZs2fr/fffd9o/5rrrrtP111+v+fPn55n3qaeeUqNGjdSpUyfdeOONqlSpUlaRcIXnnntOI0eOVOPGjXXo0CF99tlnCgwMlCTdcMMNmjlzpl5++WU1aNBAS5cu1T//+U+nx7/44osqU6aMWrRooR49eqhTp0451pz8/vvvWrNmjQYNGuSy3ICvclhF2RgNwKsNGTJEu3btKpbDa7/44guNGTNG27dvl5+fd/6t9OijjyoxMVGvv/666SiAx2NnWgB6/vnn1aFDB4WFhWnRokWaM2eOXn311WJ5ra5du+qnn37S77//rpiYmGJ5DdMqVKigMWPGmI4BeAXWqADQHXfcoZUrVyo5OVk1a9bUiBEjNHToUNOxAICiAgAA3Jd3biAGAABegaICAADcFkUFAAC4LYoKAABwWxQVAADgtigqAADAbVFUAACA26KoAAAAt/X/J+Yjt6Z/fRcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "%matplotlib inline\n", "\n", @@ -1852,9 +1612,7 @@ { "cell_type": "markdown", "id": "e18f8b68", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see here that we reach a plateau. What is actually happening?" ] @@ -1862,22 +1620,237 @@ { "cell_type": "markdown", "id": "c3a12d7d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## With Lasso Regression" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "9b43226e", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2. 2.]\n", + "Training MSE for OLS\n", + "3.0\n", + "[1.99995 1.99980002]\n", + "[1.999925 1.9997 ]\n", + "[1.99993978 1.99975913]\n", + "[1.99990966 1.99963865]\n", + "[1.99992746 1.99970988]\n", + "[1.99989119 1.99956475]\n", + "[1.99991263 1.99965056]\n", + "[1.99986894 1.99947574]\n", + "[1.99989476 1.99957911]\n", + "[1.99984213 1.99936853]\n", + "[1.99987324 1.99949306]\n", + "[1.99980985 1.99923939]\n", + "[1.99984732 1.99938942]\n", + "[1.99977096 1.99908384]\n", + "[1.9998161 1.99926459]\n", + "[1.99972412 1.99889649]\n", + "[1.99977849 1.99911427]\n", + "[1.9996677 1.99867081]\n", + "[1.9997332 1.99893323]\n", + "[1.99959975 1.99839899]\n", + "[1.99967865 1.99871521]\n", + "[1.99951789 1.99807158]\n", + "[1.99961294 1.99845267]\n", + "[1.9994193 1.99767721]\n", + "[1.99953381 1.99813653]\n", + "[1.99930055 1.99720219]\n", + "[1.9994385 1.99775587]\n", + "[1.99915751 1.99663003]\n", + "[1.9993237 1.99729756]\n", + "[1.99898521 1.99594086]\n", + "[1.99918546 1.9967458 ]\n", + "[1.99877769 1.99511075]\n", + "[1.99901896 1.99608161]\n", + "[1.99852772 1.99411088]\n", + "[1.99881845 1.99528218]\n", + "[1.99822663 1.99290653]\n", + "[1.998577 1.9943201]\n", + "[1.99786397 1.99145589]\n", + "[1.99828624 1.99316252]\n", + "[1.99742715 1.98970859]\n", + "[1.99793613 1.99176998]\n", + "[1.99690099 1.98760396]\n", + "[1.99751458 1.99009525]\n", + "[1.99626723 1.98506893]\n", + "[1.99700706 1.98808176]\n", + "[1.99550387 1.98201547]\n", + "[1.9963961 1.98566191]\n", + "[1.99458439 1.97833757]\n", + "[1.99566069 1.98275501]\n", + "[1.99347688 1.97390753]\n", + "[1.9947756 1.97926491]\n", + "[1.99214288 1.96857153]\n", + "[1.99371056 1.97507735]\n", + "[1.99053607 1.96214429]\n", + "[1.99242921 1.97005689]\n", + "[1.98860067 1.95440267]\n", + "[1.99088801 1.9640435 ]\n", + "[1.98626946 1.94507785]\n", + "[1.9890348 1.95684892]\n", + "[1.98346152 1.93384608]\n", + "[1.98680716 1.9482527 ]\n", + "[1.98007934 1.92031737]\n", + "[1.98413059 1.93799826]\n", + "[1.9760055 1.90402199]\n", + "[1.98091621 1.92578916]\n", + "[1.97109854 1.88439414]\n", + "[1.97705827 1.91128596]\n", + "[1.96518808 1.86075233]\n", + "[1.97243128 1.89410423]\n", + "[1.95806892 1.83227569]\n", + "[1.96688672 1.87381451]\n", + "[1.94949387 1.79797548]\n", + "[1.96024953 1.84994524]\n", + "[1.93916519 1.75666075]\n", + "[1.95231424 1.82198978]\n", + "[1.92672425 1.70689701]\n", + "[1.94284104 1.78941903]\n", + "[1.9117391 1.64695641]\n", + "[1.93155188 1.75170092]\n", + "[1.89368944 1.57475775]\n", + "[1.91812702 1.70832814]\n", + "[1.87194855 1.48779421]\n", + "[1.90220243 1.65885453]\n", + "[1.84576158 1.38304631]\n", + "[1.88336879 1.60293962]\n", + "[1.81421927 1.25687709]\n", + "[1.86117291 1.54039921]\n", + "[1.77622646 1.10490583]\n", + "[1.83512277 1.47125748]\n", + "[1.73046398 0.9218559 ]\n", + "[1.80469739 1.39579407]\n", + "[1.6753429 0.70137162]\n", + "[1.76936315 1.31457796]\n", + "[1.60894938 0.43579751]\n", + "[1.72859758 1.22847924]\n", + "[1.52897814 0.11591257]\n", + "[1.68192193 1.13865173]\n", + "[1.4326525 0. ]\n", + "[1.62894215 1.04648335]\n", + "[1.31662793 0. ]\n", + "[1.56939714 0.95351665]\n", + "[1.17687593 0. ]\n", + "[1.50321091 0.86134827]\n", + "[1.00854414 0. ]\n", + "[1.43054282 0.77152076]\n", + "[0.8057879 0. ]\n", + "[1.35182854 0.68542204]\n", + "[0.5615673 0. ]\n", + "[1.26780278 0.60420593]\n", + "[0.26740272 0. ]\n", + "[1.17949575 0.52874252]\n", + "[0. 0.]\n", + "[1.0881981 0.45960079]\n", + "[0. 0.]\n", + "[0.99539415 0.39706038]\n", + "[0. 0.]\n", + "[0.90266948 0.34114547]\n", + "[0. 0.]\n", + "[0.81160425 0.29167186]\n", + "[0. 0.]\n", + "[0.7236674 0.24829908]\n", + "[0. 0.]\n", + "[0.64012627 0.21058097]\n", + "[0. 0.]\n", + "[0.56198284 0.17801022]\n", + "[0. 0.]\n", + "[0.48994188 0.15005476]\n", + "[0. 0.]\n", + "[0.42441033 0.12618549]\n", + "[0. 0.]\n", + "[0.3655222 0.10589577]\n", + "[0. 0.]\n", + "[0.31318084 0.08871404]\n", + "[0. 0.]\n", + "[0.26710969 0.07421084]\n", + "[0. 0.]\n", + "[0.22690428 0.06200174]\n", + "[0. 0.]\n", + "[0.19207979 0.0517473 ]\n", + "[0. 0.]\n", + "[0.16211139 0.04315108]\n", + "[0. 0.]\n", + "[0.13646574 0.0359565 ]\n", + "[0. 0.]\n", + "[0.11462415 0.02994311]\n", + "[0. 0.]\n", + "[0.09609807 0.02492265]\n", + "[0. 0.]\n", + "[0.08043851 0.02073509]\n", + "[0. 0.]\n", + "[0.06724062 0.01724499]\n", + "[0. 0.]\n", + "[0.05614483 0.01433809]\n", + "[0. 0.]\n", + "[0.04683565 0.01191824]\n", + "[0. 0.]\n", + "[0.039039 0.00990475]\n", + "[0. 0.]\n", + "[0.03251863 0.00823002]\n", + "[0. 0.]\n", + "[0.02707227 0.00683748]\n", + "[0. 0.]\n", + "[0.02252765 0.0056799 ]\n", + "[0. 0.]\n", + "[0.01873869 0.00471782]\n", + "[0. 0.]\n", + "[0.01558197 0.00391839]\n", + "[0. 0.]\n", + "[0.01295356 0.0032542 ]\n", + "[0. 0.]\n", + "[0.01076611 0.00270244]\n", + "[0. 0.]\n", + "[0.00894639 0.00224413]\n", + "[0. 0.]\n", + "[0.0074331 0.00186347]\n", + "[0. 0.]\n", + "[0.00617499 0.00154733]\n", + "[0. 0.]\n", + "[0.00512927 0.00128479]\n", + "[0. 0.]\n", + "[0.00426027 0.00106677]\n", + "[0. 0.]\n", + "[0.00353823 0.00088573]\n", + "[0. 0.]\n", + "[0.00293838 0.00073541]\n", + "[0. 0.]\n", + "[0.0024401 0.00061058]\n", + "[0. 0.]\n", + "[0.00202624 0.00050694]\n", + "[0. 0.]\n", + "[0.00168251 0.00042089]\n", + "[0. 0.]\n", + "[0.00139705 0.00034944]\n", + "[0. 0.]\n", + "[0.00115999 0.00029012]\n", + "[0. 0.]\n", + "[0.00096314 0.00024087]\n", + "[0. 0.]\n", + "[0.00079968 0.00019998]\n", + "[0. 0.]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAGwCAYAAACHJU4LAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVRklEQVR4nO3dd3hT1RsH8G+6dxmllNFCAWXJHhVEhmwqVkBAQLYoQwRRhKJgHYjwQ0BBZEorIIgsESkUZMpeRWTvvUfaQmlpe35/HJM2dNCUpCfj+3me+9ybm5N734SWvD1TI4QQICIiIrJADqoDICIiIsoOExUiIiKyWExUiIiIyGIxUSEiIiKLxUSFiIiILBYTFSIiIrJYTFSIiIjIYjmpDuBZpKWl4erVq/D29oZGo1EdDhEREeWCEALx8fEoXrw4HBxyrjOx6kTl6tWrCAwMVB0GERER5cGlS5dQsmTJHMtYdaLi7e0NQL5RHx8fxdEQERFRbsTFxSEwMFD/PZ4Tq05UdM09Pj4+TFSIiIisTG66bbAzLREREVksJipERERksZioEBERkcWy6j4quZWamorHjx+rDoPsjLOzMxwdHVWHQURk1Ww6URFC4Pr167h//77qUMhOFShQAAEBAZznh4goj2w6UdElKf7+/vDw8OCXBeUbIQQePnyImzdvAgCKFSumOCIiIutks4lKamqqPkkpXLiw6nDIDrm7uwMAbt68CX9/fzYDERHlgc12ptX1SfHw8FAcCdkz3c8f+0gREeWNzSYqOmzuIZX480dE9GxsPlEhIiIi68VEhYiIiCwWExV6JufPn4dGo0FsbGy2ZTZv3gyNRmOVw8R79eqF119/XXUYRER2y2ZH/VizXr16ISoqCu+++y5mzJhh8NzAgQPx448/omfPnoiMjAQgR5WMHj0a0dHRuHHjBgoWLIhq1aohIiIC9erVAwCULl0aFy5cyHSvcePGYeTIkVnG0bhxY2zZsgWAnLwsMDAQnTp1QkREBFxdXQEAgYGBuHbtGvz8/Ez19vMku/en06hRI2zevNno63733XcQQjxDZGRVrl8HkpKyfs7BAQgMTH984wbw6FH21ypVKv345k0gMTH7skFBgK4/061bwMOH2ZcNDJSxAMCdO0BCQvZlS5QAnP77b/7uXSA+PvuyxYsDzs7y+N49IC4u+7LFigEuLvJYqwVy+iOkaFHAzU0ex8XJa2fH3x/4b7Qc4uNlzNkpUgTQDZZISJCfRXb8/ABPT3n88KH8jLNTqBCgW9E3MVH+22WnYEFAtyDuo0fyZyI7BQoAvr7yODkZuHYt+7I+PvLaAPD4MXD1avZlvb1lzACQkgJcuZJ9WS8vQDcKNjUVuHw5+7KenvJzswTCimm1WgFAaLXaTM8lJiaKo0ePisTERAWRPZuePXuKwMBA4evrKx4+fKg/n5iYKAoUKCCCgoJEz5499ecbNGggQkJCxMaNG8X58+fF7t27xddffy1Wr16tL1OqVCnxxRdfiGvXrhlsCQkJ2cbRqFEj0a9fP3Ht2jVx4cIFsXTpUuHt7S1Gjhxp1PvZtGmTACDu3btn1OuMcfPmTf17WrZsmQAgTpw4oT93584dg/LJyclmiyUja/45tEv16gkBZL0VKGBYtlmz7Ms6OxuWfe217MsCQmT8eezSJeey9++nl3377ZzLXrmSXvb993Mue/Jketnw8JzLxsaml/3qq5zLbt+eXnbSpJzLxsSkl505M+eyK1eml12wIOeyCxeml12+POeys2all123LueyU6akl/3775zLjh2bXvbAgZzLjholy127JsScOTmXHTpUlj10SIiePXMu26+fLLtnjxC1a+dctmtXYU45fX8/yT6bfh48yH578i+knMo++RdSVmXyqGbNmggKCsLy5cv155YvX47AwEDUqFFDf+7+/fv4+++/MX78eDRp0gSlSpVC3bp1ER4ejtDQUINrent7IyAgwGDz1P2VkQ0PDw8EBAQgKCgIHTp0QPPmzRETE6N/PqumnzVr1uD555+Hu7s7mjRpgvPnz2e67uzZsxEYGAgPDw+0a9cOkyZNQoECBQzK/PHHH6hVqxbc3NxQpkwZfP7550hJSckyziJFiujfU6H//rrw9/fXnytcuDBmzJiBsLAweHp64quvvkJqair69u2L4OBguLu7o3z58vjuu+8Mrvtk00/jxo3x/vvv4+OPP0ahQoUQEBCAiIiIHD9DsmAtWwKNGwOnT8vHLi7yr//stozMVdbZWU3ZjCPUnJxMV9Yhw9eMo6OashnnMHJwMH1ZIeR3h6ur3Fxc5Oft7Cw/H1fX9JqtP/4A3nrL8DN88mdFV3bTJuDtt7Mup/s31dWCXbwIREXlrmxCArBvX/ZlnZzSy1oCs6ZMZpbnGpWcssg2bQzLenhkX7ZRI8Oyfn6Zy+RBz549RVhYmJg0aZJo2rSp/nzTpk3F5MmTRVhYmL5G5fHjx8LLy0sMHTpUPHr0KNtrlipVSkyePNmoOBo1aiSGDBmifxwbGyuKFi0qQkJC9OfOnTsnAIiDBw8KIYS4ePGicHV1FUOGDBHHjx8XCxYsEEWLFhVAeo3K33//LRwcHMT//vc/ceLECfHDDz+IQoUKCV9fX/11165dK3x8fERkZKQ4c+aMiImJEaVLlxYRERFPjTurGhwAwt/fX8ydO1ecOXNGnD9/XiQnJ4sxY8aIPXv2iLNnz4oFCxYIDw8P8euvv+pfp/u3yPiZ+Pj4iIiICHHy5EkRFRUlNBqNiMn4l2AGrFGxcF5e8vf01CnVkZClevRIiNOnhdi0SYiff5Y1IwMGCHH8eHqZ77/Pfe3PwoU5l/3ll/SyGzYIUa2aEC++KETjxkK0aiXE668L0bmzED16CLF1a3rZEyeEGD1aiC+/FOKbb4T49lsZ148/ylqZo0fTy966JWP64w8h1qyRNVl//SXE5s1CbNsmxOXL5vo09YypUWGiYsGJyq1bt4Srq6s4d+6cOH/+vHBzcxO3bt0ySFSEEGLp0qWiYMGCws3NTdSvX1+Eh4eLQ4cOGVyzVKlSwsXFRXh6ehpsmzZtyjaORo0aCWdnZ+Hp6SlcXFwEAOHg4CCWLl2qL/NkohIeHi4qVqwo0tLS9GVGjBhhkDh07txZhIaGGtyrW7duBonKyy+/LL7++muDMvPnzxfFihV76ueXXaIyVFdFmoOBAweKDh066B9nlag0aNDA4DV16tQRI0aMyPJ6TFQsWHJy+u/pE02DZGceP5aJR1xc+rlffhEiMFAIjSbr//9XrEgv++uv8pybm3xNjRqyabBTJyHeeUeIffvSy16+LBOELVtkE9CpU7KJJy5OiJSUfHvLqhmTqNhnZ9qcOp89Oc15Th2pHJ5oOcuiieNZ+Pn5ITQ0FFFRURBCIDQ0NMtOqx06dEBoaCi2bduGnTt3Yu3atZgwYQLmzJmDXr166csNHz7c4DEAlChRIscYunXrhk8++QRxcXEYP348fHx80KFDh2zLHzt2DC+++KLBRGe6Dr06J06cQLt27QzO1a1bF6tXr9Y/3r9/P/bu3YuxY8fqz6WmpuLRo0d4+PBhnmYcrl27dqZzM2bMwJw5c3DhwgUkJiYiOTkZ1atXz/E6VatWNXhcrFgx/Zo+ZEUydurUdXIk23fnDnDggNwOHgSOHAFOnJCdVn//HXjtNVnOxQW4dEkeu7nJDs+BgXIrWRJ47rn0a4aFye+VpzSlA5AdnJ/y/y4Zss9EJTc/TOYum0t9+vTBe++9BwD44Ycfsi3n5uaG5s2bo3nz5hgzZgzefvttfPbZZwaJiZ+fH8qVK2fU/X19ffWvWbBgASpXroy5c+eib9++WZYXQjz1mkKITDO2Pvm6tLQ0fP7552jfvn2m17s92faeS0/2x1myZAk++OADfPvtt6hXrx68vb3xv//9D7t3787xOs5PtN1qNBqkpaXlKSZSSJeo+Ppm/gOFbENqqhxhoxtJtHQp0LFj1mU9PIDbt9MfN24M7NgBlCkjRyPlNMu0rm8KmYV9JipWpFWrVkhOTgYAtGzZMtevq1SpElauXGnSWJydnTFq1CiEh4ejS5cuWdZqZHXfXbt2GTyuUKEC9uzZY3Bu3xMdu2rWrIkTJ04YnVgZY9u2bahfvz4GDhyoP3fmzBmz3Y8sjG7oq25oJ1k/IYDjx4G1a4GYGGD7dmDsWGDwYPl8pUpyX64cULOm3KpWleczDvsG5DDeJ2qDSQ0mKhbO0dERx44d0x8/6c6dO+jYsSP69OmDqlWrwtvbG/v27cOECRMQFhZmUDY+Ph7Xr183OOfh4QEf3TwAudC1a1eMGjUK06dPx0cffZTp+f79++Pbb7/FsGHD8O6772L//v36+V50Bg8ejIYNG2LSpElo27YtNm7ciOjoaINaljFjxuDVV19FYGAgOnbsCAcHB/zzzz84fPgwvvrqq1zHm5Ny5crh559/xrp16xAcHIz58+dj7969CA4ONsn1ycLpalR081WQdUpOBtatA1avlgnKxYuGz+/enZ6oVKgg/92fGGFIls0+hydbGR8fn2yTCS8vL4SEhGDy5Mlo2LAhXnjhBYwePRr9+vXDtGnTDMqOGTMGxYoVM9g+/vhjo2JxcXHBe++9hwkTJiAhi74+QUFBWLZsGf744w9Uq1YNM2bMwNdff21Q5qWXXsKMGTMwadIkVKtWDWvXrsUHH3xg0KTTsmVLrF69GuvXr0edOnXw4osvYtKkSSiVcRKtZ9S/f3+0b98enTt3RkhICO7cuWNQu0I2LiVFVukXLao6EnoW8fFAu3bArFkySXF1BVq0ACZNkn1QMg7ZdXBgkmKFNCI3nQosVFxcHHx9faHVajN9kT969Ajnzp1DcHBwnvs0UP7p168fjh8/jm3btqkOxaT4c0hkQv/+C8ycCVy4AKxalX6+d2/ZR7BNG9m3JA+d7Sl/5fT9/SQ2/ZASEydORPPmzeHp6Yno6GhERUVh+vTpqsMiIkuTmAj89ptMUHbsSD9/+bIcfQMA8+apiY3yBRMVUmLPnj2YMGEC4uPjUaZMGXz//fd4O6cZGInIvty/D/zwAzBlSvpoHCcnORT4nXfkekNkF5iokBJLlixRHQLZu9GjgW3bgCFDZB8HsixLlgCffiqPS5WSyUnv3kxQ7BA70xKRfYqNBbZsyXnVXco/8fHA0aPpj3v0AJo1AxYulGsxjRrFJMVOsUaFiOwThydbhrQ0OTJn1CjAx0d2mNUtoLh+veroyAKwRoWI7JNuwjcmKups2wbUrg306QNcvy6TlifnQSG7x0SFiOwTa1TUSUgABg0CGjaUc534+AATJ8p1d8qWVR0dWRg2/RCR/RGCU+ircu0a0KABcPasfNyvn5zmvkgRtXGRxWKNClE2IiMjUYCzWNqmxEQ59TrAGpX8FhAgVx4OCgI2bJAzyjJJoRwwUbFAvXr1gkajQf/+/TM9N3DgQGg0GoNVkW/evIl3330XQUFBcHV1RUBAAFq2bImdO3fqy5QuXRoajSbT9s0332QbR+PGjTF06FBTvjWz0H1eOW150blzZ5w8edLE0ZJFiI+X0+e7uwPe3qqjsX0XLgCPHsljjUZ2nj18GGjaVG1cZBWYqFiowMBALF68GImJifpzjx49wqJFixAUFGRQtkOHDjh06BCioqJw8uRJrFq1Co0bN8ZdXdX2f7744gtcu3bNYBusW6zLin333XcG7wkA5s2bl+mcjm416qdxd3eHv7+/yeMlC1C0KHDjBvDggfziJPPZsEGuUty/v2xyA+Tnb8RiqGTfmKhYqJo1ayIoKAjLly/Xn1u+fDkCAwNRo0YN/bn79+/j77//xvjx49GkSROUKlUKdevWRXh4OEJDQw2u6e3tjYCAAIPN09MzzzGOGDECzz//PDw8PFCmTBmMHj0ajx8/1j9/6NAhNGnSBN7e3vDx8UGtWrWwb98+AMCFCxfQtm1bFCxYEJ6enqhcuTLWrFmjf+2WLVtQt25duLq6olixYhg5ciRSUlKyjMPX19fgPQFAgQIF9I/ffPNNvPfeexg2bBj8/PzQvHlzAMCkSZNQpUoVeHp6IjAwEAMHDjRYaPHJpp+IiAhUr14d8+fPR+nSpeHr64s333wT8fHxef4MSTEmKeYjhOwg27Kl7A909KjsREtkJPtMVB48yH7TVU/mpmyG2o5syz6D3r17Y16GNSx++ukn9OnTx6CMl5cXvLy8sHLlSiQlJT3T/Yzl7e2NyMhIHD16FN999x1mz56NyZMn65/v1q0bSpYsib1792L//v0YOXIknJ2dAQCDBg1CUlIStm7disOHD2P8+PHw8vICAFy5cgVt2rRBnTp1cOjQIfz444+YO3cuvvrqqzzHGhUVBScnJ2zfvh0zZ84EADg4OOD777/Hv//+i6ioKGzcuPGpq0mfOXMGK1euxOrVq7F69Wps2bIlx+YzIrv06BHw1lvA8OFyyHHv3sDWrWxmo7wRVkyr1QoAQqvVZnouMTFRHD16VCQmJmZ+ocz1s97atDEs6+GRfdlGjQzL+vllLpMHPXv2FGFhYeLWrVvC1dVVnDt3Tpw/f164ubmJW7duibCwMNGzZ099+aVLl4qCBQsKNzc3Ub9+fREeHi4OHTpkcM1SpUoJFxcX4enpabBt2rQp2zgaNWokhgwZkuu4J0yYIGrVqqV/7O3tLSIjI7MsW6VKFREREZHlc6NGjRLly5cXaWlp+nM//PCD8PLyEqmpqU+NA4BYsWKF/nGjRo1E9erVn/q6JUuWiMKFC+sfz5s3T/j6+uoff/bZZ8LDw0PExcXpzw0fPlyEhIRke80cfw5Jnd9/l7+/X3yhOhLbEx8vxCuvyP//nJyEmDZNiAy/y0RC5Pz9/SQOT7Zgfn5+CA0NRVRUFIQQCA0NhZ+fX6ZyHTp0QGhoKLZt24adO3di7dq1mDBhAubMmWPQ6Xb48OEGjwGgRIkSeY5v6dKlmDJlCk6fPo2EhASkpKQYLNc9bNgwvP3225g/fz6aNWuGjh07oux/cyS8//77GDBgAGJiYtCsWTN06NABVatWBQAcO3YM9erVM+gE+9JLLyEhIQGXL1/O1EcnN2rXrp3p3KZNm/D111/j6NGjiIuLQ0pKCh49eoQHDx5k2yRWunRpeGf4q7BYsWK4efOm0fGQYqdPy+nzn+Hnn7IghFw3aeNGwMsLWLUKaNJEdVRk5eyz6SchIftt2TLDsjdvZl82Otqw7Pnzmcs8oz59+iAyMhJRUVGZmn0ycnNzQ/PmzTFmzBjs2LEDvXr1wmeffWZQxs/PD+XKlTPY3N3d8xTXrl278Oabb6J169ZYvXo1Dh48iE8++cSgo2pERASOHDmC0NBQbNy4EZUqVcKKFSsAAG+//TbOnj2L7t274/Dhw6hduzamTp0KABBCZBqpI/7rhJfXETxPJh4XLlxAmzZt8MILL2DZsmXYv38/fvjhBwAw6GfzJF3TlY5Go0FaWlqeYiKFONmbeWg0wMiRQPHiwF9/MUkhk7DPRMXTM/vNzS33ZZ/8ks+qzDNq1aoVkpOTkZycjJYtW+b6dZUqVcKDZ+wjk5Pt27ejVKlS+OSTT1C7dm0899xzuHDhQqZyzz//PD744APExMSgffv2Bn1uAgMD0b9/fyxfvhwffvghZs+erY99x44d+uQEAHbs2AFvb+9nqgHKaN++fUhJScG3336LF198Ec8//zyuXr1qkmuTFWCiYloZflfRtKmssapbV108ZFPY9GPhHB0dcezYMf3xk+7cuYOOHTuiT58+qFq1Kry9vbFv3z5MmDABYWFhBmXj4+Nx/fp1g3MeHh4GzTVPunXrFmJjYw3OBQQEoFy5crh48SIWL16MOnXq4M8//9TXlgBAYmIihg8fjjfeeAPBwcG4fPky9u7diw4dOgAAhg4ditatW+P555/HvXv3sHHjRlSsWBGAnCtmypQpGDx4MN577z2cOHECn332GYYNGwYHB9Pk1mXLlkVKSgqmTp2Ktm3bYvv27ZgxY4ZJrk1WgLPSms6NG8DrrwM//QT89zuc6Y84omdgnzUqVsbHxyfbZMLLywshISGYPHkyGjZsiBdeeAGjR49Gv379MG3aNIOyY8aMQbFixQy2p41y+eWXX1CjRg2DbcaMGQgLC8MHH3yA9957D9WrV8eOHTswevRo/escHR1x584d9OjRA88//zw6deqE1q1b4/PPPwcApKamYtCgQahYsSJatWqF8uXLY/r06QBkv5k1a9Zgz549qFatGvr374++ffvi008/fZaP0UD16tUxadIkjB8/Hi+88AIWLlyIcePGmez6ZOFYo2Ia8fFAmzbArl3Au++qjoZslEZkrF+3MnFxcfD19YVWq830Rf7o0SOcO3cOwcHBcHuyOYcon/Dn0EKFhAB79gC//w689prqaKxTcjIQGiondCtSBNixAyhXTnVUZCVy+v5+EmtUiMj+ODoCzs6sUcmrtDSgVy+ZpHh6AmvWMEkhs1GaqMTHx2Po0KEoVaoU3N3dUb9+fezdu1dlSERkD3bsAJKS5Cq+ZBwhgA8/BBYtApycgOXLgSyG/xOZitJE5e2338b69esxf/58HD58GC1atECzZs1w5coVlWERkT3QaDiFfl789BMwZYo8jowEWrRQGQ3ZAWWJSmJiIpYtW4YJEyagYcOGKFeuHCIiIhAcHIwff/xRVVhERJST114DmjcHvvkG6NZNdTRkB5QNT05JSUFqamqmDobu7u74+++/s3xNUlKSwXo2cXFxT72PFfcVJhvAnz8LdPUq0LUrUKyYbL4g4xQpIie7NNFUAURPo+wnzdvbG/Xq1cOXX36Jq1evIjU1FQsWLMDu3btx7dq1LF8zbtw4+Pr66rfAwMBsr6+bQfThw4dmiZ8oN3Q/f0/OaEsK3bghp8/fskV1JNYjNdVwJm5HRzabUb5ROuHb/Pnz0adPH5QoUQKOjo6oWbMmunbtigMHDmRZPjw8HMOGDdM/jouLyzZZcXR0RIECBfTrsHh4eOR5+nUiYwkh8PDhQ9y8eRMFChTIcrI+UkQ32RtH/OTeF1/IbciQ9P4pRPlEaaJStmxZbNmyBQ8ePEBcXByKFSuGzp07Izg4OMvyrq6ucHV1zfX1AwICAICLxpEyBQoU0P8ckoXQTfbGWWlzZ80amaQAQJ06amMhu2QRU+h7enrC09MT9+7dw7p16zBhwgSTXFej0aBYsWLw9/fPcaE5InNwdnZmTYolYo1K7t24AfToIY8HDGDnWVJCaaKybt06CCFQvnx5nD59GsOHD0f58uXRu3dvk97H0dGRXxhEJHH6/NwRQiYnd+4A1aoBkyerjojslNJu21qtFoMGDUKFChXQo0cPNGjQADExMex4SETmw0Qld5YsAVaskJO6RUYCRjS7E5mS0hqVTp06oVOnTipDICJ78/ixnD6ffVSyFx8PvPeePP7kE6B6daXhkH2z2UUJiYiyJYQccutkEd30LFN0NDBtmqxVcXFRHQ3ZGGO+v/lbSkT2R6NhkvI0rVvLjUgxTi1IRETSrVvA5cuqoyAywESFiOzLm28C7doBZ8+qjsTyDB8OVK4MrFypOhIiPSYqRGRf1q6VX8ScW8nQ3r1AVBQQFyfXQSKyEExUiMh+pKYCWq085vDkdELI6fEBOcFbSIjaeIgyYKJCRPbj/v30YyYq6RYtAnbuBDw9gXHjVEdDZICJChHZD91kb15eci4VAh48AD7+WB6PGgUUL642HqInMFEhIvuhW+eHk72lmzABuHIFKF0ayLA6PZGlYKJCRPaD0+dnlpgIODoCEycCbm6qoyHKhIkKEdmPBw9kkw8TlXQTJgAnTgDt26uOhChLnJqRiOxH+/ZAUhKQnKw6EstStqzqCIiyxRoVIrIvGg1XAgaAb74B/vlHdRRET8VEhYjI3uzZA4SHA7VqAdevq46GKEdMVIjIfowfD7z+OvDnn6ojUSsiQu67dgUCApSGQvQ0TFSIyH7s3An8/rt9L7y3ezcQHS1H+owerToaoqdiokJE9oPDk9NrU7p3B8qVUxoKUW4wUSEi+2HvE77t3CkXZXR0BD79VHU0RLnCRIWI7Ie916joalN69OCQZLIaTFSIyH7Yc6IiBNCyJVCiBGtTyKowUSEi+5CUBDx8KI/tselHo5Fr+Zw/D5QpozoaolxjokJE9kGrldPnazSAj4/qaNRx4oTkZF2YqBCRffD3l7Uq8fGAg5391zd5MvDbb0BKiupIiIzG1JqI7IdGA3h6qo4if925A3zyiVwleetW4OWXVUdEZBQ7+7OCiMjOzJwpk5SaNYEGDVRHQ2Q0JipEZB9iYuT0+d9+qzqS/JOUBEydKo+HDZM1SkRWhk0/RGQfjh6V0+e7uamOJP/8+qtcdLB4caBjR9XREOUJa1SIyD7oZqUtXFhtHPlFCNmJFgAGDwZcXNTGQ5RHTFSIyD7cuSP39pKobN4MxMYCHh7AO++ojoYoz9j0Q0T2QZeo2Mtkb87OQP36QPXq9vOeySYxUSEi+2BvTT8NGgDbtwPJyaojIXombPohIvtgbzUqOuybQlaOiQoR2Yf4eLm39RqVx4/lkOTbt1VHQmQSTFSIyD6cOAEkJAC1a6uOxLxWrQLef1++TyFUR0P0zNhHhYjsg71Mnz97ttx37coJ3sgmsEaFiMhWnD8vZ+AFgL59lYZCZCpMVIjI9p0/D4SFAUOGqI7EvH76STb3NG0KlC2rOhoik2CiQkS27/Jl2Xfjzz9VR2I+KSnA3LnymBO8kQ1hokJEts8e5lCJjgauXgX8/GTtEZGNUJqopKSk4NNPP0VwcDDc3d1RpkwZfPHFF0hLS1MZFhHZGnuYPv/ECTkbbc+egKur6miITEbpqJ/x48djxowZiIqKQuXKlbFv3z707t0bvr6+GGLrbclElH/sYbK3jz4CevTgkGSyOUoTlZ07dyIsLAyhoaEAgNKlS2PRokXYt2+fyrCIyNbYQ9MPAPj7q46AyOSUNv00aNAAf/31F06ePAkAOHToEP7++2+0adMmy/JJSUmIi4sz2IiInsqWa1SEAM6dUx0FkdkoTVRGjBiBLl26oEKFCnB2dkaNGjUwdOhQdOnSJcvy48aNg6+vr34LDAzM54iJyCpptXJvizUqu3cDZcoAzZuz2YdsktJE5ddff8WCBQvwyy+/4MCBA4iKisLEiRMRFRWVZfnw8HBotVr9dunSpXyOmIis0qJFwIMHtjkJ2sKFcl+0KGeiJZuktI/K8OHDMXLkSLz55psAgCpVquDChQsYN24cevbsmam8q6srXNmbnYiMpdEAHh6qozC9x4+BX3+Vx926qY2FyEyU1qg8fPgQDg6GITg6OnJ4MhFRbmzYANy6BRQpIpt+iGyQ0hqVtm3bYuzYsQgKCkLlypVx8OBBTJo0CX369FEZFhHZmo4dAW9vYOJE2+pQu2CB3L/5JuDENWbJNmmEUNf7Kj4+HqNHj8aKFStw8+ZNFC9eHF26dMGYMWPg4uLy1NfHxcXB19cXWq0WPj4++RAxEVmdxMT0Zp/79wFfX6XhmExCguyX8vAhsGsXEBKiOiKiXDPm+1tpovKsmKgQ0VNduQKULAk4Oso+HbbS4XThQuCtt+Tig6dO2c77IrtgzPc36wqJyLZlnEPFlr7M27UDFi+WQ5Jt6X0RPYGJChHZNludldbDA+jcWXUURGbH1ZOJyLbZw4KERDaMiQoR2TZdjYotjfbp0QMYOzY9CSOyYWz6ISLbdu+e3NtKjcrp08D8+YCDg23OtEv0BCYqRGTbhg8H3ntPjvixBb/9JvdNmwIBAWpjIcoHTFSIyLbZ2vT5y5bJfceOauMgyifso0JEZC3OnQP275fNPq+/rjoaonzBRIWIbFt4ONC7N3DokOpInt3y5XLfsKFc34fIDjBRISLbtmoVEBlpGyNkdM0+b7yhNg6ifMREhYhsW8aZaa1ZaipQpoxcq6hdO9XREOUbdqYlItslhO3MTOvoKFdLfvwYcHZWHQ1RvmGNChHZroSE9GHJ1p6o6DBJITvDRIWIbJeu2cfVFXB3VxvLs7h3D/j3X1lDRGRnmKgQke3K2OxjzSsM//YbUKUK0KmT6kiI8h0TFSKyXbayzs/SpXJfu7baOIgUYGdaIrJdzZoBDx7IzVrduQNs3CiPO3RQGwuRAkxUiMi2eXhY9xT6q1bJocnVqgHlyqmOhijfsemHiMiS6Zp9WJtCdoqJChHZrrlz5fT5f/6pOpK8SUgANmyQx+3bq42FSBEmKkRku7ZskdPnHz2qOpK8Wb8eSE4GypYFKlVSHQ2REuyjQkS2y9qnz2/dGoiOlp2BrXl4NdEzYKJCRLbL2qfPd3MDWrVSHQWRUmz6ISLbZe01KkTERIWIbJg116hMnQp8/DFw5IjqSIiUYtMPEdmmtDS5Rg5gnYnKzJkySaleHahcWXU0RMqwRoWIbJNWK5MVwPqafs6dk0mKo6PsUEtkx1ijQkS2qWBB4OFD2fzj4qI6GuP88YfcN2gg3weRHWONChHZLnd3oEQJ1VEYT5eovPaa2jiILAATFSIiS6LVAps3y+O2bZWGQmQJmKgQkW3avBno1Ut2SrUm69YBKSlA+fLAc8+pjoZIOSYqRGSbDh0CoqKAjRtVR2Kc+HigaFE2+xD9h51picg2WescKn37yoUUExNVR0JkEVijQkS2STcrrbUlKgDg4AB4eqqOgsgiMFEhIttkjdPnX7+ePvcLEQFgokJEtsoam35atACKFwd27lQdCZHFYB8VIrJN1lajcvkycPgwoNFwtA9RBqxRISLbZG01KmvXyn3duoCfn9pYiCwIa1SIyDYdOSKTFWv50tclKlzbh8gAExUisk3WNH3+48fA+vXymIkKkQGlTT+lS5eGRqPJtA0aNEhlWERE+WvnTiAuTtb+1K6tOhoii6I0Udm7dy+uXbum39b/9xdFx44dVYZFRNYuNhbo3h2YPFl1JLkTHS33LVrIOVSISE9p00+RIkUMHn/zzTcoW7YsGjVqlGX5pKQkJCUl6R/HxcWZNT4islL//gssWABcvQp88IHqaJ6uQwc5f0o2//cR2TOL6aOSnJyMBQsWYNiwYdBoNFmWGTduHD7//PN8joyIrM6VK3JvLX1Uatdmkw9RNiymjnHlypW4f/8+evXqlW2Z8PBwaLVa/Xbp0qX8C5CIrIe1JSpElC2LqVGZO3cuWrdujeLFi2dbxtXVFa6urvkYFRFZJWtKVH76CShWDGjcWI5UIiIDFpGoXLhwARs2bMDy5ctVh0JEtsBaEpWUFGDYMECrlSN/XnxRdUREFscimn7mzZsHf39/hIaGqg6FiGyBtSQqu3bJJKVwYaBOHdXREFkk5YlKWloa5s2bh549e8LJySIqeIjImqWlATdvymNLT1QyDkt2dFQbC5GFUp4ZbNiwARcvXkSfPn1Uh0JEtsDBAUhIAK5dk30/LJkuUeFstETZ0gghhOog8iouLg6+vr7QarXw8fFRHQ4RUe7duAEEBMjj69eBokXVxkOUj4z5/lbe9ENEZJc2bJD76tWZpBDlgIkKEdmWVauAt94Cfv5ZdSQ5271b7lu0UBsHkYVjokJEtmX3bmDhwvREwFJ99x1w7BgwcKDqSIgsmvLOtEREJmUtQ5M1GqBCBdVREFk81qgQkW2xlkSFiHKFiQoR2RZrSFS6dgU6dQIOH1YdCZHFY6JCRLbF0hOVR4+AFSuA336Tc74QUY74W0JEtiMhAYiLk8eWmqj8/bdMVooXBypVUh0NkcVjZ1oish3Xr8tOqp6egKVOAhkTI/ctWshYiShHTFSIyHaUKwckJQG3b6uOJHsZExUieio2/RCRbXF2ttw1fq5fBw4dksfNmqmNhchKMFEhIsovumnza9YEihRRGwuRlWCiQkS2Y+pUoFs3YO1a1ZFkzcMDqFuXqyUTGYF9VIjIdmzaJIf+1q+vOpKstW8vN+tdtJ4o37FGhYhsh6XPoaLD0T5EucZEhYhshyUnKmfPps/xQkS5ZlSiMmHCBCQmJuofb926FUlJSfrH8fHxGMiVQIlIhdRUOaoGsMxEZcAAoFAhYPFi1ZEQWRWNELlvLHV0dMS1a9fg7+8PAPDx8UFsbCzKlCkDALhx4waKFy+O1NRU80T7hLi4OPj6+kKr1cLHUid3IqL8cfWqTFAcHeVcKo6OqiNK9+iRTFISE4F//wUqV1YdEZFSxnx/G1Wj8mROY0SOQ0RkXrpmn4AAy0pSAGDHDpmkBARw2nwiI7GPChHZhtu3ZSdVS2z20c2f0qwZO9ISGYnDk4nINrRuDSQnW2aH1fXr5b55c7VxEFkhoxOVOXPmwMvLCwCQkpKCyMhI+Pn5AZCdaYmIlHFykn1BLMndu8D+/fK4aVO1sRBZIaM605YuXRqaXFRbnjt37pmCyi12piUii7d0KdCxI1CxInD0qOpoiCyCMd/fRtWonD9//lniIiIyn48+kiN/PvwQqFVLdTTp6tcHpk8HXF1VR0JkldhHhYhsw5o1wLFjQN++qiMxVLy4nEOFiPLEqFE/u3fvRnR0tMG5n3/+GcHBwfD398c777xjMAEcEVG+seRZaYkoz4xKVCIiIvDPP//oHx8+fBh9+/ZFs2bNMHLkSPzxxx8YN26cyYMkIspRQkL6aB9LSlQ2bABmzAAuXFAdCZHVMipRiY2NRdMMvdYXL16MkJAQzJ49G8OGDcP333+PJUuWmDxIIqIc6WpTvL3lZilmz5bNPpGRqiMhslpGJSr37t1D0aJF9Y+3bNmCVq1a6R/XqVMHly5dMl10RES5YYnNPmlpwF9/yeNmzdTGQmTFjEpUihYtqh96nJycjAMHDqBevXr65+Pj4+Hs7GzaCImInsYSE5XYWODOHVnDU7eu6miIrJZRiUqrVq0wcuRIbNu2DeHh4fDw8MDLL7+sf/6ff/5B2bJlTR4kEVGOEhLk+j6WlKjoZqNt3BjgH3BEeWbU8OSvvvoK7du3R6NGjeDl5YXIyEi4uLjon//pp5/QokULkwdJRJSjAQPksOSEBNWRpMu4vg8R5ZlRM9PqaLVaeHl5wfGJFUrv3r0Lb2/vfGv+4cy0RGSREhOBggWBpCTgyBGumEz0BLPNTNunT59clfvpp5+MuSwRkW05fBhISZGTvVWsqDoaIqtmVKISGRmJUqVKoUaNGshDRQwRkendvQu0aQNUqAD89BPgYFTXO/OoW1d2pD17FsjF+mhElD2jEpX+/ftj8eLFOHv2LPr06YO33noLhSxtpVIisi8nTgC7d8uRP5aQpOj4+gI1aqiOgsjqGfVbPX36dFy7dg0jRozAH3/8gcDAQHTq1Anr1q1jDQsRqXHihNyXL682DiIyC6P//HB1dUWXLl2wfv16HD16FJUrV8bAgQNRqlQpJFhSj3sisg/Hj8u9pSQqK1YAL74oV0wmomf2TPWkGo0GGo0GQgikpaWZKiYiotyztBqVdetkU9SpU6ojIbIJRicqSUlJWLRoEZo3b47y5cvj8OHDmDZtGi5evAgvLy+jA7hy5QreeustFC5cGB4eHqhevTr2799v9HWIyE5ZWqKim+iN86cQmYRRnWkHDhyIxYsXIygoCL1798bixYtRuHDhPN/83r17eOmll9CkSRNER0fD398fZ86cQYECBfJ8TSKyIykpwOnT8tgSEpWzZ+Xm5AQ0bKg6GiKbYFSiMmPGDAQFBSE4OBhbtmzBli1bsiy3fPnyXF1v/PjxCAwMxLx58/TnSpcubUxIRGTPbt0CSpYEbt4EgoJUR5O+COGLL1rWKs5EVsyoRKVHjx7QmHBOgFWrVqFly5bo2LEjtmzZghIlSmDgwIHo169fluWTkpKQlJSkfxwXF2eyWIjIChUrJmswkpMtY2iybtr85s3VxkFkQ/I0hb6puLm5AQCGDRuGjh07Ys+ePRg6dChmzpyJHj16ZCofERGBzz//PNN5TqFPRMqlpQH+/nKit+3bgfr1VUdEZLGMmUJfaaLi4uKC2rVrY8eOHfpz77//Pvbu3YudO3dmKp9VjUpgYCATFSJS79494N13gb17gZMnuWIyUQ6MSVSU1pUWK1YMlZ5YrKtixYq4ePFiluVdXV3h4+NjsBGRHWvfHmjQANi1S3UkchHCJUtkUxSTFCKTMaqPiqm99NJLOKEbWvifkydPolSpUooiIiKrsmMHcOMG8MRK7kpxbR8ik1Jao/LBBx9g165d+Prrr3H69Gn88ssvmDVrFgYNGqQyLCKyBvfvyyQFUD80OSlJzufCpUSITE5polKnTh2sWLECixYtwgsvvIAvv/wSU6ZMQbdu3VSGRUTWQFcbW6wYoLoZ+O+/5erN9eqpjYPIBilt+gGAV199Fa+++qrqMIjI2ljSjLS62WgtIRYiG2MBEw8QEeWBJSYqnDafyOSYqBCRddIlKhUqqI3j1i3g4EF5zESFyOSYqBCRdfLzAwID1Scqf/0lO9FWqSL7yxCRSSnvo0JElCczZqiOQNI1+7RooTYOIhvFGhUiorwSgokKkZmxRoWIrI8QljGxmhCyZicmBnj5ZdXRENkk1qgQkfX5/nvZH2TMGLVxODgAbdoAU6YA7u5qYyGyUUxUiMj6HD8OXL8OpKaqjoSIzIyJChFZH0uYQyU5Gfj0U2DTJiAtTV0cRDaOiQoRWR9LSFR27QLGjgU6d1YXA5EdYKJCRNZFqwWuXpXHKhOVmBi5b9ZM9lUhIrPgbxcRWZc9e+Q+OBgoUEBdHByWTJQvmKgQkXXZuVPuVa5UfPcusHevPG7eXF0cRHaAiQoRWZegIKBhQ6BJE3UxbNwo51CpVAkoUUJdHER2gBO+EZF16dVLbirp+qewNoXI7FijQkRkrNhYuWf/FCKzY40KEVmPq1cBDw+1nWgBOTT50CG1o46I7ARrVIjIekREAAULAhMmqI3DwQGoUUMmTURkVkxUiMh66Eb8PPec2jiIKN8wUSEi66DVAkeOyGNVQ5MfPQLKlZOdeePj1cRAZGfYR4WIrMOePXJIcOnSQECAmhi2bQPOnAEePgS8vNTEQGRnWKNCRNbBEiZ6W7tW7lu1AjQadXEQ2REmKkRkHSwtUSGifMFEhYgsX1oasHu3PFaVqFy8CBw9Kkf8NGumJgYiO8Q+KkRk+VJSgG++kf1UqlVTE8O6dXIfEgIUKqQmBiI7xESFiCyfiwvwzjtyU4XNPkRKMFEhIsqN6tWBU6eA1q1VR0JkVzRCCKE6iLyKi4uDr68vtFotfHx8VIdDROayaBFQoQJQtSrg6Kg6GiJ6RsZ8f7NGhYgsW1wc0K2bnEPl2jV1c6gQkRIc9UNEls1SJnpLTFRzbyI7x0SFiCyb6vlTrl8HGjYECheWtTtElK+YqBCRZdMNC27QQM39Y2LkvlIlgH3hiPIdExUislzXrgE7dsjj115TE0N0tNxzWDKREkxUiMhyrVwp+6eEhAAlS+b//VNS0udP4bBkIiWYqBCR5dIlCR06qLn/9u3A/fuyf8qLL6qJgcjOcXgyEVmu334DNm4EqlRRc//Vq+W+TRvO30KkCBMVIrJcLi5q+4b88Yfct22rLgYiO8dEhYgoOytWAH/+CbRooToSIrvFPipEZHni44HKlYHhw4GkJHVxVKwIfPQR4OurLgYiO8dEhYgsz59/AkePAqtWyeYfIrJbShOViIgIaDQagy2A63gQ0fLlct++PaDR5P/9tVqgc2fg55+BtLT8vz8R6Snvo1K5cmVs2LBB/9iRPeuJ7FtiIrBmjTxu315NDOvWAUuWAP/8A/TooSYGIgJgAYmKk5NTrmtRkpKSkJShvTqO624Q2Z6YGODBAyAwEKhdW00MumHJr76q5v5EpKe8j8qpU6dQvHhxBAcH480338TZs2ezLTtu3Dj4+vrqt8DAwHyMlIjyxbJlcq+q2Sc1Nb1Gh4kKkXIaIYRQdfPo6Gg8fPgQzz//PG7cuIGvvvoKx48fx5EjR1C4cOFM5bOqUQkMDIRWq4UPFwsjsn7JyUDRonI22K1bgZdfzv8Ytm+XCyAWKADcugU4Ka94JrI5cXFx8PX1zdX3t9LfwNYZ1s6oUqUK6tWrh7JlyyIqKgrDhg3LVN7V1RWurq75GSIR5aekJKBXL9n0Ur++mhh0zT6tWzNJIbIAypt+MvL09ESVKlVw6tQp1aEQkQre3sDkycCxY+qmrGf/FCKLYlGJSlJSEo4dO4ZixYqpDoWIVFJVk/HgAeDhATg7q526n4j0lCYqH330EbZs2YJz585h9+7deOONNxAXF4eePXuqDIuIVPjkE9k/RCVPT2D3buDGDaBQIbWxEBEAxYnK5cuX0aVLF5QvXx7t27eHi4sLdu3ahVKlSqkMi4jy286dwNdfA02ayCRBtYIFVUdARP9R2lNs8eLFKm9PRJZi/Hi5795djvpRIT4eSElhkkJkYSyqjwoR2aFjx4Dff5dzpgwfri6OhQsBf3/ggw/UxUBEmTBRISK1JkyQ+9dfBypUUBfHsmWyRoXrjRFZFCYqRKTOpUuyJgMARoxQF8fdu8CmTfK4Qwd1cRBRJkxUiEid774DHj8GGjcGQkLUxbFqlZw6v2pVoFw5dXEQUSZMVIhInVKlgGrVgCxmos5Xy5fLvarVmokoW0rX+nlWxqwVQEQWTAg1CxACcrRPkSJy+v7Dh4EXXlATB5EdMeb7mzUqRKSeqiQFkCslJyUBzz8PVK6sLg4iyhJX3CKi/CcE8McfwCuvAF5eamNp0gSYPh1wc1ObMBFRlpioEFH+++cfICxMTq52/Trg4qIuFn9/YMAAdfcnohyx6YeI8t+KFXLfsKHaJIWILB5rVIgo/+lG2bRrpzaOiRPlQoQdOwJ+fmpjIaIsMVEhovx1+rQcXePoCLRtqy6OR4+AL78E4uLkSJ+XX1YXCxFli00/RJS/dM0+TZoAhQqpi2P1apmkBAYCL72kLg4iyhETFSLKX5bS7PPLL3LfpQvgwP8KiSwVfzuJKP/cvAns3i2PX39dXRz37gF//imPu3VTFwcRPRX7qBBR/vH3B86cAbZvB4oXVxfHsmVAcrLsm1K1qro4iOipmKgQUf4KDpabSroVm1mbQmTx2PRDRPYlJUXOhuvsLPunEJFFY6JCRPlj/XqgUyfg55/VxuHkJKfvv3VLrt5MRBaNiQoR5Y/Nm4HffgO2bVMdieTrqzoCIsoFJipElD8OHJD7mjXVxXD9OnDhgrr7E5HRmKgQkfkJAezfL49r1VIXx7RpQOnSQHi4uhiIyChMVIjI/K5ckX1CHB2BKlXUxCBE+iRv1aqpiYGIjMZEhYjMT9fsU6kS4O6uJoadO4Fz5+QihK+9piYGIjIaExUiMj9LaPb56Se5b98e8PBQFwcRGYWJChGZn1Yr5y1R1ZE2Lg5YvFge9+unJgYiyhMmKkRkflOmAPHxQJ8+au6/aBHw4AFQoQLQoIGaGIgoTziFPhHlD1dXuamQsTZFo1ETAxHlCRMVIrJ9f/4pJ5sLDVUdCREZiU0/RGReEybITrRz56qLwcMD6NkT8PNTFwMR5QkTFSIyr5075fBkrTb/752aKudPISKrxUSFiMxLN4eKiqHJUVFA5crAggX5f28iMgkmKkRkPrdvAxcvyuPq1fP//rNnA8eOyZlxicgqMVEhIvM5eFDuy5XL/9WKDx8Gdu0CnJyAXr3y995EZDJMVIjIfFTOSDt7ttyHhQFFi+b//YnIJJioEJH56Pqn5PeMtPHxwM8/y2POREtk1ZioEJH5lCgBlC2b/zUqP/0kRxk99xzQvHn+3puITIqJChGZz+TJwOnTQNOm+XfP1FQ5ZT8ADBsGOPC/OSJrxt9gIrItDg5AZCTQpQvQo4fqaIjoGVlMojJu3DhoNBoMHTpUdShEZArx8WomW9NogEaNgF9+kTPSEpFVs4hEZe/evZg1axaqVq2qOhQiMpV33wUKFgTmz1cdCRFZMeWJSkJCArp164bZs2ejYMGCqsMhIlMQQs5hotUCRYrk33179wY+/BC4di3/7klEZqU8URk0aBBCQ0PRrFmzp5ZNSkpCXFycwUZEFujUKeDcOcDZGXjppfy55+nTcsr8SZOA+/fz555EZHZOKm++ePFiHDhwAHv37s1V+XHjxuHzzz83c1RE9Myio+X+5ZcBb+/8uefkybImp00boGLF/LknEZmdshqVS5cuYciQIViwYAHc3Nxy9Zrw8HBotVr9dunSJTNHSUR5oktU2rTJn/vduQPMmyePP/oof+5JRPlCWY3K/v37cfPmTdTKMBFUamoqtm7dimnTpiEpKQmOjo4Gr3F1dYWrq2t+h0pExnj4ENi8WR63bp0/95w2DUhMBGrUABo3zp97ElG+UJaoNG3aFIcPHzY417t3b1SoUAEjRozIlKQQkZXYvBlISgKCgvKnCebuXdkvBQBGjpTDk4nIZihLVLy9vfHCCy8YnPP09EThwoUznSciK1K+PPDZZ4CnZ/4kDRMnAnFxQLVqwBtvmP9+RJSvlHamJSIbVLYsEBGRf/cbPBhISABateJ0+UQ2SCOEiqkjTSMuLg6+vr7QarXw8fFRHQ4RERHlgjHf3/zzg4hM56+/gOXLZVOMuT1+bP57EJFyTFSIyHT+9z+gQwdg1izz32vAADn8+cgR89+LiJRhokJEppGfw5JPnZIrJEdHy2n6ichmMVEhItPQDUsODAQqVTLvvSIigNRUWaNSv75570VESjFRISLT0M1G27q1eYcl79kDLFokj7/6ynz3ISKLwESFiEwjY6JiLqmpsm+KEED37nImWiKyaUxUiOjZnToFnDkjV0tu2tR895kxAzhwAPD1lR13icjmMVEhomenWwG9QQPzrZYsBLBwoTweOxYoWtQ89yEii8KZaYno2XXtCtStK9fdMReNRnbY/flnoHdv892HiCwKExUiMo1y5cx/DxcX4O23zX8fIrIYbPohomdjzloUQM5AO2sWkJxs3vsQkUViokJEeXfmDBAQIFctTkkxzz0mTwbefRdo3lz2UyEiu8JEhYjybuZMWePx4AHgZIaW5AMHgE8/lce9epl3fhYiskhMVIgobxITgblz5fHAgaa//oMHQJcuMhFq104mKkRkd5ioEFHe/Pab7J8SFCSnsje1oUOBkyeBEiWA2bNZm0Jkp5ioEFHeTJ8u9+++Czg6mvbaS5cCc+bI5GT+fKBwYdNen4isBhMVIjLe/v3A7t1yJtq+fU177eRkYNgweRweDjRpYtrrE5FVYaJCRMabNUvu33jD9DPEurgAGzcC77wjV0kmIrvGCd+IyHjjxwOVKwP165vn+uXKyRFFRGT3mKgQkfEKFADef9+01/zuO+C558zTMZeIrBabfogo986cMc/Ebr/9Jkf5tG0L/POP6a9PRFaLiQoR5c7Dh0DTpkDNmsDp06a77o4dQPfu8vi994CqVU13bSKyekxUiCh3vvkGuHABuH8fKFbMNNc8fRp47TUgKUnuJ00yzXWJyGYwUSGipzt9WnagBYApUwBPz2e/5smTwCuvAHfuALVqAb/8Yvr5WIjI6rEzLRHlTAjZcTY5GWjZUk5n/6yuXgUaNgRu3AAqVABWrzZN8kNENoc1KkSUs8WLgehoObnb99+bZir7YsVkwlO9OrB1q1yBmYgoC6xRIaLs7dkD9Okjj0eMAJ5/3jTX1WiAH36QCw96e5vmmkRkk1ijQkTZK1sWqF1bDht+1llio6KAzp3lasgA4ODAJIWInoo1KkSUvcKFgfXrgdTUvHd0TUwEBg8G5s6Vj+vVk3OmEBHlAmtUiMhQYiKwbFn6Yze3vHd0PXVKJiZz58rmni+/NP2MtkRk05ioEFG6xESgWze52OCXX+b9OkIAixbJZqNDh4AiRYCYGODTT2WTDxFRLvF/DCKSLlwAXn4ZWLFCrmDcsGHer/XJJ0DXrkBcHNCgAXDwINCsmeliJSK7wUSFiICNG2Xtx/79sl/KmjVAo0Z5v1737rKjbESEvHaJEiYLlYjsCzvTEtkzIYBvv5VDj9PS5Do+y5YBpUvn/hrJycDChXISt08+kecqVgQuXQJ8fc0SNhHZDyYqRPZMl1ykpQE9ewI//gi4u+futQ8fyk6y//ufTEocHIDWrWWyAzBJISKTYKJCZG9u3ZKdWwHZJPPRR0Dx4sDAgbmbdfbcOTknyvTp8lqAnFn2ww9NNyEcEdF/mKgQ2YtDh4BZs4A5c4Dt22WfFAAYOzb31/jzT+DVV9Mfly4tm4169ZLDmImITIyJCpEtu31b9h+JjARiY9PPr1iRnqhkRQg5B8ratUCBAkCPHvJ8w4ayk2xICNC7N9Cxo1wDiIjITJioENmihAQgLAzYti19ynoXF+C114ABA4BXXjEsn5ICHD0q1/bZvRv46y/ZxAPIPie6RMXbm51kiShfKU1UfvzxR/z44484f/48AKBy5coYM2YMWrdurTIsIuvw+DFw4gTwzz9yc3cHPvtMPuflBZw9K8vUqiWbZrp0AQoVAq5ckclISEj6tapVk4lKRi4ucl6V1q1lDYuu/wqTFCLKR0oTlZIlS+Kbb75BuXLlAABRUVEICwvDwYMHUblyZZWhEan1+LHsqKrVyqG+OhERwM6dwPnzssZDV1sCAD4+cgSP03+/1n37AjduyNE5v/8OTJ0qa0MSE2XNiFabnny88IJ8rk4doG5d4KWXgMaNZcJDRKSQRgghVAeRUaFChfC///0Pffv2fWrZuLg4+Pr6QqvVwsfHx7SB3L4t/1rNTnCwHCkBAHfvAseOZV+2VCmgZEl5HBcHHD6cfdmSJWV5QFbfHzqUfdnixWUcgPwyOngw+7IBAXIlXEDOe7F3b/Zl/fyA8uXlcUqK/Os7O4UKpX+RpqXJL9HsFCgAZExAt2+X+6x+BH19gSpV0h/v2mX4pZzxdV5e6UNiAWDHDiApKT2mjNzdgfr10x9v3Qo8eCCvlZaWXj4tTZZt2TK97Jo1wJ078rnUVLmlpMi9qyvQr1962enTgYsXZcxJSfIzT0qSm5MTsGBBetnOnYEDB2QciYly08Xv5iYf65QrB5w5k/5Yo5FbWpocHhwXl74uT7t2wMqVmT9bJyd5nR07gIIF5bn792Wiw+ntiSgfGPX9LSxESkqKWLRokXBxcRFHjhzJssyjR4+EVqvVb5cuXRIAhFarNX1An34qhPz6ynrr0iW97IQJOZd99dX0srNn51z2lVfSyy5dmnPZkJD0sn/9lXPZKlXSyx46lHPZ555LL3vlSs5lAwPTyyYl5VzW39/wM86pbMGChmU1muzLenoalnV0zL6sq6thWWfn7Ms6ORmWdXfPvqyDg2FZb++c319GhQvnXDY5Ob3siy/mXPb+/fSy06cL0a+fEF9+KURUlBCbNglx5ozh9YiIFNBqtbn+/lbemfbw4cOoV68eHj16BC8vL6xYsQKVKlXKsuy4cePw+eef509gT1vS3inDR/e0shlHRTxtnoqM133aX7fGlDVmZIaTET8Wz1JWo8m6NgXI/Jk6OMiai9xc19Ex+7JPfg7OzrJWJCsuLoaPPT1lTUfGf0NdjYarq2HZEiVkjYpGI+NxcJCbo2PmYbytW8saOTc3wMNDbl5eslbJ3d3ws/j6azlJm6enLFOwoCxXoIDcZ3x/AwZk/b6IiKyI8qaf5ORkXLx4Effv38eyZcswZ84cbNmyJctkJSkpCUm6KnHIqqPAwEDzNP0QERGRWRjT9KM8UXlSs2bNULZsWcycOfOpZc3aR4WIiIjMwpjvb4vrOSeEMKg1ISIiIvultI/KqFGj0Lp1awQGBiI+Ph6LFy/G5s2bsXbtWpVhERERkYVQmqjcuHED3bt3x7Vr1+Dr64uqVati7dq1aN68ucqwiIiIyEIoTVTmzp2r8vZERERk4SyujwoRERGRDhMVIiIislhMVIiIiMhiMVEhIiIii8VEhYiIiCwWExUiIiKyWExUiIiIyGIxUSEiIiKLxUSFiIiILJbSmWmflW7h57i4OMWREBERUW7pvrd13+M5sepEJT4+HgAQGBioOBIiIiIyVnx8PHx9fXMsoxG5SWcsVFpaGq5evQpvb29oNBqTXjsuLg6BgYG4dOkSfHx8THptS8D3Z/1s/T3a+vsDbP898v1ZP3O9RyEE4uPjUbx4cTg45NwLxaprVBwcHFCyZEmz3sPHx8dmfwABvj9bYOvv0dbfH2D775Hvz/qZ4z0+rSZFh51piYiIyGIxUSEiIiKLxUQlG66urvjss8/g6uqqOhSz4Puzfrb+Hm39/QG2/x75/qyfJbxHq+5MS0RERLaNNSpERERksZioEBERkcViokJEREQWi4kKERERWSwmKkZISkpC9erVodFoEBsbqzock3nttdcQFBQENzc3FCtWDN27d8fVq1dVh2Uy58+fR9++fREcHAx3d3eULVsWn332GZKTk1WHZjJjx45F/fr14eHhgQIFCqgOxySmT5+O4OBguLm5oVatWti2bZvqkExm69ataNu2LYoXLw6NRoOVK1eqDslkxo0bhzp16sDb2xv+/v54/fXXceLECdVhmdSPP/6IqlWr6idBq1evHqKjo1WHZTbjxo2DRqPB0KFDldyfiYoRPv74YxQvXlx1GCbXpEkTLFmyBCdOnMCyZctw5swZvPHGG6rDMpnjx48jLS0NM2fOxJEjRzB58mTMmDEDo0aNUh2aySQnJ6Njx44YMGCA6lBM4tdff8XQoUPxySef4ODBg3j55ZfRunVrXLx4UXVoJvHgwQNUq1YN06ZNUx2KyW3ZsgWDBg3Crl27sH79eqSkpKBFixZ48OCB6tBMpmTJkvjmm2+wb98+7Nu3D6+88grCwsJw5MgR1aGZ3N69ezFr1ixUrVpVXRCCcmXNmjWiQoUK4siRIwKAOHjwoOqQzOb3338XGo1GJCcnqw7FbCZMmCCCg4NVh2Fy8+bNE76+vqrDeGZ169YV/fv3NzhXoUIFMXLkSEURmQ8AsWLFCtVhmM3NmzcFALFlyxbVoZhVwYIFxZw5c1SHYVLx8fHiueeeE+vXrxeNGjUSQ4YMURIHa1Ry4caNG+jXrx/mz58PDw8P1eGY1d27d7Fw4ULUr18fzs7OqsMxG61Wi0KFCqkOg7KQnJyM/fv3o0WLFgbnW7RogR07diiKivJKq9UCgM3+vqWmpmLx4sV48OAB6tWrpzockxo0aBBCQ0PRrFkzpXEwUXkKIQR69eqF/v37o3bt2qrDMZsRI0bA09MThQsXxsWLF/H777+rDslszpw5g6lTp6J///6qQ6Es3L59G6mpqShatKjB+aJFi+L69euKoqK8EEJg2LBhaNCgAV544QXV4ZjU4cOH4eXlBVdXV/Tv3x8rVqxApUqVVIdlMosXL8aBAwcwbtw41aHYb6ISEREBjUaT47Zv3z5MnToVcXFxCA8PVx2yUXL7/nSGDx+OgwcPIiYmBo6OjujRoweEhU9abOx7BICrV6+iVatW6NixI95++21FkedOXt6fLdFoNAaPhRCZzpFle++99/DPP/9g0aJFqkMxufLlyyM2Nha7du3CgAED0LNnTxw9elR1WCZx6dIlDBkyBAsWLICbm5vqcOx3Cv3bt2/j9u3bOZYpXbo03nzzTfzxxx8G/0GmpqbC0dER3bp1Q1RUlLlDzZPcvr+sfggvX76MwMBA7Nixw6KrMo19j1evXkWTJk0QEhKCyMhIODhYdp6el3/DyMhIDB06FPfv3zdzdOaTnJwMDw8P/Pbbb2jXrp3+/JAhQxAbG4stW7YojM70NBoNVqxYgddff111KCY1ePBgrFy5Elu3bkVwcLDqcMyuWbNmKFu2LGbOnKk6lGe2cuVKtGvXDo6Ojvpzqamp0Gg0cHBwQFJSksFz5uaUb3eyMH5+fvDz83tque+//x5fffWV/vHVq1fRsmVL/PrrrwgJCTFniM8kt+8vK7rcNSkpyZQhmZwx7/HKlSto0qQJatWqhXnz5ll8kgI827+hNXNxcUGtWrWwfv16g0Rl/fr1CAsLUxgZ5YYQAoMHD8aKFSuwefNmu0hSAPm+Lf3/zNxq2rQpDh8+bHCud+/eqFChAkaMGJGvSQpgx4lKbgUFBRk89vLyAgCULVsWJUuWVBGSSe3Zswd79uxBgwYNULBgQZw9exZjxoxB2bJlLbo2xRhXr15F48aNERQUhIkTJ+LWrVv65wICAhRGZjoXL17E3bt3cfHiRaSmpurn+SlXrpz+Z9aaDBs2DN27d0ft2rVRr149zJo1CxcvXrSZfkUJCQk4ffq0/vG5c+cQGxuLQoUKZfo/x9oMGjQIv/zyC37//Xd4e3vr+xX5+vrC3d1dcXSmMWrUKLRu3RqBgYGIj4/H4sWLsXnzZqxdu1Z1aCbh7e2dqU+Rrg+jkr5GSsYaWbFz587Z1PDkf/75RzRp0kQUKlRIuLq6itKlS4v+/fuLy5cvqw7NZObNmycAZLnZip49e2b5/jZt2qQ6tDz74YcfRKlSpYSLi4uoWbOmTQ1v3bRpU5b/Xj179lQd2jPL7ndt3rx5qkMzmT59+uh/NosUKSKaNm0qYmJiVIdlViqHJ9ttHxUiIiKyfJbfUE9ERER2i4kKERERWSwmKkRERGSxmKgQERGRxWKiQkRERBaLiQoRERFZLCYqREREZLGYqBAREZHFYqJCZIMaN26MoUOHqg4jS3fu3IG/vz/Onz8PANi8eTM0Go3ZF1LM630iIyNRoEABo15Tp04dLF++3KjXEFHWmKgQ0VNdu3YNXbt2Rfny5eHg4JBtErRs2TJUqlQJrq6uqFSpElasWJGpzLhx49C2bVuULl3avEErNHr0aIwcORJpaWmqQyGyekxUiOipkpKSUKRIEXzyySeoVq1almV27tyJzp07o3v37jh06BC6d++OTp06Yffu3foyiYmJmDt3Lt5+++38Cl2J0NBQaLVarFu3TnUoRFaPiQqRjbt37x569OiBggULwsPDA61bt8apU6cMysyePRuBgYHw8PBAu3btMGnSJIPmjtKlS+O7775Djx494Ovrm+V9pkyZgubNmyM8PBwVKlRAeHg4mjZtiilTpujLREdHw8nJKceVue/cuYMuXbqgZMmS8PDwQJUqVbBo0SKDMo0bN8bgwYMxdOhQFCxYEEWLFsWsWbPw4MED9O7dG97e3ihbtiyio6MzXX/79u2oVq0a3NzcEBISkmk5+8jISAQFBek/izt37hg8f+bMGYSFhaFo0aLw8vJCnTp1sGHDBoMyjo6OaNOmTaa4ich4TFSIbFyvXr2wb98+rFq1Cjt37oQQAm3atMHjx48ByC/u/v37Y8iQIYiNjUXz5s0xduxYo++zc+dOtGjRwuBcy5YtsWPHDv3jrVu3onbt2jle59GjR6hVqxZWr16Nf//9F++88w66d+9uUDMDAFFRUfDz88OePXswePBgDBgwAB07dkT9+vVx4MABtGzZEt27d8fDhw8NXjd8+HBMnDgRe/fuhb+/P1577TX9Z7F792706dMHAwcORGxsLJo0aYKvvvrK4PUJCQlo06YNNmzYgIMHD6Jly5Zo27YtLl68aFCubt262LZtW+4+PCLKnpI1m4nIrHRLsp88eVIAENu3b9c/d/v2beHu7i6WLFkihBCic+fOIjQ01OD13bp1E76+vjle+0nOzs5i4cKFBucWLlwoXFxc9I/DwsJEnz59DMps2rRJABD37t3L9v20adNGfPjhhwYxNGjQQP84JSVFeHp6iu7du+vPXbt2TQAQO3fuNLjP4sWL9WXu3Lkj3N3dxa+//iqEEKJLly6iVatWBvfu3Llztp+FTqVKlcTUqVMNzv3+++/CwcFBpKam5vhaIsoZa1SIbNixY8fg5OSEkJAQ/bnChQujfPnyOHbsGADgxIkTqFu3rsHrnnycWxqNxuCxEMLgXGJiItzc3HK8RmpqKsaOHYuqVauicOHC8PLyQkxMTKYai6pVq+qPHR0dUbhwYVSpUkV/rmjRogCAmzdvGrwuY7NToUKFDD6LY8eOZWqWevLxgwcP8PHHH6NSpUooUKAAvLy8cPz48Uzxubu7Iy0tDUlJSTm+XyLKmZPqAIjIfIQQ2Z7XJRBPJhM5vS4nAQEBuH79usG5mzdv6hMGAPDz88O9e/dyvM63336LyZMnY8qUKahSpQo8PT0xdOhQJCcnG5RzdnY2eKzRaAzO6d5TbkbeZPwsnmb48OFYt24dJk6ciHLlysHd3R1vvPFGpvju3r0LDw8PuLu7P/WaRJQ91qgQ2bBKlSohJSXFoH/HnTt3cPLkSVSsWBEAUKFCBezZs8fgdfv27TP6XvXq1cP69esNzsXExKB+/fr6xzVq1MDRo0dzvM62bdsQFhaGt956C9WqVUOZMmUydf59Frt27dIf37t3DydPnkSFChUAyM8r4/NPltfF16tXL7Rr1w5VqlRBQECAfk6YjP7991/UrFnTZHET2SsmKkQ27LnnnkNYWBj69euHv//+G4cOHcJbb72FEiVKICwsDAAwePBgrFmzBpMmTcKpU6cwc+ZMREdHZ6pliY2NRWxsLBISEnDr1i3ExsYaJB1DhgxBTEwMxo8fj+PHj2P8+PHYsGGDwZwrLVu2xJEjR3KsVSlXrhzWr1+PHTt24NixY3j33Xcz1dQ8iy+++AJ//fUX/v33X/Tq1Qt+fn54/fXXAQDvv/8+1q5diwkTJuDkyZOYNm0a1q5dmym+5cuXIzY2FocOHULXrl2zrLXZtm1bps7FRGQ8JipENm7evHmoVasWXn31VdSrVw9CCKxZs0bfTPLSSy9hxowZmDRpEqpVq4a1a9figw8+yNSXpEaNGqhRowb279+PX375BTVq1ECbNm30z9evXx+LFy/GvHnzULVqVURGRuLXX3816B9TpUoV1K5dG0uWLMk23tGjR6NmzZpo2bIlGjdujICAAH0iYQrffPMNhgwZglq1auHatWtYtWoVXFxcAAAvvvgi5syZg6lTp6J69eqIiYnBp59+avD6yZMno2DBgqhfvz7atm2Lli1bZqo5uXLlCnbs2IHevXubLG4ie6UReWmMJiKb1q9fPxw/ftwsw2vXrFmDjz76CP/++y8cHGzzb6Xhw4dDq9Vi1qxZqkMhsnrsTEtEmDhxIpo3bw5PT09ER0cjKioK06dPN8u92rRpg1OnTuHKlSsIDAw0yz1U8/f3x0cffaQ6DCKbwBoVIkKnTp2wefNmxMfHo0yZMhg8eDD69++vOiwiIiYqREREZLlss4GYiIiIbAITFSIiIrJYTFSIiIjIYjFRISIiIovFRIWIiIgsFhMVIiIislhMVIiIiMhiMVEhIiIii/V/Q1S6XH7Exj0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import os\n", "import numpy as np\n", @@ -1922,7 +1895,7 @@ " # and then make the prediction\n", " ypredictRidge = X @ Ridgebeta\n", " MSERidgePredict[i] = MSE(y,ypredictRidge)\n", - " RegLasso = linear_model.Lasso(lmb)\n", + " RegLasso = linear_model.Lasso(lmb,fit_intercept=False)\n", " RegLasso.fit(X,y)\n", " ypredictLasso = RegLasso.predict(X)\n", " print(RegLasso.coef_)\n", @@ -1940,22 +1913,39 @@ { "cell_type": "markdown", "id": "becd3a54", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Another Example, now with a polynomial fit" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "46f816fd", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 2.03099776 -0.17917768 5.18029127]\n", + "Training MSE for OLS\n", + "0.009163470508352218\n", + "Test MSE OLS\n", + "0.008675369724975977\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAG1CAYAAAAMU3WaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABxJ0lEQVR4nO3dd3xTZfvH8U+S7t1SoAXa0rJn2QiIgIIFKg8qbkAB4QGRJYpYlaGPivBTQEXFCbgAB6KiLJENyiygTJE9ZLdldOb8/ggEQmmh0JKm/b5fr7yac851kitpIFfv+z73bTIMw0BERETERZmdnYCIiIjIjVAxIyIiIi5NxYyIiIi4NBUzIiIi4tJUzIiIiIhLUzEjIiIiLk3FjIiIiLg0FTMiIiLi0lTMiIiIiEtTMSMiIiIuzanFzJIlS+jQoQNlypTBZDIxc+bMHGN79+6NyWRi/PjxNy0/ERERKfycWsycOXOG2NhYJkyYkGvczJkz+eOPPyhTpsxNykxERERchZszn7xdu3a0a9cu15gDBw7Qr18/5s6dS3x8fJ6fw2q1cvDgQfz9/TGZTNebqoiIiNxEhmGQkpJCmTJlMJtzb3txajFzNVarla5duzJkyBBq1KhxTeekpaWRlpZm3z5w4ADVq1cvqBRFRESkAO3bt49y5crlGlOoi5nRo0fj5ubGgAEDrvmcUaNG8dJLL2Xbv2/fPgICAvIzPRERESkgycnJRERE4O/vf9XYQlvMrF27lrfeeot169blqXsoISGBwYMH27cvvBkBAQEqZkRERFzMtdQAhfbS7KVLl3LkyBEiIyNxc3PDzc2NPXv28PTTT1O+fPkcz/P09LQXLipgREREir5C2zLTtWtXWrdu7bAvLi6Orl270r17dydlJSIiIoWNU4uZ06dP8/fff9u3d+3aRWJiIiEhIURGRlKiRAmHeHd3d8LCwqhSpcrNTlVEREQKKacWM2vWrKFVq1b27QtjXR577DEmT558U3PJysoiIyPjpj6nFG/u7u5YLBZnpyEi4vKcWsy0bNkSwzCuOX737t35noNhGBw+fJhTp07l+2OLXE1QUBBhYWGaA0lE5AYU2jEzN8uFQqZUqVL4+PjoS0VuCsMwOHv2LEeOHAEgPDzcyRmJiLiuYl3MZGVl2QuZy8fniBQ0b29vAI4cOUKpUqXU5SQicp0K7aXZN8OFMTI+Pj5OzkSKqwufPY3XEhG5fsW6mLlAXUviLPrsiYjcOBUzIiIi4tJUzEiB2r17NyaTicTExBxjFi1ahMlkcrkrylq2bMmgQYOcnYaISLGnYsYFdevWDZPJRJ8+fbId69u3LyaTiW7dutn3HTlyhN69exMZGYmnpydhYWHExcWxcuVKe0z58uUxmUzZbq+//nqOebRs2dIe5+HhQYUKFUhISHBYtTwiIoJDhw5Rs2bN/HnxN6Bbt27cfffd+fZ4M2bM4H//+1++PZ6IiFyfYn01kyuLiIhg2rRpjBs3zn5VTGpqKlOnTiUyMtIhtlOnTmRkZDBlyhRiYmL4999/WbBgASdOnHCIe/nll+nVq5fDvqutVtqrVy9efvll0tPTWb16tX2piVGjRgFgsVgICwu7odd6s2VkZODu7n7VuJCQkJuQjYhIIWYYYLWCk6/GVMuMi6pXrx6RkZHMmDHDvm/GjBlERERQt25d+75Tp06xbNkyRo8eTatWrYiKiqJRo0YkJCQQHx/v8Jj+/v6EhYU53Hx9fXPNw8fHh7CwMCIjI+nUqRNt2rRh3rx59uNX6mb65ZdfqFy5Mt7e3rRq1eqKkyF+9NFHRERE4OPjwz333MPYsWMJCgpyiPnpp5+oX78+Xl5exMTE8NJLL5GZmXnFPEeOHMmUKVP44Ycf7K1JixYtsuf39ddf07JlS7y8vPjiiy84fvw4Dz/8MOXKlcPHx4datWoxdepUh8e8vJupfPnyvPbaa/To0QN/f38iIyP58MMPc33/RERcTmoqzJkD/fpBdDRMn+7sjFTMXM4wDM6mZ970W15mQr6ge/fuTJo0yb796aef0qNHD4cYPz8//Pz8mDlzpkP3T0HYsGEDy5cvz7VVY9++fdx77720b9+exMREevbsyXPPPecQs3z5cvr06cPAgQNJTEykTZs2vPrqqw4xc+fOpUuXLgwYMIDNmzfzwQcfMHny5GxxFzzzzDM88MADtG3blkOHDnHo0CGaNm1qPz506FAGDBjAli1biIuLIzU1lfr16zNr1iz+/PNP/vvf/9K1a1f++OOPXN+DN998kwYNGrB+/Xr69u3LE088wdatW6/21omIuI6FC6FdO3j3XdizB375xdkZqZvpcucysqg+fO5Nf97NL8fh45G3X0fXrl1JSEiwty4sX76cadOmsWjRInuMm5sbkydPplevXkycOJF69erRokULHnroIWrXru3weEOHDuXFF1902Ddr1ixatmyZYw7vvfceH3/8MRkZGaSnp2M2m3n33XdzjH///feJiYlh3LhxmEwmqlSpwqZNmxg9erQ95p133qFdu3Y888wzAFSuXJkVK1Ywa9Yse8yrr77Kc889x2OPPQZATEwM//vf/3j22WcZMWJEtuf18/PD29ubtLS0K3Z7DRo0iHvvvddh34XnB+jfvz9z5szhm2++oXHjxjm+vvbt29O3b1/A9n6OGzeORYsWUbVq1RzPEREplKxWWLQIvvwSqlWDC/8ntmoFlSrZft51F9xxh1PTBBUzLi00NJT4+HimTJmCYRjEx8cTGhqaLa5Tp07Ex8ezdOlSVq5cyZw5cxgzZgwff/yxw0DhIUOGOGwDlC1bNtccOnfuzAsvvEBycjKjR48mICCATp065Ri/ZcsWbrnlFof5VZo0aeIQs23bNu655x6HfY0aNXIoZtauXcvq1asdWmKysrJITU3l7NmzeZ4IsUGDBg7bWVlZvP7660yfPp0DBw6QlpZGWlraVbvdLi0QTSYTYWFh9iULRERcwqlTMGmSreVl507bvgoV4OmnwWQCLy/Yvt2pKV5OxcxlvN0tbH45zinPez169OhBv379AHJtEfHy8qJNmza0adOG4cOH07NnT0aMGOFQvISGhlKxYsU8PX9gYKD9nC+++IIaNWrwySef8Pjjj18x/lq60wzDyDaZ3OXnWa1WXnrppWytKWB7rXl1eZHy5ptvMm7cOMaPH0+tWrXw9fVl0KBBpKen5/o4l3exmUwmrFZrnvMREbnp/voL3nkHPv8czp617QsMhAcegC5dnJvbVaiYuYzJZMpzd48ztW3b1v4FGxd37UVY9erVmTlzZr7m4u7uzvPPP09CQgIPP/zwFVtHrvS8v//+u8N21apVWbVqlcO+NWvWOGzXq1ePbdu25an48vDwICsr65pily5dSseOHely/h+w1Wplx44dVKtW7ZqfT0TEpYwYAd99Z7tfsyb07w+dO0MuLdInzqSzaNsR6kQEEVPS7yYlmp3rfGvLFVksFrZs2WK/f7njx49z//3306NHD2rXro2/vz9r1qxhzJgxdOzY0SE2JSWFw4cPO+zz8fEhICDgmvN55JFHeP7553nvvfccxpxc0KdPH958800GDx5M7969Wbt2LZMnT3aI6d+/P7fddhtjx46lQ4cO/Pbbb8yePduhtWb48OHcddddREREcP/992M2m9m4cSObNm3ilVdeuWJu5cuXZ+7cuWzbto0SJUoQGBiY4+uoWLEi3333HStWrCA4OJixY8dy+PBhFTMiUnRs3gwBAVCunG175EjbpdYDBsBtt9m6lC5jGAZbD6fw29Yj/Lb1COv2nsQwYFDrSgxqXfnm5n8JXc1UBAQEBORYcPj5+dG4cWPGjRvHbbfdRs2aNRk2bBi9evViwoQJDrHDhw8nPDzc4fbss8/mKRcPDw/69evHmDFjOH36dLbjkZGRfPfdd/z000/ExsYyceJEXnvtNYeYZs2aMXHiRMaOHUtsbCxz5szhqaeecug+iouLY9asWcyfP5+GDRtyyy23MHbsWKKionLMrVevXlSpUoUGDRpQsmRJli9fnmPssGHDqFevHnFxcbRs2ZKwsLB8nXBPRMRpjhyBHj1srS8jR17cX7OmrWWmRYtshcz2f1MYM2crzccspN1bS/m/udtYu8dWyFQPD6CUf9679/OTybiea4JdSHJyMoGBgSQlJWX7wk9NTWXXrl1ER0df1zgLuXl69erF1q1bWbp0qbNTyVf6DIrITZOVBRMnwosv2gb5gm08zNSpYM7etnEkJZUZ6w7wQ+JBthxKtu/3cjfTrEIot1crRasqpSgT5F0g6eb2/X05dTNJofTGG2/Qpk0bfH19mT17NlOmTOG9995zdloiIq7p99+hb19Yv962Xbeu7Wqly64mNQyDdXtPMWXFbmb/eYiMLFt7h7vFRIvKpbi7bhnuqFoabw/nzvh7ORUzUiitWrWKMWPGkJKSQkxMDG+//TY9e/Z0dloiIq7nq69sVyMZBgQFwauvQu/eDksQZGRZ+SHxIJNX7OLPAxdbYepGBnFf/XLE1wonyMfDCclfGxUzUih9/fXXzk5BRKRoaNsWypSxTW73f/8HpUrZD2VmWfl+/QHe+e1v9p6wXY7t4WamY2wZHm1Snlrlcr5QojBRMSMiIlKUGAYsWGArXkwmCAmBjRttP8/LPN8S885vO9h93FbEhPp50L1ZNA83iiTEt/C2wlyJihkREZGi4uxZePxxmDbNNovvhYlRLylkVu06wfAf/mTr4RTbIV8Pet8WQ9cmUS41z9qlXDNrERERcXToEHTsCKtXg5sbJCc7HD6Sksrrv2xlxvoDAAT5uNP7tgo82iQKX0/XLgdcO3sRERGBDRugQwfYt8/WCvP997aJ74Asq8FnK3czdt52UtIyMZng4UaRDLmzCsEu1p2UExUzIiIirmzWLHj4YTh9GqpUsW2fX+rl4KlzPDU9kT92nQCgdrlA/texJrERQU5MOP+pmBEREXFVmzfbupasVtuA32++geBgAGZtPMjzMzaRnJqJr4eFhPbVeLhRJBZz9mUKXJ2WM5ACtXv3bkwmE4mJiTnGLFq0CJPJxKkLM1KKiMi1qV4dxo6FXr1g9mwIDuZ0WibPfLOBfl+tJzk1k9iIIH4e0Jwut0QVyUIGVMy4pG7dumEymejTp0+2Y3379sVkMtHtwgh24MiRI/Tu3ZvIyEg8PT0JCwsjLi6OlStX2mPKly+PyWTKdnv99ddzzKNly5b2OA8PDypUqEBCQgJpaWn2mIiICA4dOkTNmjXz58Vfp5xe34Vby5Ytr/uxW7ZsyaBBg/ItVxGRq8rMvHh/4ED48ENwd2f3sTN0nLCMb9fux2yC/rdX5Ns+TSgfmvPK10WBuplcVEREBNOmTWPcuHF4e9vWxUhNTWXq1KlERkY6xHbq1ImMjAymTJlCTEwM//77LwsWLODEiRMOcS+//DK9evVy2Ofv759rHr169eLll18mPT2d1atX0717dwBGjRoF2FbyDgsLu6HXmh9Wr15NVlYWACtWrKBTp05s27bNvt6Hh0fRGAQnIsXAJ5/Yipc5c+xdSgB//HOc3l+s5dTZDMIDvXjrobo0ig7J5YGKDrXMuKh69eoRGRnJjBkz7PtmzJhBREQEdevWte87deoUy5YtY/To0bRq1YqoqCgaNWpEQkIC8fHxDo/p7+9PWFiYw83XN/dq3sfHh7CwMCIjI+nUqRNt2rRh3rx59uNX6mb65ZdfqFy5Mt7e3rRq1Yrdu3dne9yPPvqIiIgIfHx8uOeeexg7dixBQUEOMT/99BP169fHy8uLmJgYXnrpJTIv/WvlEiVLlrS/ppDz8y2UKlXKvm/r1q3cdttteHt7ExERwYABAzhz5oz9/Pfee49KlSrh5eVF6dKlue+++wBbK9nixYt566237K08V3o9IiL54pNPoGdPWLUKJk+27/5mzT66fPIHp85mEFsukB+ebFZsChlQMZOzM2dyvqWmXnvsuXNXj71O3bt3Z9KkSfbtTz/9lB49ejjE+Pn54efnx8yZMx26fwrChg0bWL58Oe7u7jnG7Nu3j3vvvZf27duTmJhIz549ee655xxili9fTp8+fRg4cCCJiYm0adOGV1991SFm7ty5dOnShQEDBrB582Y++OADJk+enC3uWmzatIm4uDjuvfdeNm7cyPTp01m2bBn9+vUDYM2aNQwYMICXX36Zbdu2MWfOHG47f8njW2+9RZMmTejVqxeHDh3i0KFDRERE5DkHEZGrmjUL/vtf2/0BA2DQIKxWg9dnb2XItxvJyDKIrx3O9N5NKBXg5dxcbzajiEtKSjIAIykpKduxc+fOGZs3bzbOnTuX/UTbhNBXvrVv7xjr45NzbIsWjrGhodlj8uixxx4zOnbsaBw9etTw9PQ0du3aZezevdvw8vIyjh49anTs2NF47LHH7PHffvutERwcbHh5eRlNmzY1EhISjA0bNjg8ZlRUlOHh4WH4+vo63BYuXJhjHi1atDDc3d0NX19fw8PDwwAMs9lsfPvtt/aYXbt2GYCxfv16wzAMIyEhwahWrZphtVrtMUOHDjUA4+TJk4ZhGMaDDz5oxMfHOzxX586djcDAQPt28+bNjddee80h5vPPPzfCw8Ov+v4tXLjQ4fm6du1q/Pe//3WIWbp0qWE2m41z584Z3333nREQEGAkJyfn+D4MHDjwqs97Jbl+BkVELli/3jB8fW3fGY8/bhhWq5GZZTWe/jrRiBo6y4gaOst4c+5WIyvLetWHchW5fX9fTmNmXFhoaCjx8fFMmTIFwzCIj48nNDQ0W1ynTp2Ij49n6dKlrFy5kjlz5jBmzBg+/vhjh4HCQ4YMcdgGKFu2bK45dO7cmRdeeIHk5GRGjx5NQEAAnTp1yjF+y5Yt3HLLLZhMF0fUN7lsCfpt27Zxzz33OOxr1KgRs2bNsm+vXbuW1atXO7TEZGVlkZqaytmzZ/Hx8ck170utXbuWv//+my+//NK+zzAMrFYru3btok2bNkRFRRETE0Pbtm1p27Yt99xzT56eQ0Tkuh08CHfdZWvJv+MOeP99sgx49tuNfLduPxaziTGdatOpfjlnZ+o0KmZycvp0zscuWTYdgCNHco41X9aTl8/jKXr06GHvDnn33XdzjPPy8qJNmza0adOG4cOH07NnT0aMGOFQvISGhlLx/ERL1yowMNB+zhdffEGNGjX45JNPePzxx68YbxjGVR/TMAyHYudK51mtVl566SXuvffebOd7eeWtedVqtdK7d28GDBiQ7VhkZCQeHh6sW7eORYsWMW/ePIYPH87IkSNZvXp1tnE8IiL5rls3OHAAqlWDb78ly+LGkG83MGPdASxmE289VIe7apdxdpZOpWImJ1cZ+HpTYq9B27ZtSU9PByAuLu6az6tevTozZ87M11zc3d15/vnnSUhI4OGHH75iy8WVnvf333932K5atSqrVq1y2LdmzRqH7Xr16rFt27Y8F19XUq9ePf76669cH8vNzY3WrVvTunVrRowYQVBQEL/99hv33nsvHh4e9iulRETy3bvvQvfu8PnnZAUEMuSbDcxYbytk3n6oLvG1w52dodNpALCLs1gsbNmyhS1btmC5vMUIOH78OLfffjtffPEFGzduZNeuXXzzzTeMGTOGjh07OsSmpKRw+PBhh1vyZQuVXc0jjzyCyWTivffeu+LxPn36sHPnTgYPHsy2bdv46quvmHzJiHyA/v3788svvzB27Fh27NjBBx98wOzZsx1aa4YPH85nn33GyJEj+euvv9iyZQvTp0/nxRdfzFO+AEOHDmXlypU8+eSTJCYmsmPHDn788Uf69+8PwKxZs3j77bdJTExkz549fPbZZ1itVqpUqQLY5rD5448/2L17N8eOHcNqteY5BxGRHFWqBEuXYo0qz7PfblQhcwUqZoqAgIAA+3wpl/Pz86Nx48aMGzeO2267jZo1azJs2DB69erFhAkTHGKHDx9OeHi4w+3ZZ5/NUy4eHh7069ePMWPGcPoKXXWRkZF89913/PTTT8TGxjJx4kRee+01h5hmzZoxceJExo4dS2xsLHPmzOGpp55y6D6Ki4tj1qxZzJ8/n4YNG3LLLbcwduxYoqKi8pQvQO3atVm8eDE7duygefPm1K1bl2HDhhEebvtPIigoiBkzZnD77bdTrVo1Jk6cyNSpU6lRowYAzzzzDBaLherVq1OyZEn27t2b5xxERBwsXQqLF1/cNpkY9+t2+xiZdx5WIXMpk3EtgxhcWHJyMoGBgSQlJWX7wk9NTWXXrl1ER0fneZyF3Fy9evVi69atLF261Nmp5Ct9BkUkm2PHIDYWDh+2rX79n//w7dr9PPPNBgDG3FebBxoU/Skgcvv+vpzGzEih9MYbb9CmTRt8fX2ZPXs2U6ZMybHrSkSkyDAM6NHDdgVT1apwxx2s3HmchBkbAejbskKxKGTyyqndTEuWLKFDhw6UKVMGk8nkMDA0IyODoUOHUqtWLXx9fSlTpgyPPvooBw8edF7CctOsWrWKNm3aUKtWLSZOnMjbb79Nz549nZ2WiEjBmjABfvoJPDxg2jT+OWvQ54u1tgnxaoXzzJ1VnJ1hoeTUlpkzZ84QGxtL9+7ds81NcvbsWdatW8ewYcOIjY3l5MmTDBo0iP/85z/ZrmyRoufrr792dgoiIjdXYiI884zt/htvcLJiNXq8t5ykcxnUiQjizQdiMRfRVa9vlFOLmXbt2tGuXbsrHgsMDGT+/PkO+9555x0aNWrE3r17sy2mKCIi4rLOnYOHH4b0dOjQAWvfJxk0ZQ27j5+lXLA3Hz3aAC/37Fesio1LjZlJSkrCZDLlOlFZWlqawxpEeb20WERE5Kb78kvYuhXCw+HTT/lk+W4Wbz+Kp5uZjx9rQEl/T2dnWKi5TDGTmprKc889xyOPPJLrqOZRo0bx0ksv3cTMREREbtDjj4OnJ4SEsDHVjTFztwIwvEN1qoblfiWPuMg8MxkZGTz00ENYrdarXtGSkJBAUlKS/bZv376blKWIiMh1Mpmga1dS7riT/lPXk5Fl0K5mGI800pCKa1HoW2YyMjJ44IEH2LVrF7/99ttVrzX39PTE01PNcSIi4gKWLYNatSAwEMMweHHmn+w5fpayQd68fm/tbOvUyZUV6mLmQiGzY8cOFi5cSIkSJZydkoiISP44fBg6dAAvL1i8mG+Tvfkh8aBtqYKH6xLo4+7sDF2GU7uZTp8+TWJiIomJiQDs2rWLxMRE9u7dS2ZmJvfddx9r1qzhyy+/JCsry75e0IWFFUWcZeTIkdSpU8fZaYiIK3vqKTh1CsqWZV9wOCN+/AuAwW0qUz8q2Lm5uRinFjNr1qyhbt261K1bF4DBgwdTt25dhg8fzv79+/nxxx/Zv38/derUcVgvaMWKFc5M2+m6deuGyWSiT58+2Y717dsXk8lEt27d7PuOHDlC7969iYyMxNPTk7CwMOLi4li5cqU9pnz58phMpmy3119/Pcc8WrZsyaBBg/LzpRWYyZMn53oVXF4988wzLFiwIN8eT0SKmTlzYNo0MJsxPviA4bO2cDY9i0bRIfRpUcHZ2bkcp3YztWzZktyWhiriy0bdkIiICKZNm8a4cePw9vYGbFd8TZ06NdscPJ06dSIjI4MpU6YQExPDv//+y4IFCzhx4oRD3Msvv0yvXr0c9vn7+xfsCylk0tPT8fDwuGqcn58ffn5+NyEjESlyMjNtrTIAAwbws3s4C7etx8NiZtS9tbBoYrw8c4mrmSS7evXqERkZyYwZM+z7ZsyYQUREhL2lC+DUqVMsW7aM0aNH06pVK6KiomjUqBEJCQnEx8c7PKa/vz9hYWEON19f3+vOcejQoVSuXBkfHx9iYmIYNmwYGRkZ9uMbNmygVatW+Pv7ExAQQP369e2zO+/Zs4cOHToQHByMr68vNWrU4JdffrGfu3jxYho1aoSnpyfh4eE899xzZGZmXjGPRYsW0b17d/s8RSaTiZEjRwK2FqlXXnmFbt26ERgYaC/mrpb75d1M3bp14+677+aNN94gPDycEiVK8OSTTzqcIyICwKef2uaUKVGCpGdfYOSPmwHo26oCFUrqj6TrUagHADuFYUDG2Zv/vO4+tkvz8qB79+5MmjSJzp07A/Dpp5/So0cPFi1aZI+50IIwc+ZMbrnllpt6pZe/vz+TJ0+mTJkybNq0iV69euHv78+zzz4LQOfOnalbty7vv/8+FouFxMRE3N1tA96efPJJ0tPTWbJkCb6+vmzevNneEnLgwAHat29Pt27d+Oyzz9i6dSu9evXCy8vLXqRcqmnTpowfP57hw4ezbds2AIdWlf/7v/9j2LBhvPjii9ec+5UsXLiQ8PBwFi5cyN9//82DDz5InTp1srV2iUgxduYMjBhhuz9sGK+vPMSx02lUKOnLEy3VvXS9VMxcLuMsvFbm5j/v8wfBI2+tIF27diUhIYHdu3djMplYvnw506ZNcyhm3NzcmDx5Mr169WLixInUq1ePFi1a8NBDD1G7dm2Hxxs6dKjDFzrArFmzaNmy5XW9pEsfq3z58jz99NNMnz7dXhDs3buXIUOGULVqVQAqVapkj9+7dy+dOnWiVq1aAMTExNiPvffee0RERDBhwgRMJhNVq1bl4MGDDB06lOHDh2M2OzY4enh4EBgYiMlkIiwsLFuet99+O89cWA/lGnO/kuDgYCZMmIDFYqFq1arEx8ezYMECFTMictG5c9CmDaxYwZq2DzB10joAXrunFp5uWq7geqmYcWGhoaHEx8czZcoUDMMgPj6e0NDQbHGdOnUiPj6epUuXsnLlSubMmcOYMWP4+OOPHQYKDxkyxGEboGzZsted37fffsv48eP5+++/OX36NJmZmQ7zBA0ePJiePXvy+eef07p1a+6//34qVLD9ZTJgwACeeOIJ5s2bR+vWrenUqZO9+NqyZQtNmjRxmH+hWbNmnD59mv379+d53a4GDRrkOfcrqVGjBhbLxf+MwsPD2bRpU55yEZEiLjQUPvuM9KQUEj6xFTIPNoigcYymHrkRKmYu5+5jayVxxvNehx49etCvXz8A3n333RzjvLy8aNOmDW3atGH48OH07NmTESNGOBQvoaGhVKxY8bryuNzvv//OQw89xEsvvURcXByBgYFMmzaNN9980x4zcuRIHnnkEX7++Wdmz57NiBEjmDZtGvfccw89e/YkLi6On3/+mXnz5jFq1CjefPNN+vfvj2EY2SaSujBY/HommLp8XNC15H4lF7rILjCZTFit1jznIyJF30fr/mXHkdOU8PUgoX1VZ6fj8jQA+HImk62752bfrnOWx7Zt25Kenk56ejpxcXHXfF716tU5c+bMdT3ntVi+fDlRUVG88MILNGjQgEqVKrFnz55scZUrV+app55i3rx53HvvvUyaNMl+LCIigj59+jBjxgyefvppPvroI3vuK1ascLjabcWKFfj7++fYkuTh4UFWVla+5i4ics127rStiv333xw7ncZ7C/8G4IX4agT5XP0KSsmdWmZcnMViYcuWLfb7lzt+/Dj3338/PXr0oHbt2vj7+7NmzRrGjBlDx44dHWJTUlI4fPiwwz4fH59cu1eOHj1qn/TwgrCwMCpWrMjevXuZNm0aDRs25Oeff+b777+3x5w7d44hQ4Zw3333ER0dzf79+1m9ejWdOnUCYNCgQbRr147KlStz8uRJfvvtN6pVqwbY5tIZP348/fv3p1+/fmzbto0RI0YwePDgbONlLihfvjynT59mwYIFxMbG4uPjg4/PlVvDrpa7iEievfiibV6ZlBTe7vU6Z9KzqFU2kLvrXH9XvlyklpkiICAgIMeCw8/Pj8aNGzNu3Dhuu+02atasybBhw+jVqxcTJkxwiB0+fLjD5ITh4eG5DngF+Oqrr+wTH164TZw4kY4dO/LUU0/Rr18/6tSpw4oVKxg2bJj9PIvFwvHjx3n00UepXLkyDzzwAO3atbOveJ6VlcWTTz5JtWrVaNu2LVWqVLEvMlq2bFl++eUXVq1aRWxsLH369OHxxx/PNnj5Uk2bNqVPnz48+OCDlCxZkjFjxuQYe7XcRUTyZMMGWyFjMnHgmRf56o+9ACS0r4pZc8rkC5NRxGemS05OJjAwkKSkpGxf+KmpqezatYvo6Gi8vLyclKEUZ/oMihQDDz0E06fDQw/x5F1D+HnTIVpWKcnk7o2cnVmhltv39+XUMiMiIlJQduyAb74BYEu3J/l50yFMJniunQb95icVMyIiIgVlzBiwWjHi4xmx2zausVO9clQNy72lQfJGxYyIiEhBOHAApkwBYM1DvVm16wSebmYGt6ns5MSKHl3NJCIiUhD8/WHECIx163n+SABwmu7NoikT5O3szIocFTMiIiIFISAAXniBHxMPsGNaIkE+7lp/qYCom0lERKSAWK0G7y3cCUDPW6MJ9Ha/yhlyPVTMiIiI5KeUFGjbFn74gd82H2bbvyn4ebrRtUl5Z2dWZKmbSUREJD99+CHMnYuxaxfv9bMtw9Lllii1yhQgtcyIiIjkl8xMeOstAHZ2e4J1B1LwdDPz+K3RTk6saFMxIyIikl9mzYJ9+yA0lNeC6wHwYMMISvp7Ojmxok3FjAvq1q0bJpOJPn36ZDvWt29fTCYT3bp1s+87cuQIvXv3JjIyEk9PT8LCwoiLi2PlypX2mPLly2MymbLdXn/99RzzaNmyJYMGDcrPl1YgLrxfud2u18iRI6lTp07+JSsiru38GnJH7u/Mb7tTsJhN9Goe4+Skij6NmXFRERERTJs2jXHjxuHtbZuzIDU1lalTpxIZGekQ26lTJzIyMpgyZQoxMTH8+++/LFiwgBMnTjjEvfzyy/Tq1cthn7+/f8G+kJvgrbfecijKwsPDmTRpEm3btnViViJS5GzfDvPng8nE+Iqt4Ah0rFOGiBAfZ2dW5KllxkXVq1ePyMhIZsyYYd83Y8YMIiIiqFu3rn3fqVOnWLZsGaNHj6ZVq1ZERUXRqFEjEhISiI+Pd3hMf39/wsLCHG6+vr7XnePQoUOpXLkyPj4+xMTEMGzYMDIyMuzHN2zYQKtWrfD39ycgIID69euzZs0aAPbs2UOHDh0IDg7G19eXGjVq8Msvv9jPXbx4MY0aNcLT05Pw8HCee+45MjMzr5hHYGCgw2sCCAoKsm9nZWXx4IMPEhwcTIkSJejYsSO7d++2n79o0SIaNWqEr68vQUFBNGvWjD179jB58mReeuklNmzYYG/hmTx58nW/XyLi4s63ypxuHcdXR9wwmaCv5pW5KdQyk5MzZ3I+ZrHApSsc5xZrNoO3d+6x11kwdO/enUmTJtG5c2cAPv30U3r06MGiRYvsMX5+fvj5+TFz5kxuueUWPD1vXr+tv78/kydPpkyZMmzatIlevXrh7+/Ps88+C0Dnzp2pW7cu77//PhaLhcTERNzdbaP9n3zySdLT01myZAm+vr5s3rwZPz8/AA4cOED79u3p1q0bn332GVu3bqVXr154eXkxcuTIPOV49uxZWrVqRfPmzVmyZAlubm688sortG3blo0bN2I2m7n77rvp1asXU6dOJT09nVWrVmEymXjwwQf5888/mTNnDr/++itgK5xEpJhq3hxWreKrBh3ACndWL03FUq7fuu0SjCIuKSnJAIykpKRsx86dO2ds3rzZOHfuXPYTIedb+/aOsT4+Oce2aOEYGxqaPSaPHnvsMaNjx47G0aNHDU9PT2PXrl3G7t27DS8vL+Po0aNGx44djccee8we/+233xrBwcGGl5eX0bRpUyMhIcHYsGGDw2NGRUUZHh4ehq+vr8Nt4cKFOebRokULY+DAgdec95gxY4z69evbt/39/Y3JkydfMbZWrVrGyJEjr3js+eefN6pUqWJYrVb7vnfffdfw8/MzsrKyrpoHYHz//feGYRjGJ598ku2x0tLSDG9vb2Pu3LnG8ePHDcBYtGjRFR9rxIgRRmxs7FWfMye5fgZFxOUcTUk1KiX8bEQNnWWs23PC2em4tNy+vy+nbiYXFhoaSnx8PFOmTGHSpEnEx8cTGhqaLa5Tp04cPHiQH3/8kbi4OBYtWkS9evWydYkMGTKExMREh1vjxo2vO79vv/2WW2+9lbCwMPz8/Bg2bBh79+61Hx88eDA9e/akdevWvP766+zcudN+bMCAAbzyyis0a9aMESNGsHHjRvuxLVu20KRJE4eBu82aNeP06dPs378/TzmuXbuWv//+G39/f3srVkhICKmpqezcuZOQkBC6detGXFwcHTp04K233uLQoUPX/Z6ISNE2ffU+0q0GseUCqRsZ7Ox0ig0VMzk5fTrn23ffOcYeOZJz7OzZjrG7d2ePuQE9evRg8uTJTJkyhR49euQY5+XlRZs2bRg+fDgrVqygW7dujBgxwiEmNDSUihUrOty8va9vQbTff/+dhx56iHbt2jFr1izWr1/PCy+8QHp6uj1m5MiR/PXXX8THx/Pbb79RvXp1vv/+ewB69uzJP//8Q9euXdm0aRMNGjTgnXfeAcAwjGxXIBmGAZDnK5OsViv169fPVsRt376dRx55BIBJkyaxcuVKmjZtyvTp06lcuTK///77db0vIlIErVkDb7xB5tFjfPn7HgDN9nuTqZjJia9vzrdLx8tcLfbyYuBKMTegbdu2pKenk56eTlxc3DWfV716dc7kNtbnBi1fvpyoqCheeOEFGjRoQKVKldizZ0+2uMqVK/PUU08xb9487r33XiZNmmQ/FhERQZ8+fZgxYwZPP/00H330kT33FStW2AsYgBUrVuDv70/ZsmXzlGe9evXYsWMHpUqVylbIXTr+pW7duiQkJLBixQpq1qzJV199BYCHhwdZWVl5ek4RKWLGjYMhQzjUdxAHk1IJ9nHnrtrhzs6qWFEx4+IsFgtbtmxhy5YtWCyWbMePHz/O7bffzhdffMHGjRvZtWsX33zzDWPGjKFjx44OsSkpKRw+fNjhlpycnOvzHz16NFurxuHDh6lYsSJ79+5l2rRp7Ny5k7ffftve6gJw7tw5+vXrx6JFi9izZw/Lly9n9erVVKtWDYBBgwYxd+5cdu3axbp16/jtt9/sx/r27cu+ffvo378/W7du5YcffmDEiBEMHjwYszlvH+nOnTsTGhpKx44dWbp0Kbt27WLx4sUMHDiQ/fv3s2vXLhISEli5ciV79uxh3rx5bN++3Z5L+fLl2bVrF4mJiRw7doy0tLQ8Pb+IuLgjR+CbbwD4oGobAB5sGImXe/b/j6UAFfQAHme77gHAhdiFAcA5uXQAcGpqqvHcc88Z9erVMwIDAw0fHx+jSpUqxosvvmicPXvWfk5UVJQBZLv17t07x+dp0aLFFc8ZMWKEYRiGMWTIEKNEiRKGn5+f8eCDDxrjxo0zAgMDDcOwDbJ96KGHjIiICMPDw8MoU6aM0a9fP/vvol+/fkaFChUMT09Po2TJkkbXrl2NY8eO2Z970aJFRsOGDQ0PDw8jLCzMGDp0qJGRkXFN7x+XDAA2DMM4dOiQ8eijjxqhoaGGp6enERMTY/Tq1ctISkoyDh8+bNx9991GeHi44eHhYURFRRnDhw+3DzROTU01OnXqZAQFBRmAMWnSpGvK4QJX/QyKyHn/93+GAca5eg2MqKGzjPLPzTL2Hj/j7KyKhLwMADYZxiVt9UVQcnIygYGBJCUlERAQ4HAsNTWVXbt2ER0djdflXUciN4E+gyIuzDCgdm34809+6juC/v4NaV2tFB8/1tDZmRUJuX1/X07dTCIiItdj3Tr4808MT09G+dUCNPDXWVTMiIiIXI8pUwDY07wNB01elC/hQ/OK2afHkIKnYkZEROR6pKVheHjwcUxzALrcEoXZfP0L18r1UzEjIiJyPT74gPW//8XUwKp4uZu5v36EszMqtlTMAEV8DLQUYvrsibi2aTvPkGW20KF2GQJ93J2dTrFVrIuZC4sanj171smZSHF14bN34bMoIi7g1CnYvp2z6Zn8vNG2vMn9DdQq40zFetVsi8VCUFAQR44cAcDHxyfP0+GLXA/DMDh79ixHjhwhKCjoihMeikgh9fnnMGAAR+55mDOVOxMZ4kPD8lqHyZmKdTEDEBYWBmAvaERupqCgIPtnUERcxPmrmJZ42v7tdqpXTn8IO1mxL2ZMJhPh4eGUKlWKjIwMZ6cjxYi7u7taZERczV9/wdq1GG5uvFWqAQD31svbmnCS/4p9MXOBxWLRF4uIiOTufKvMrsatOO4dSOPoECJCfJyclDh1APCSJUvo0KEDZcqUwWQyMXPmTIfjhmEwcuRIypQpg7e3Ny1btuSvv/5yTrIiIlK8Wa3w5ZcAfFrxNgDuq1/OmRnJeU4tZs6cOUNsbCwTJky44vExY8YwduxYJkyYwOrVqwkLC6NNmzakpKTc5ExFRKTYW7ECDh4k0z+Ar0Nr4u1uoV2tcGdnJTi5m6ldu3a0a9fuiscMw2D8+PG88MIL3HvvvQBMmTKF0qVL89VXX9G7d+8rnpeWlkZaWpp9Ozk5Of8TFxGR4ue77wDYUL8l6W7u3FsrDD9PjdYoDArtPDO7du3i8OHD3HnnnfZ9np6etGjRghUrVuR43qhRowgMDLTfIiJ07b+IiOSDV18lfdrXjKnUBoD76qmLqbAotMXM4cOHAShdurTD/tKlS9uPXUlCQgJJSUn22759+wo0TxERKSZ8fJhbtRl/hERTNsibW2JKODsjOa/Qt49dfu2+YRi5Xs/v6emJp6dnQaclIiLF0Ldr9wO2y7G1qGThUWiLmQsTiR0+fJjw8IsDrI4cOZKttUZERKTAZGVBmzacbXIr61Nrg7s399TV3DKFSaHtZoqOjiYsLIz58+fb96Wnp7N48WKaNm3qxMxERKRYWb4cFi7E7e23OGd2o0aZAGJK+jk7K7mEU1tmTp8+zd9//23f3rVrF4mJiYSEhBAZGcmgQYN47bXXqFSpEpUqVeK1117Dx8eHRx55xIlZi4hIsfL11wD8Xrs5GRZ34mvrcuzCxqnFzJo1a2jVqpV9e/DgwQA89thjTJ48mWeffZZz587Rt29fTp48SePGjZk3bx7+/v7OSllERIqTrCz7JdmTy9qWL4jX3DKFjskwDMPZSRSk5ORkAgMDSUpKIiAgwNnpiIiIK1m8GFq2JM0/gJq9p1A1MpSf+t/q7KyKhbx8fxfaMTMiIiJOd76LaWUtdTEVZipmREREruSSLqYpZRsC6mIqrFTMiIiIXElyMtx5J6fDy7EsKpbYcoFaIbuQKrTzzIiIiDhVcDB89hn/nbiCjN0n1cVUiKllRkREJAdHUlL5fc9JANqri6nQUsuMiIjI5Q4ehGPHmHvaH6sBsRFBlAtWF1NhpZYZERGRy02aBLGxlBvSH4C71CpTqKmYERERudwPPwAwNzAGgHa1wpyZjVyFihkREZFLHTwIq1cDsCCmEXUj1cVU2KmYERERudSsWQD8HVODo37BtK2hVpnCTsWMiIjIpX78EYAfIuoD0KZ6aWdmI9dAxYyIiMgFZ87Ar78CMKdCY2JK+hJT0s/JScnVqJgRERG5YP58SEvjWKly7AiNVKuMi9A8MyIiIhe0b0/m7Dm89ukyMJloU03FjCtQMSMiInKBhwerKtZnRkwmJXw9qBsZ7OyM5Bqom0lEROQS8zb/C8Ad1UphMZucnI1cC7XMiIiIAIwbh7F3LzupCZ5htFYXk8tQy4yIiAjAhx9iGj+ewH+24+lmpnmlks7OSK6RihkREZF//oGtW8myuLE4pj7NK4Xi7WFxdlZyjVTMiIiIzJkDwJboGqR4+uqSbBejYkZERGT2bAB+LlsHkwlur6pixpWomBERkeItLQ1++w2AxdH1qRsRREl/TycnJXmhYkZERIq3pUvh7FlOBZZgc6loWquLyeXo0mwRESneTpzAWqYMC0Kra9ZfF6ViRkREircHHmBhzdsY9vEyygV7U7GUFpZ0NepmEhGRYm/R9mOc9fCmZZWSmEya9dfVqJgREZHi6/hxjKwsFm0/AkDLyqWcnJBcDxUzIiJSfPXsibV0aar/vgAPi5mmFUs4OyO5DhozIyIixVN6OixYgCUlhYP+JWkUHYKPh74WXZFaZkREpHhauRJSUkj2D+bPsAq0qKy1mFyVihkRESmezs/6uzCqDobJTMsqKmZclYoZEREpns6vx7Qguj5lg3RJtitTMSMiIsXPwYOwYQNWk4ml5evSQpdkuzQVMyIiUvzMnQvA1nJVOOkTqPEyLk7DtkVEpPhp2JBTg55hytY03C0mmlUMdXZGcgPUMiMiIsVPzZrMvK8v02PjaBAVgp+n/rZ3ZSpmRESkWFq0/SiArmIqAlSKiohI8fLbb6Qnp7BxcyZYvGihYsblqWVGRESKl9Gj8bjnbjqun0tYgBdVSvs7OyO5QYW6mMnMzOTFF18kOjoab29vYmJiePnll7Farc5OTUREXFFqKixZAsDS8nW1SnYRUai7mUaPHs3EiROZMmUKNWrUYM2aNXTv3p3AwEAGDhzo7PRERMTVrFgBqakcDyjB3yUieKqSupiKgkJdzKxcuZKOHTsSHx8PQPny5Zk6dSpr1qxxcmYiIuKSfv0VgEURtTGZTTStoFWyi4JC3c106623smDBArZv3w7Ahg0bWLZsGe3bt8/xnLS0NJKTkx1uIiIiAMyfD8Cy8nWpVTaQYF8PJyck+aFQt8wMHTqUpKQkqlatisViISsri1dffZWHH344x3NGjRrFSy+9dBOzFBERl3DiBKxdC8DyqFg6aaK8IqNQt8xMnz6dL774gq+++op169YxZcoU3njjDaZMmZLjOQkJCSQlJdlv+/btu4kZi4hIobVsGRgG/5SK4oh/CZqrmCkyCnXLzJAhQ3juued46KGHAKhVqxZ79uxh1KhRPPbYY1c8x9PTE09Pz5uZpoiIuIIOHfhn2Vqeff9XvNzN1IsKdnZGkk8KdcvM2bNnMZsdU7RYLLo0W0RE8s5kYqERzJpyNWhYPgQvd4uzM5J8UqhbZjp06MCrr75KZGQkNWrUYP369YwdO5YePXo4OzUREXFBy3bYljBoXkldTEVJoS5m3nnnHYYNG0bfvn05cuQIZcqUoXfv3gwfPtzZqYmIiCv58kusM2bg41YLohtya0XNL1OUFOpixt/fn/HjxzN+/HhnpyIiIq7shx8wz5hBhVu9Ca3VjKphWsKgKCnUY2ZERERumNUKCxYAsCyqDk0rhGI2awmDokTFjIiIFG0bNsCJE5zz9GZjeCVu1XiZIkfFjIiIFG2//QbAynI1ybS4afBvEaRiRkREirbzxczyyNpUKOlLeKC3kxOS/KZiRkREiq6MDFiyBICVUbHcqll/iyQVMyIiUnT9+y/UqsUJv2C2lCrPrZV0SXZRpGJGRESKrnLl2P/zrzTu8ylmi4VbYkKcnZEUABUzIiJSpK3YeZwMizux5QLx93J3djpSAAr1pHkiIiLXLSMDzp1j5c7jADStoPEyRZVaZkREpGhavhwjJIR7RvYFoGmFEk5OSAqKihkRESmafvsNU1YWJ8weeLiZqRcV7OyMpIComBERkaJp4UIAVkTGUj8yGC93i5MTkoKiYkZERIqeM2fg998BWBFVm2YV1cVUlKmYERGRomfZMsjM5GBQafYHhdFEg3+LNBUzIiJS9JxfwmBZRC18PSzULhfo5ISkIOWpmBkzZgznzp2zby9ZsoS0tDT7dkpKCn379s2/7ERERK7H+WJmRVQsjaJDcLfob/eiLE+/3YSEBFJSUuzbd911FwcOHLBvnz17lg8++CD/shMREbkevXqxuuEdrIyspfllioE8TZpnGEau2yIiIoVBxuM96bY/kjPpWTTR/DJFntrdRESkyNm4P4kz6VkE+bhTPTzA2elIAdNyBiIiUrRMm8aW1CAwDJrElMBsNjk7IylgeS5mPv74Y/z8/ADIzMxk8uTJhIba+iMvHU8jIiJy0x0/Dg8/TBdgXL8vaFqhprMzkpsgT8VMZGQkH330kX07LCyMzz//PFuMiIiIUyxeDMD2klEc9w3S/DLFRJ6Kmd27dxdQGiIiIvng/BIGKyNqUsrfkwolfZ2ckNwMGgAsIiJFx6JFAKyMrE3TCiUwmTRepjjIUzHzxx9/MHv2bId9n332GdHR0ZQqVYr//ve/DpPoiYiI3DRHj8KffwLwR0RNXZJdjOSpmBk5ciQbN260b2/atInHH3+c1q1b89xzz/HTTz8xatSofE9SRETkqs63ymwtWZ6TPoGaLK8YyVMxk5iYyB133GHfnjZtGo0bN+ajjz5i8ODBvP3223z99df5nqSIiMhV2buYalE2yJuIEB/n5iM3TZ4GAJ88eZLSpUvbtxcvXkzbtm3t2w0bNmTfvn35l52IiMi1GjWKr0vXZtqeTHUxFTN5apkpXbo0u3btAiA9PZ1169bRpEkT+/GUlBTc3d3zN0MREZFrERDAlyVqsq1keZrEqJgpTvJUzLRt25bnnnuOpUuXkpCQgI+PD82bN7cf37hxIxUqVMj3JEVERK4mJTWDPw8kAahlppjJUzfTK6+8wr333kuLFi3w8/Nj8uTJeHh42I9/+umn3HnnnfmepIiISK7eeIPj2/dS3lqdzCpVKBPk7eyM5CbKUzFTsmRJli5dSlJSEn5+flgsFofj33zzDf7+/vmaoIiIyFV99BHlt28n+t5hlIhu6uxs5CbLUzHTo0ePa4r79NNPrysZERGRPDt4ELZvx2oysSqiBi+ri6nYyVMxM3nyZKKioqhbty6GYRRUTiIiItfu/CXZf5WKIdnLT+NliqE8FTN9+vRh2rRp/PPPP/To0YMuXboQEhJSULmJiIhc3YX1mCJrExPqS+kALycnJDdbnq5meu+99zh06BBDhw7lp59+IiIiggceeIC5c+eqpUZERJzjwmR5UbW5Ra0yxVKeF5r09PTk4YcfZv78+WzevJkaNWrQt29foqKiOH36dEHkKCIicmX798Pff5NlMrOmXHXNL1NM5amb6XImkwmTyYRhGFit1vzKSURE5Nr88w/WEqH86R5Miqcvt6iYKZby3DKTlpbG1KlTadOmDVWqVGHTpk1MmDCBvXv34ufnVxA5ioiIXNlttzHvt0S63z+SSqX8KOnv6eyMxAnyVMz07duX8PBwRo8ezV133cX+/fv55ptvaN++PWZznuuia3LgwAG6dOlCiRIl8PHxoU6dOqxdu7ZAnktERFzPyl0nOeETqKuYirE8dTNNnDiRyMhIoqOjWbx4MYsXL75i3IwZM/IluZMnT9KsWTNatWrF7NmzKVWqFDt37iQoKChfHl9ERFxYVhaYzaz85ziAxssUY3kqZh599FFMJlNB5ZLN6NGjiYiIYNKkSfZ95cuXv2nPLyIihdjnn5P1wovExbRke7OHaaxiptjK86R5N9OPP/5IXFwc999/P4sXL6Zs2bL07duXXr165XhOWloaaWlp9u3k5OSbkaqIiNxsCxdiOXgAr8h0qob5E+LrcfVzpEgqmIEu+eSff/7h/fffp1KlSsydO5c+ffowYMAAPvvssxzPGTVqFIGBgfZbRETETcxYRERuCsNwmCxP42WKN5NRiGe78/DwoEGDBqxYscK+b8CAAaxevZqVK1de8ZwrtcxERESQlJREQEBAgecsIiI3wc6dULEiGRY3ag2cxts9mnFnjTBnZyX5KDk5mcDAwGv6/i7ULTPh4eFUr17dYV+1atXYu3dvjud4enoSEBDgcBMRkSLmfKvM+vDKpHt4abxMMVeoi5lmzZqxbds2h33bt28nKirKSRmJiEihcEkXU40ygQR6uzs5IXGmQl3MPPXUU/z++++89tpr/P3333z11Vd8+OGHPPnkk85OTUREnOWS8TK/a7yMUMiLmYYNG/L9998zdepUatasyf/+9z/Gjx9P586dnZ2aiIg4S1oaPPwwmyOqsq5sVc0vI4V7AHB+yMsAIhERcQ37T57l1tELsZhNbBhxJ36eN7TUoBRCRWYAsIiIyJWs3Gmb9bd2uUAVMnJjq2aLiIjcVIYBixaxZr8XoCUMxEYtMyIi4jq2bIHbb+fF/7bBYs3S4F8BVMyIiIgrOX8V08bSFTC7u9EgKsTJCUlhoGJGRERcxyXzy9SNCMbbw+LkhKQwUDEjIiKuwWqFRYsAWBEZyy3qYpLzVMyIiIhr2LABjh/njIc3G8MrafCv2KmYERER1/DbbwD8Ua4GZk8P6kYGOTcfKTRUzIiIiGtYsACA5VGx1I8Mxstd42XERvPMiIiIaxgzhm9LVOdXnyrcp/EycgkVMyIi4hKs1WvwaqU7OXk2Q/PLiAN1M4mIiEvYcjiZk2cz8PWwEBsR5Ox0pBBRy4yIiBR+r77KsRR3AlKjqF8lGneL/haXi1TMiIhI4Xb2LLz8Mi3S0wnp9QHNKoY6OyMpZFTaiohI4bZsGaSncyigJLuDy6iYkWxUzIiISOF2fn6ZZZGxlPDzpEppfycnJIWNihkRESnczs8vsyKqNk0qlMBsNjk5ISlsVMyIiEjhdfIkrF0L2IoZdTHJlaiYERGRwmvxYjAMdoaU41//UJpVUDEj2amYERGRwmvLFsC2hEG5YG8iS/g4OSEpjHRptoiIFF4JCYyNuo3py/+mpVplJAdqmRERkULt16NW/vUPpWlFLWEgV6aWGRERKZwMgxNnM9h8KBmApmqZkRyoZUZERAqn//4Xo2VLmuzZSJXS/pT093R2RlJIqZgREZHCxzBg1ixKrFkJGOpiklypmBERkcLnzz/h8GHOuXuxtmx1XZItuVIxIyIihc/cuQCsjKhJlocHjWNCnJyQFGYqZkREpPCZNw+ApdF1qV0uEH8vdycnJIWZihkRESlczp2DJUsAWFK+HrdVKunkhKSwUzEjIiKFy9KlkJbG4YBQdpYox22VNV5Gcqd5ZkREpHDx8SGpdVtmn3TH39ud2HJBzs5ICjkVMyIiUrjceitTnnubsfO3065iKG4WdSJI7vQJERGRQmfJ9qMANNd4GbkGapkREZHCY8cOUqywft8pAJpX0ngZuTq1zIiISOExciT+VSvz+MpviQn1JSLEx9kZiQtQMSMiIoWD1Qrz5wOQWKYKt1VWF5NcGxUzIiJSOGzYAEePctbDm/VlquiSbLlmKmZERKRwOD/r7/LIWuDhQeNoLS4p10bFjIiIFA7n12NaEl2PBlEh+HrqGhW5NipmRETE+ZKTbTP/AkvL19V4GckTlypmRo0ahclkYtCgQc5ORURE8tP8+ZCZye6QsuwOKatLsiVPXKYNb/Xq1Xz44YfUrl3b2amIiEh+u/NOtr07mQlz/iLUz4Pq4QHOzkhciEu0zJw+fZrOnTvz0UcfERwcnGtsWloaycnJDjcRESnk/P2ZGdOYn6q34NaKoZjNJmdnJC7EJYqZJ598kvj4eFq3bn3V2FGjRhEYGGi/RURE3IQMRUTkRl1YwkDjZSSvCn0307Rp01i3bh2rV6++pviEhAQGDx5s305OTlZBIyJSmH38Mad37ubs8UhMJcpqPSbJs0JdzOzbt4+BAwcyb948vLy8rukcT09PPD09CzgzERHJN++/j9+6ddRr/xSBsTUo6a//wyVvCnU309q1azly5Aj169fHzc0NNzc3Fi9ezNtvv42bmxtZWVnOTlFERG7EwYOwbh1Wk4lFMfW5o2opZ2ckLqhQt8zccccdbNq0yWFf9+7dqVq1KkOHDsVisTgpMxERyRe//ALApjKVOe4bxO3VVMxI3hXqYsbf35+aNWs67PP19aVEiRLZ9ouIiAuaNQuA+TENCQ/00iXZcl0KdTeTiIgUYamp8OuvACys0JDbq5bCZNIl2ZJ3hbpl5koWLVrk7BRERCQ/LF4MZ85wNKAEf5WK4Wl1Mcl1crliRkREiohDh8gKDGR++YZ4eVhoWkFLGMj1UTEjIiLO0a0bH5S7hYmzNnBrxVC83HVRh1wfjZkRERGnmb/9OMleftxetbSzUxEXppYZERG5+Y4d45inH4n7TgFwu+aXkRuglhkREbn52rXDu2pl6uzfSo0yAYQFXtss7yJXopYZERG5uXbvhjVr8DaZ2RsURme1ysgNUsuMiIjcXDNmALAmsibHfYO4o5rGy8iNUTEjIiI317ffAjCrUhNC/TypVTbQyQmJq1MxIyIiN8+BA7ByJQBzKjfl9qolMZs166/cGBUzIiJy85zvYtoQWZ0j/iVoVyvcyQlJUaBiRkREbp7zXUw/VmxKgJcbzTTrr+QDFTMiInLzjBzJ6nYPMqdyU9pUD8PDTV9DcuN0abaIiNw01hYt6bcyi3+T02hfK8zZ6UgRoZJYRERumnV7T/Jvchr+nm7cWkldTJI/1DIjIiIF7/hxeOUVNoXVA0JoXb00nm5aWFLyh1pmRESk4P3wA4wfT7O3XgKgXU11MUn+UTEjIiIF77vvAPixYhN8PSzcVrmkkxOSokTFjIiIFKzjx2H+fABmV27GHdVK4+WuLibJPypmRESkYE2bBhkZbCtTkZ2hEbqKSfKdihkRESlYn30GwPSqrfDxsNCyilbJlvylYkZERArO1q2wahVWs4UfqregVdVS6mKSfKdLs0VEpODs2YNRrhwr/ctx3DeIeK3FJAVALTMiIlJw4uJYu2gd/Vr3P9/FpKuYJP+pmBERkQL13YZDnPQJpH2tcHw81CEg+U/FjIiIFIwNG0g9l8asDYcA6FSvnJMTkqJKxYyIiOS/06ehWTMoVw7/owcpF+xN4+gQZ2clRZSKGRERyX8zZsCZM5x09+agf0nurVcOs9nk7KykiFIxIyIi+e/83DJfVW4BJhOd6pV1ckJSlKmYERGR/LVvH/z2GwDf12hFo/IhRJXwdXJSUpSpmBERkfz12WdgGCTGxLI/sDT31dfAXylYKmZERCT/ZGbCBx8A8Fm12/FyN9NOazFJAVMxIyIi+WfJEti3j9MBwcyqehtta4Th7+Xu7KykiFMxIyIi+ef220lbtYYX2vUn3c2d++pHODsjKQZUzIiISL761bMMP5RvRHigF00qlHB2OlIMqJgREZH8kZoKwBe/7wHg3nplsWhuGbkJVMyIiMiNO3ECypbl1INdWL/tABazic6No5ydlRQTKmZEROTGffopnDjB2VVrSHXzpG2NMMoEeTs7KykmVMyIiMiNycqCd98F4N0abcFkonuz8s7NSYoVrcUuIgXv0CH444+cj8fGQnS07f7Ro7B8ec6xNWtCxYq2+ydO2C4FzknVqrYbQFISLFyYc2ylSlCjhu3+mTMwf37OsTExULu27X5qKsyZk3NsZCTUq2e7n5EBP/+cc2zZstCwoe2+YcAPP+QcGxYGt9xycfvHH8FqvXJsaCjceuvF7Z9/tuVyJcHB0KLFxe25c+HcuSvHBgTA7bfDL7/A7t2ke/tyyuJJz+MbqL8+HRIvGS/j4wN33nlxe+FC2+/kSjw9oV27i9tLl8Lx41eOdXODu+66uL1iBRw5cuVYgLvvvnj/jz9sn82cdOgAFovt/tq1tpmNc9K+PXh42O4nJsLu3TnH3nmn7f0A2LQJdu7MOfaOO8Df33Z/yxbYti3n2JYtISjIdn/7dti8OefYW2+1fS7A9vybNuUc26QJlC5tu79nD6xf73i8WjWoUiXn828GoxB77bXXjAYNGhh+fn5GyZIljY4dOxpbt27N02MkJSUZgJGUlFRAWYpINlar4/aPPxqG7ev5yrf3378Yu2BB7rFvvHExduXK3GNfeuli7MaNuccOHXoxdufO3GP79bsYe+hQ7rHdu1+MTU7OPfbBBy/GZmXlHnvXXY7vsadnzrGtWjnGhoTkHNu4sWNsRETOsTVr2mLatDEMME55++ccGx3t+Lj16uUcW7q0Y2zz5jnH+vk5xsbF5RxrMjnG3ntv7u/xuXMXY7t2zT322LGLsb175x67Z8/F2MGDc4/dsuVi7LBhuceuWXMxdtSo3GMXL74Y+/bbucfOnn0x9pNPsh9/5RWjIOTl+7tQt8wsXryYJ598koYNG5KZmckLL7zAnXfeyebNm/H11TofIoXWlCnw3nswYAB06WL7a79p05zjL/zVBxAYmHtsmTIX7/v55R4bcckcJz4+ucdGRV287+mZe+yFViQAd/fcYy+0IgGYzbnHVq7suJ1bbLVqjttNmkB6+pVjL7Q4XdCoESQnX1tsgwaO7+OlYmJsrRDz52OYzawuW42S6aepVTYQs+myq5gu/b2BrTXOy+vKjxsS4rhds6atK+tKLrRwXFC9OqSkXDn28pyqVMn9Pb40vlKl3GPdLvk6rVAh99gLLTgA5cvnHut9ybijiIjcYy/9XixbNvfYgICL98PDc4+90NoDUKpU9thyzl+uwmQYhuHsJK7V0aNHKVWqFIsXL+a22267pnOSk5MJDAwkKSmJgEt/eSJScNq3h9mz4eWXYdgwZ2cjBemRR2DqVJY1aE2XOwYx4I5KDG5T+erniVxFXr6/C3XLzOWSzvevhlxetV8iLS2NtLQ0+3ZyTn99iEjBOHkSfv3Vdv/++52bixS8CRP4t1Q5RpyOwd1iokvjSGdnJMWQy1zNZBgGgwcP5tZbb6VmzZo5xo0aNYrAwED7LSKn5lERKRg//GAbXFqz5sXBt1J0hYQwuukj7AyNIL5WOKUCcug6EilALlPM9OvXj40bNzJ16tRc4xISEkhKSrLf9uU2+lxE8t8339h+qlWmaDtzBgyDw0mpzNpguyKoW7Poq5wkUjBcopupf//+/PjjjyxZsoRyVxlo5Onpiaen503KTEQcnDp18ZJmFTNFW5cucOwY3907iPQsLxqVD6FORJCzs5JiqlAXM4Zh0L9/f77//nsWLVpEdLSqfpFC7ccfbV1MNWpkv9pGio7162HmTAyTiR+rd4HgcjylQb/iRIW6mHnyySf56quv+OGHH/D39+fw4cMABAYG4u2tabJFCp3oaOjUyXbpr9UKaRqAXyS9Mgw84c+mbTgUHEzraE+alLHAuVPOzkycxc0T3J33vVyoL802XT4nwHmTJk2iW7du1/QYujRbxAky02BicziWy2ylIlJ03DoYWo/I14csMpdmF+I6S0Rys32OChkRuWkKdTEjIi5k+nSoU8c2q+qG6bZ9TQfAHcOdmpbko5nfwwMPYnh40Oaxt9gdFM43fZpQNyLY2ZmJs5mce3G0ihkRuXFJSfDoo7bp9BP/gB3zbPtjHwaLu3Nzk/zz1XSwwvy2Xfk7KILbq5aibvlSzs5KRMWMiOSDn36yFTLVqkH6RrBmQFgtKF3d2ZlJfvr6a/599yMG7i0JoGULpNBwmUnzRKQQmzTJ9vO++y52MdV+yHn5SMGwWHgxqAHn3Ly4s3ppapYNdHZGIoCKGRG5UYmJ8NtvYLHAfXfC/lW2/vNa9zk7M8kPhgGTJ8PZs/y6+V/mb/4XN7OJZ+KqODszETsVMyJyY8aNs/28/344scx2P6Yl+Ic5LSXJRz/+CN27Y42tw/9mJALwePNoKpf2d25eIpdQMSMi1+/gQbiwXtqgQbBRXUxFypkztt8r8Ef9Vuw5nUnZIG8G3lHJuXmJXEYDgEXk+v31F/j7Q/XqUMYMJ/4Bdx+oGu/szCQ/DBoEu3eTUbYcvSPiABjRoTo+HvrqkMJFn0gRuX5t2sC+fXD4MGx817avWgfw9HNuXnLjvvkGPv4Yw2Ri1INDSbZ4ckfVUrSpXtrZmYlko24mEbkxPj4QWQ7+nGHbrv2gc/ORG7dnD/TqBcD2bn351D0aL3czI/9TI8dlZkScScWMiOSd1Qq//mr7CfD3r3DuBPiVhugWzs1NbtxTT0FSEpkNG/FolK3LsP/tlYgI8XFyYiJXpm4mEcm7n3+G//wHmjaFZctg9ce2/bXuB4v+W3F5772HYTLzQuMu/HvCSsVSfvRqHuPsrERypJYZEcm7sWNtP2+9Ffavhp0LwOwGDXs6Ny/JH2FhfDRwNNNPuOPhZubth+ri4aavCym89OkUkbxZtQoWLQI3N+jfHxa+Zttf5xEIiXZqanID/vkHPv0UgHV7TzJmjm3V8xEdqlO9TIAzMxO5KrUHi0jejBhh+9mlC2Ttg38W2lplmj/j3Lzk+h07Bm3bwo4dnE0+Tf/UmmRaDeJrh/NIo0hnZydyVSpmROTarVwJc+bYli548UVYNMi2v24XCI5yampync6ds41/2rEDIzKSYabKHDh1jqgSPrx+by1dvSQuQd1MInLtRo60/XzsMbAcgl1LwOyuVhlXlZUFnTvbitSgIL7934d8dygLD4uZdx+ph7+Xu7MzFLkmapkRkWuTkgLHj9vGyrz4IizsZ9tf71EIinBubpJ3hmGb4ff778HDg2VvfMyzWzIBeCG+mlbEFpeiYkZEro2/P6xeDRs3AvtgzzKweEDzp52dmVyPoUNhwgQANo9+l247vTEMg8eaRPFoE3UZimtRMSMi185kgtq1YVI723b9bhBY1qkpyXWKtA3s3ffSaDodL0emNYsOsWUY0UGz/Irr0ZgZEbm6L76wdTMBbJ4Je1eCxRNuHezUtOQG9OvHngXL6UAdzmVkcVvlkrx5fyxmswoZcT0qZkQkd0uXQteuUKUKHNsPP58f7HvrIAgId2pqkgfJydC7N5w4AcA/R0/z4O/nOHU2gzoRQUzsUk8T44nLUjeTiOQsPd22Tg9Ax46weCScPQalqusKJleybRvcfz9s2gR79rD2g6/oOWUNJ89mULGUH5O6NcTHQ18H4rpUhotIzl54AdauheBgeKgJbPoGTGboOAHcPJydnVyLqVOhQQNbIVO6NCt6PMUjH/3BybMZxJYLZNp/byHYV79LcW0qZkTkyubMgTfesN1//x1Y/artftP+ULa+8/KSa5OaCn36wCOPwOnT0LIlUyf+QOf1maRlWmldrRRT/3sLoX6ezs5U5IapXVFEsjt82DYxHkDfvuC1Ak4fhhIVoWWCc3OTqztwAO66CxITwWQiM+F5/lf/fqb8vh+ALrdEMrJDDdws+ntWigYVMyKS3dChcOQI1KoFT7SHbx8BTNDxXXD3dnZ2cjUhIbarz0qWZO/bH9DrUAm2rbIVMkPbVqVPixhdfi1FiooZEclu3DjIzITe98OsJ2z7GveByFucm5fk7NdfoVUr27pZ3t5Yv5rK9INZjFhzkvTMFEr4ejDmvtrcUa20szMVyXdqYxSR7EJCYPwI+P1pSE2CiFvgjuHOzkquZN06iI+HNm3gvfcA2HfiLI9tspLw+3HSM63cXrUUcwbdpkJGiiy1zIiIzbRpsGULvPQSnNgFn/3Hdhl2eCx0/ho8fJydoVxqwwbbwp8zZ9q23dxIO36CcbO38umyXaRnWfFyN/NCfHW6NI5Ut5IUaSpmRAQ++sg2oZphQLVoODwWUg5ByarQ5Xvw0qKDhcaqVfB//wfffmvbNpmwPvwIv9zTk5FbMzi2eCcAt1YMZeR/alCxlJ8TkxW5OVTMiBR3Y8fC0+cXi+z2CPz7DpzaA8HR8OgP4FvCufmJozfftBUyJhNZ99/P/E7/5Y39bvy95jQAMaG+vBBfjdurllJrjBQbKmZEiqu0NHjxxYtzyXTrABUXwcnTEFAOHvsR/MOcmmKxZhi2CQu/+srWalalim1///6ku3vw/R2PMPaQO/+uSwUgwMuNga0r0/WWKC1LIMWOihmR4mjNGnj0UdsYGYBHGkLkIsgwQWRT6PQRBJZzaorF1l9/2cYvTZsGf/9t25eVhTF+PGv3nOS7I4H8VKELp7dlAmmUDvCkR7NoHm4cSYCXu1NTF3EWFTMixdG5c7ZCJjQEOvhD5DYwWaDFc9D8abDov4ab6vRpeP55mD37YgED4O3Nmbj2zA2P5a03FrHn+Fn7ocql/ejVPIaOdcqqJUaKPf2PJVIcpKXBihW2eUgAojygd30I3A7eJyEwAu79CKKaODfP4uDcOdsg3qNH4b77bPt8fGD6dDhyBMPdnVO33c6Seq35KLgGfyZZ4RTAWXw9LLSrFc699crSJKaExsSInKdiRqQoO34cJk6Ed96BY8dgyTfw9xT4ZyGEAWZ3qPMItHkZvIOdnW3Rk5lpW7F63Trb7Y8/bF18GRlQqhR06kSm1WDzoWROPv40f50x87lvRQ5Zz3cXJVmxmE00rVCCe+uVJa5GmFa3FrkC/asQKWrOnbPNBjtzpm3cxdnzXROB7vBBF4h2A7ObrYhp/jQEl3dmtkVDVhbs3m3rIoqLu7g/Ph7mzcsWnhpamn+q1OXNd3/j9yMZnEnPAmqAN2CFIB93WlUpxe1VS3Fb5ZIEemssjEhuVMyIFCV//AG3336xgAEIM0MTT6jhBh5eEPsgNH8GgqOcl6crW7IEVq6EXbvgn39stz17bK0wACdPkuTpy4GT5/AqVZ5yXj7si6rMX2EVWREYxfLwauwLLA0mE+y/eCVSw/IhNIwOoVF0CLHlgrCY1YUkcq1coph57733+L//+z8OHTpEjRo1GD9+PM2bN3d2WiI3X1aW7YvzQtfFyuVQqRw82gL+/Qt2LIP0sxBggipuUN0doj2h4h1Q8z6o0g68Apz9KpzHMODMGTh1CsLDbesYASxbZuv+OXHC1jX377+2hTYv/Pz7b4yQEFLSMjF99hX+n3yQ7aEz3DzYVyqC/sNm8JevbdkAn5JtSO0fj9Vsscd5uZupGx5AzTKB1CgTQGxEEFVK+2NW8SJy3Qp9MTN9+nQGDRrEe++9R7Nmzfjggw9o164dmzdvJjIy0tnpieSf1NSLX6aGAVWi4exxOHsCHh8E2/+G/UcgI8vxvDJmCJx+cfsJPygfaRvMG30bVL0LfEJu6kvJk6ws2wDltDRIT794PzXVdqtfH8znr9ZZtQq2b7ftP3fO1gJ1/qdx9iwZr44iw8ubjCwrlnFv4jnjO0hJwXTmDObTKZjPnMGUZXv/5s5bS1JwSVIzsqgz/hNqfzc5xxQfGvEdawLKkWk16HAkgJY1WrE/sDT7gsLYF1iavUFh/OsX4lC0hPh6EBUZRIWSfsSU9CUm1I+KpXyJDvVTq4tIPjMZhmE4O4ncNG7cmHr16vH+++/b91WrVo27776bUaNGXfX85ORkAgMDSUpKIiAg//4iTTp5jLNbN2E5eCjHmMzoaIygIADMx45i2bc/h0iDrIhIjFDbTKumY8ex7Nub4+NmlSuHUbKkLfbESSx7duccW6YsRulSto1Tp3DbtSvn2PBwjLDzk6SdPo3bjh32Y6bLPiaZpUtjlC17PvYs7tu35vDSDLJKlcRaLsK2fe4c7ps355xDaAmsUeUBA9LScN+0yf44tp+XPHRIMJkVKpxPKAP3tesuxhrWC1GYDIOs4CCyqla1HbNa8VixEqxW+/EL+7FayQoOIrN2zfNPZuA1dwGmzAzbl6A1C6xWTNYsjKxMjOBA0po3AiMLrJn4fvE9pnPnMGVkYMrMgPR0TGlpmNLTsYb6k/rQbZgyUzFlpeI3bDqmU2cxncvAdDYdU7r14muLcMPU45K1kMalQPL5F28BSpihpBnKWsiMDuZs3TqcC6zI6ZJ1yTzmhTXNEyMryzY/idWAzEwMqxWruzvHWrS2vVTDoMTiX/E4ctgWm5kFWZm27pLMTKxmN7Y+0pMsw8BqQMUZX+K/+29MmZnnb7bXaMrIxABmPfemLdZqcNunbxL55xosGemYszIxZ2Rgybx4e/7duWSZzGRmGfR893karZqf42finlGzOe3hTZbVYPD00dy1Zk6OsQ2f/IyjfrbCbcSvH9B97U9XjMswW4jr8S7/lLDNpdPxr4XcsXM1J739OeXlz3GfQI75Bp//GcS+wDDS3WzjVrzdLQT7uBPq70kpf09K+ntS0t+LUv6elAv2plywN2WCvDVQV+QG5eX7u1D/a0tPT2ft2rU899xzDvvvvPNOVqxYccVz0tLSSEtLs28nJycXSG6bfxxHk2ljYU5qzkGdfaDi+bd4XTr8lEvs/d62LgGAPzPgu3M5x97tBbEetvvbMmBaLrHxXtDgfOw/mfD52Zxj23hCU0/b/f2Z8EkusS09ocX52H+zYOKZnGObeUBrL9v9E1Z453TOsQ3dob237f5pK7yZS2wdd+h4PjbNgNdTco6t4Qb3nS8OrAb8L5fYym7w8CWFxCvJkJVDbLQFknwvbn+cDDn9mstZ8Km47uL2gUsKlAtMgJcJk4dtf5rhznH8KdkkE/e0TNs4i3QrJAOnrLAzjQOroEVQP/tD/DxpADWO/HPFFI74BvOffp/bt7/5YhS1Dly5uEzx8OZur8b27c+++Zaau9dfMTbLZGZC4yfs23U3bSZi+4YrxgLMW7+PNHfb5yfuTCaNLjmWZnEnzc3j/E93/jl0iiRv2y9gfUA5AsrXJd3iRqq7F6luHpxz9yTVzYNUN0/S3DzsjzO99p38XqEe6d4+ZHj7kuHjR6a3D5kBgZh8fCjlYSHS3YK3uwX3+l1Z69ENH083fNwtxHi5Eevljr+XG/5e7gR6uxPs606wjwde7hZEpHAp1MXMsWPHyMrKonRpx2XrS5cuzeHDh694zqhRo3jppZcKPDeTmwcZmHHL5R3MMCxkGLYvcQ+yrjnW3bDinlssbqQb3udjwcMt52ImEwtpF2I5R25/LGZhIfV8rJuRhqdbzsVMFmZ7rMVIx8s952LGiplz52PNRga5XZhhNZk5a4/Nwscj52LGajZz2rAVEiYjC3/PFFsxcMElc3BY3SwkGf727WDf8497IeSSn1meFo4ZJbBiwsBEmZDTtmLGBJjP/zx/ywjwYAcxZGHGiplaERswZRgOcYbZBCY4E+jLHFqThgepePJAle/xPpeKFTNWqwmrYcbIACPTxF7PsvSxjCHN5InJZOKDbYMpe+IQhslka5wymbCazBg+Jo4HlSKqhI8tPZOJI+GReFlMtlizGavZjGGy/UzxDSC2XCAmkwmzCQ5Uq4MpJBir2YLVYsGwuGE1m7Fa3Mjw9KJ1tVKYTSbMJhN723Rg3r91MNzcMCwWrG7uGG7uGO5uWN3ceOyWSCwWCxYz/Ft6ADNOdQYPDwx3D/Bwt93cPMDTg4ToSljc3TCbTVjbvccME5i9PDC5e2CxmHEzm3Ezm7BYTEwwm7CYTLhZzFieaIq7xYTFbMLdYotxt5hxs5hwM5t51M2Mh8Vsj9E8LCLFQ6HuZjp48CBly5ZlxYoVNGlycTKvV199lc8//5ytW7N3bVypZSYiIiLfu5lERESk4BSZbqbQ0FAsFku2VpgjR45ka625wNPTE09Pz5uRnoiIiBQChXpBDw8PD+rXr8/8+Y6DA+fPn0/Tpk2dlJWIiIgUJoW6ZQZg8ODBdO3alQYNGtCkSRM+/PBD9u7dS58+fZydmoiIiBQChb6YefDBBzl+/Dgvv/wyhw4dombNmvzyyy9ERWn2UhERESnkA4DzQ0HNMyMiIiIFJy/f34V6zIyIiIjI1aiYEREREZemYkZERERcmooZERERcWkqZkRERMSlqZgRERERl6ZiRkRERFyaihkRERFxaSpmRERExKUV+uUMbtSFCY6Tk5OdnImIiIhcqwvf29eyUEGRL2ZSUlIAiIiIcHImIiIiklcpKSkEBgbmGlPk12ayWq0cPHgQf39/TCZTvj52cnIyERER7Nu3r0iu+6TX5/qK+mvU63N9Rf016vVdP8MwSElJoUyZMpjNuY+KKfItM2azmXLlyhXocwQEBBTJD+kFen2ur6i/Rr0+11fUX6Ne3/W5WovMBRoALCIiIi5NxYyIiIi4NBUzN8DT05MRI0bg6enp7FQKhF6f6yvqr1Gvz/UV9deo13dzFPkBwCIiIlK0qWVGREREXJqKGREREXFpKmZERETEpamYEREREZemYiafpaWlUadOHUwmE4mJic5OJ9/85z//ITIyEi8vL8LDw+natSsHDx50dlr5Zvfu3Tz++ONER0fj7e1NhQoVGDFiBOnp6c5OLd+8+uqrNG3aFB8fH4KCgpydzg177733iI6OxsvLi/r167N06VJnp5SvlixZQocOHShTpgwmk4mZM2c6O6V8M2rUKBo2bIi/vz+lSpXi7rvvZtu2bc5OK1+9//771K5d2z6ZXJMmTZg9e7az0yowo0aNwmQyMWjQIKc8v4qZfPbss89SpkwZZ6eR71q1asXXX3/Ntm3b+O6779i5cyf33Xefs9PKN1u3bsVqtfLBBx/w119/MW7cOCZOnMjzzz/v7NTyTXp6Ovfffz9PPPGEs1O5YdOnT2fQoEG88MILrF+/nubNm9OuXTv27t3r7NTyzZkzZ4iNjWXChAnOTiXfLV68mCeffJLff/+d+fPnk5mZyZ133smZM2ecnVq+KVeuHK+//jpr1qxhzZo13H777XTs2JG//vrL2anlu9WrV/Phhx9Su3Zt5yVhSL755ZdfjKpVqxp//fWXARjr1693dkoF5ocffjBMJpORnp7u7FQKzJgxY4zo6Ghnp5HvJk2aZAQGBjo7jRvSqFEjo0+fPg77qlatajz33HNOyqhgAcb333/v7DQKzJEjRwzAWLx4sbNTKVDBwcHGxx9/7Ow08lVKSopRqVIlY/78+UaLFi2MgQMHOiUPtczkk3///ZdevXrx+eef4+Pj4+x0CtSJEyf48ssvadq0Ke7u7s5Op8AkJSUREhLi7DTkMunp6axdu5Y777zTYf+dd97JihUrnJSV3IikpCSAIvvvLSsri2nTpnHmzBmaNGni7HTy1ZNPPkl8fDytW7d2ah4qZvKBYRh069aNPn360KBBA2enU2CGDh2Kr68vJUqUYO/evfzwww/OTqnA7Ny5k3feeYc+ffo4OxW5zLFjx8jKyqJ06dIO+0uXLs3hw4edlJVcL8MwGDx4MLfeeis1a9Z0djr5atOmTfj5+eHp6UmfPn34/vvvqV69urPTyjfTpk1j3bp1jBo1ytmpqJjJzciRIzGZTLne1qxZwzvvvENycjIJCQnOTjlPrvX1XTBkyBDWr1/PvHnzsFgsPProoxiFfALpvL5GgIMHD9K2bVvuv/9+evbs6aTMr831vL6iwmQyOWwbhpFtnxR+/fr1Y+PGjUydOtXZqeS7KlWqkJiYyO+//84TTzzBY489xubNm52dVr7Yt28fAwcO5IsvvsDLy8vZ6Wg5g9wcO3aMY8eO5RpTvnx5HnroIX766SeH/0izsrKwWCx07tyZKVOmFHSq1+VaX9+VPqj79+8nIiKCFStWFOpm07y+xoMHD9KqVSsaN27M5MmTMZsLd71/Pb/DyZMnM2jQIE6dOlXA2RWM9PR0fHx8+Oabb7jnnnvs+wcOHEhiYiKLFy92YnYFw2Qy8f3333P33Xc7O5V81b9/f2bOnMmSJUuIjo52djoFrnXr1lSoUIEPPvjA2ancsJkzZ3LPPfdgsVjs+7KysjCZTJjNZtLS0hyOFTS3m/ZMLig0NJTQ0NCrxr399tu88sor9u2DBw8SFxfH9OnTady4cUGmeEOu9fVdyYUaOC0tLT9Tynd5eY0HDhygVatW1K9fn0mTJhX6QgZu7Hfoqjw8PKhfvz7z5893KGbmz59Px44dnZiZXCvDMOjfvz/ff/89ixYtKhaFDNhed2H/P/Na3XHHHWzatMlhX/fu3alatSpDhw69qYUMqJjJF5GRkQ7bfn5+AFSoUIFy5co5I6V8tWrVKlatWsWtt95KcHAw//zzD8OHD6dChQqFulUmLw4ePEjLli2JjIzkjTfe4OjRo/ZjYWFhTsws/+zdu5cTJ06wd+9esrKy7PMgVaxY0f6ZdRWDBw+ma9euNGjQgCZNmvDhhx+yd+/eIjXG6fTp0/z999/27V27dpGYmEhISEi2/3NczZNPPslXX33FDz/8gL+/v32sU2BgIN7e3k7OLn88//zztGvXjoiICFJSUpg2bRqLFi1izpw5zk4tX/j7+2cb43RhTKVTxj455RqqIm7Xrl1F6tLsjRs3Gq1atTJCQkIMT09Po3z58kafPn2M/fv3Ozu1fDNp0iQDuOKtqHjssceu+PoWLlzo7NSuy7vvvmtERUUZHh4eRr169YrcZb0LFy684u/rsccec3ZqNyynf2uTJk1ydmr5pkePHvbPZ8mSJY077rjDmDdvnrPTKlDOvDRbY2ZERETEpRX+QQEiIiIiuVAxIyIiIi5NxYyIiIi4NBUzIiIi4tJUzIiIiIhLUzEjIiIiLk3FjIiIiLg0FTMiIiLi0lTMiBRTLVu2ZNCgQc5O44qOHz9OqVKl2L17NwCLFi3CZDIV+OKY1/s8kydPJigoKE/nNGzYkBkzZuTpHBG5MhUzIpIvDh06xCOPPEKVKlUwm805Fkrfffcd1atXx9PTk+rVq/P9999nixk1ahQdOnSgfPnyBZu0Ew0bNoznnnsOq9Xq7FREXJ6KGRHJF2lpaZQsWZIXXniB2NjYK8asXLmSBx98kK5du7Jhwwa6du3KAw88wB9//GGPOXfuHJ988gk9e/a8Wak7RXx8PElJScydO9fZqYi4PBUzIsLJkyd59NFHCQ4OxsfHh3bt2rFjxw6HmI8++oiIiAh8fHy45557GDt2rEPXSvny5Xnrrbd49NFHCQwMvOLzjB8/njZt2pCQkEDVqlVJSEjgjjvuYPz48faY2bNn4+bmluuK7MePH+fhhx+mXLly+Pj4UKtWLaZOneoQ07JlS/r378+gQYMIDg6mdOnSfPjhh5w5c4bu3bvj7+9PhQoVmD17drbHX758ObGxsXh5edG4cWM2bdrkcHzy5MlERkba34vjx487HN+5cycdO3akdOnS+Pn50bBhQ3799VeHGIvFQvv27bPlLSJ5p2JGROjWrRtr1qzhxx9/ZOXKlRiGQfv27cnIyABsX+59+vRh4MCBJCYm0qZNG1599dU8P8/KlSu58847HfbFxcWxYsUK+/aSJUto0KBBro+TmppK/fr1mTVrFn/++Sf//e9/6dq1q0MLD8CUKVMIDQ1l1apV9O/fnyeeeIL777+fpk2bsm7dOuLi4ujatStnz551OG/IkCG88cYbrF69mlKlSvGf//zH/l788ccf9OjRg759+5KYmEirVq145ZVXHM4/ffo07du359dff2X9+vXExcXRoUMH9u7d6xDXqFEjli5dem1vnojkzClrdYuI07Vo0cIYOHCgsX37dgMwli9fbj927Ngxw9vb2/j6668NwzCMBx980IiPj3c4v3PnzkZgYGCuj305d3d348svv3TY9+WXXxoeHh727Y4dOxo9evRwiFm4cKEBGCdPnszx9bRv3954+umnHXK49dZb7duZmZmGr6+v0bVrV/u+Q4cOGYCxcuVKh+eZNm2aPeb48eOGt7e3MX36dMMwDOPhhx822rZt6/DcDz74YI7vxQXVq1c33nnnHYd9P/zwg2E2m42srKxczxWR3KllRqSY27JlC25ubjRu3Ni+r0SJElSpUoUtW7YAsG3bNho1auRw3uXb18pkMjlsG4bhsO/cuXN4eXnl+hhZWVm8+uqr1K5dmxIlSuDn58e8efOytXzUrl3bft9isVCiRAlq1apl31e6dGkAjhw54nDepV1cISEhDu/Fli1bsnWBXb595swZnn32WapXr05QUBB+fn5s3bo1W37e3t5YrVbS0tJyfb0ikjs3ZycgIs5lGEaO+y8UGZcXHLmdl5uwsDAOHz7ssO/IkSP2ogIgNDSUkydP5vo4b775JuPGjWP8+PHUqlULX19fBg0aRHp6ukOcu7u7w7bJZHLYd+E1XcsVRZe+F1czZMgQ5s6dyxtvvEHFihXx9vbmvvvuy5bfiRMn8PHxwdvb+6qPKSI5U8uMSDFXvXp1MjMzHcabHD9+nO3bt1OtWjUAqlatyqpVqxzOW7NmTZ6fq0mTJsyfP99h37x582jatKl9u27dumzevDnXx1m6dCkdO3akS5cuxMbGEhMTk23A8o34/fff7fdPnjzJ9u3bqVq1KmB7vy49fnn8hfy6devGPffcQ61atQgLC7PPmXOpP//8k3r16uVb3iLFlYoZkWKuUqVKdOzYkV69erFs2TI2bNhAly5dKFu2LB07dgSgf//+/PLLL4wdO5YdO3bwwQcfMHv27GytNYmJiSQmJnL69GmOHj1KYmKiQ2EycOBA5s2bx+jRo9m6dSujR4/m119/dZiTJi4ujr/++ivX1pmKFSsyf/58VqxYwZYtW+jdu3e2Fp8b8fLLL7NgwQL+/PNPunXrRmhoKHfffTcAAwYMYM6cOYwZM4bt27czYcIE5syZky2/GTNmkJiYyIYNG3jkkUeu2PqzdOnSbAOiRSTvVMyICJMmTaJ+/frcddddNGnSBMMw+OWXX+xdMs2aNWPixImMHTuW2NhY5syZw1NPPZVtbEvdunWpW7cua9eu5auvvqJu3bq0b9/efrxp06ZMmzaNSZMmUbt2bSZPnsz06dMdxuvUqlWLBg0a8PXXX+eY77Bhw6hXrx5xcXG0bNmSsLAwe7GRH15//XUGDhxI/fr1OXToED/++CMeHh4A3HLLLXz88ce888471KlTh3nz5vHiiy86nD9u3DiCg4Np2rQpHTp0IC4uLlsLzIEDB1ixYgXdu3fPt7xFiiuTcT0d3yJS7PXq1YutW7cWyKXFv/zyC8888wx//vknZnPR/JtryJAhJCUl8eGHHzo7FRGXpwHAInJN3njjDdq0aYOvry+zZ89mypQpvPfeewXyXO3bt2fHjh0cOHCAiIiIAnkOZytVqhTPPPOMs9MQKRLUMiMi1+SBBx5g0aJFpKSkEBMTQ//+/enTp4+z0xIRUTEjIiIirq1odkaLiIhIsaFiRkRERFyaihkRERFxaSpmRERExKWpmBERERGXpmJGREREXJqKGREREXFpKmZERETEpf0/3yXYqaiGFt8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import os\n", "import numpy as np\n", @@ -2040,9 +2030,7 @@ { "cell_type": "markdown", "id": "59596e3e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Material for lecture Thursday September 7" ] @@ -2050,9 +2038,7 @@ { "cell_type": "markdown", "id": "57fba84b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linking the regression analysis with a statistical interpretation\n", "\n", @@ -2079,9 +2065,7 @@ { "cell_type": "markdown", "id": "0b79f217", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*} \n", @@ -2095,9 +2079,7 @@ { "cell_type": "markdown", "id": "bfadc49d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The randomness of $\\varepsilon_i$ implies that\n", "$\\mathbf{y}_i$ is also a random variable. In particular,\n", @@ -2114,9 +2096,7 @@ { "cell_type": "markdown", "id": "c8c5f36d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Assumptions made\n", "\n", @@ -2128,9 +2108,7 @@ { "cell_type": "markdown", "id": "0d801f58", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", @@ -2140,9 +2118,7 @@ { "cell_type": "markdown", "id": "b29498f7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We approximate this function with our model from the solution of the linear regression equations, that is our\n", "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" @@ -2151,9 +2127,7 @@ { "cell_type": "markdown", "id": "10ac35c5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", @@ -2163,9 +2137,7 @@ { "cell_type": "markdown", "id": "f80ec09c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Expectation value and variance\n", "\n", @@ -2175,9 +2147,7 @@ { "cell_type": "markdown", "id": "458211cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*} \n", @@ -2191,9 +2161,7 @@ { "cell_type": "markdown", "id": "6a995ee9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "while\n", "its variance is" @@ -2202,9 +2170,7 @@ { "cell_type": "markdown", "id": "7e11465e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", @@ -2225,9 +2191,7 @@ { "cell_type": "markdown", "id": "c3c1ff49", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD)." @@ -2236,9 +2200,7 @@ { "cell_type": "markdown", "id": "ddcc761e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", "\n", @@ -2248,9 +2210,7 @@ { "cell_type": "markdown", "id": "101539f6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathbb{E}(\\boldsymbol{\\hat{\\beta}}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", @@ -2260,9 +2220,7 @@ { "cell_type": "markdown", "id": "a0540e76", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This means that the estimator of the regression parameters is unbiased.\n", "\n", @@ -2274,9 +2232,7 @@ { "cell_type": "markdown", "id": "e036abed", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{eqnarray*}\n", @@ -2305,9 +2261,7 @@ { "cell_type": "markdown", "id": "1a602124", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", @@ -2327,9 +2281,7 @@ { "cell_type": "markdown", "id": "f32395e8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", @@ -2339,9 +2291,7 @@ { "cell_type": "markdown", "id": "196b5a71", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see clearly that \n", "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", @@ -2352,9 +2302,7 @@ { "cell_type": "markdown", "id": "8b444609", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", @@ -2364,9 +2312,7 @@ { "cell_type": "markdown", "id": "e2aa4dea", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", "\n", @@ -2376,9 +2322,7 @@ { "cell_type": "markdown", "id": "d681a7f9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", @@ -2388,9 +2332,7 @@ { "cell_type": "markdown", "id": "cf4c0fb3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The difference is non-negative definite since each component of the\n", "matrix product is non-negative definite. \n", @@ -2400,9 +2342,7 @@ { "cell_type": "markdown", "id": "708f7d10", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving OLS from a probability distribution\n", "\n", @@ -2423,9 +2363,7 @@ { "cell_type": "markdown", "id": "6dab5a6e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", @@ -2435,9 +2373,7 @@ { "cell_type": "markdown", "id": "958c835b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Independent and Identically Distrubuted (iid)\n", "\n", @@ -2448,9 +2384,7 @@ { "cell_type": "markdown", "id": "58b97d31", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(y_i, \\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", @@ -2460,9 +2394,7 @@ { "cell_type": "markdown", "id": "f319e10b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which reads as finding the likelihood of an event $y_i$ with the input variables $\\boldsymbol{X}$ given the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", "\n", @@ -2472,9 +2404,7 @@ { "cell_type": "markdown", "id": "8ed63d03", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{y},\\boldsymbol{X}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta}).\n", @@ -2484,9 +2414,7 @@ { "cell_type": "markdown", "id": "c3e14c22", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We will write this in a more compact form reserving $\\boldsymbol{D}$ for the domain of events, including the ouputs (targets) and the inputs. That is\n", "in case we have a simple one-dimensional input and output case" @@ -2495,9 +2423,7 @@ { "cell_type": "markdown", "id": "64fc5cd4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})].\n", @@ -2507,9 +2433,7 @@ { "cell_type": "markdown", "id": "b2256460", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the more general case the various inputs should be replaced by the possible features represented by the input data set $\\boldsymbol{X}$. \n", "We can now rewrite the above probability as" @@ -2518,9 +2442,7 @@ { "cell_type": "markdown", "id": "87c5772f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", @@ -2530,9 +2452,7 @@ { "cell_type": "markdown", "id": "ad4425cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It is a conditional probability (see below) and reads as the likelihood of a domain of events $\\boldsymbol{D}$ given a set of parameters $\\boldsymbol{\\beta}$." ] @@ -2540,9 +2460,7 @@ { "cell_type": "markdown", "id": "33d200c8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Maximum Likelihood Estimation (MLE)\n", "\n", @@ -2571,9 +2489,7 @@ { "cell_type": "markdown", "id": "0ccf81a4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A new Cost Function\n", "\n", @@ -2583,9 +2499,7 @@ { "cell_type": "markdown", "id": "65442e18", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i,\\boldsymbol{X}\\vert\\boldsymbol{\\beta})},\n", @@ -2595,9 +2509,7 @@ { "cell_type": "markdown", "id": "746181e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which becomes" ] @@ -2605,9 +2517,7 @@ { "cell_type": "markdown", "id": "8db48c8a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", @@ -2617,9 +2527,7 @@ { "cell_type": "markdown", "id": "444ed592", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" ] @@ -2627,9 +2535,7 @@ { "cell_type": "markdown", "id": "cb663e4b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", @@ -2639,9 +2545,7 @@ { "cell_type": "markdown", "id": "c650bff3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to the well-known OLS equation for the optimal paramters $\\beta$" ] @@ -2649,9 +2553,7 @@ { "cell_type": "markdown", "id": "5eca9ec1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", @@ -2661,9 +2563,7 @@ { "cell_type": "markdown", "id": "faa327ae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics." ] @@ -2671,9 +2571,7 @@ { "cell_type": "markdown", "id": "31fc1a2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More basic Statistics and Bayes' theorem\n", "\n", @@ -2691,9 +2589,7 @@ { "cell_type": "markdown", "id": "3619de6c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", @@ -2703,9 +2599,7 @@ { "cell_type": "markdown", "id": "52da2f98", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "**The product rule (aka joint probability) is given by.**" ] @@ -2713,9 +2607,7 @@ { "cell_type": "markdown", "id": "890c2fed", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(X),\n", @@ -2725,9 +2617,7 @@ { "cell_type": "markdown", "id": "c2db0804", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", "\n", @@ -2737,9 +2627,7 @@ { "cell_type": "markdown", "id": "65615dc3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Marginal Probability\n", "\n", @@ -2749,9 +2637,7 @@ { "cell_type": "markdown", "id": "2c21bbaa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", @@ -2761,9 +2647,7 @@ { "cell_type": "markdown", "id": "1e96b67f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Conditional Probability\n", "\n", @@ -2773,9 +2657,7 @@ { "cell_type": "markdown", "id": "70c1bd4d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", @@ -2785,9 +2667,7 @@ { "cell_type": "markdown", "id": "a5a71be1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Bayes' Theorem\n", "\n", @@ -2797,9 +2677,7 @@ { "cell_type": "markdown", "id": "3d30a931", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", @@ -2809,9 +2687,7 @@ { "cell_type": "markdown", "id": "4a9afe64", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which we can rewrite as" ] @@ -2819,9 +2695,7 @@ { "cell_type": "markdown", "id": "76da1daa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", @@ -2831,9 +2705,7 @@ { "cell_type": "markdown", "id": "e3134714", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is Bayes' theorem. It allows us to evaluate the uncertainty in in $X$ after we have observed $Y$. We can easily interchange $X$ with $Y$." ] @@ -2841,9 +2713,7 @@ { "cell_type": "markdown", "id": "ff0ea1cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations of Bayes' Theorem\n", "\n", @@ -2860,9 +2730,7 @@ { "cell_type": "markdown", "id": "229aa3b3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example of Usage of Bayes' theorem\n", "\n", @@ -2881,9 +2749,7 @@ { "cell_type": "markdown", "id": "c5ac1aae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X=1\\vert Y=1) =0.8.\n", @@ -2893,9 +2759,7 @@ { "cell_type": "markdown", "id": "954824fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of $80\\%$ for having cancer.\n", "It is however not correct, as the following Bayesian analysis shows." @@ -2904,9 +2768,7 @@ { "cell_type": "markdown", "id": "7633d6dd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Doing it correctly\n", "\n", @@ -2917,9 +2779,7 @@ { "cell_type": "markdown", "id": "526a8146", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(Y=1) =0.004.\n", @@ -2929,9 +2789,7 @@ { "cell_type": "markdown", "id": "fb5827c4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have" ] @@ -2939,9 +2797,7 @@ { "cell_type": "markdown", "id": "ec6100c3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(X=1\\vert Y=0) =0.1.\n", @@ -2951,9 +2807,7 @@ { "cell_type": "markdown", "id": "5f8cb0aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute" ] @@ -2961,9 +2815,7 @@ { "cell_type": "markdown", "id": "90f52404", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(Y=1\\vert X=1)=\\frac{p(X=1\\vert Y=1)p(Y=1)}{p(X=1\\vert Y=1)p(Y=1)+p(X=1\\vert Y=0)p(Y=0)}=\\frac{0.8\\times 0.004}{0.8\\times 0.004+0.1\\times 0.996}=0.031.\n", @@ -2973,9 +2825,7 @@ { "cell_type": "markdown", "id": "16e14546", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "That is, in case of a positive test, there is only a $3\\%$ chance of having breast cancer!" ] @@ -2983,9 +2833,7 @@ { "cell_type": "markdown", "id": "ae7afa6d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Bayes' Theorem and Ridge and Lasso Regression\n", "\n", @@ -3000,9 +2848,7 @@ { "cell_type": "markdown", "id": "7b6a425c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Test Function for what happens with OLS, Ridge and Lasso\n", "\n", @@ -3019,10 +2865,7 @@ "cell_type": "code", "execution_count": 7, "id": "a6e3c0ad", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -3093,9 +2936,7 @@ { "cell_type": "markdown", "id": "7d8148fc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "How can we understand this?" ] @@ -3103,9 +2944,7 @@ { "cell_type": "markdown", "id": "614e8634", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Invoking Bayes' theorem\n", "\n", @@ -3117,9 +2956,7 @@ { "cell_type": "markdown", "id": "37fa6822", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\\dots, (x_{n-1},y_{n-1})],\n", @@ -3129,9 +2966,7 @@ { "cell_type": "markdown", "id": "37f0743c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is given by" ] @@ -3139,9 +2974,7 @@ { "cell_type": "markdown", "id": "238ed35a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", @@ -3151,9 +2984,7 @@ { "cell_type": "markdown", "id": "219f1556", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In Bayes' theorem this function plays the role of the so-called likelihood. We could now ask the question what is the posterior probability of a parameter set $\\boldsymbol{\\beta}$ given a domain of events $\\boldsymbol{D}$? That is, how can we define the posterior probability" ] @@ -3161,9 +2992,7 @@ { "cell_type": "markdown", "id": "0706cf9e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D}).\n", @@ -3173,9 +3002,7 @@ { "cell_type": "markdown", "id": "4caa60d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Bayes' theorem comes to our rescue here since (omitting the normalization constant)" ] @@ -3183,9 +3010,7 @@ { "cell_type": "markdown", "id": "eac35960", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D})\\propto p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})p(\\boldsymbol{\\beta}).\n", @@ -3195,9 +3020,7 @@ { "cell_type": "markdown", "id": "6a67663d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We have a model for $p(\\boldsymbol{D}\\vert\\boldsymbol{\\beta})$ but need one for the **prior** $p(\\boldsymbol{\\beta}$!" ] @@ -3205,9 +3028,7 @@ { "cell_type": "markdown", "id": "0d7f08cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ridge and Bayes\n", "\n", @@ -3221,9 +3042,7 @@ { "cell_type": "markdown", "id": "1dc770da", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta})=\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\beta_j^2}{2\\tau^2}\\right)}.\n", @@ -3233,9 +3052,7 @@ { "cell_type": "markdown", "id": "ac7359f8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Our posterior probability becomes then (omitting the normalization factor which is just a constant)" ] @@ -3243,9 +3060,7 @@ { "cell_type": "markdown", "id": "e524e9cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta\\vert\\boldsymbol{D})}=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\beta_j^2}{2\\tau^2}\\right)}.\n", @@ -3255,9 +3070,7 @@ { "cell_type": "markdown", "id": "165565d5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can now optimize this quantity with respect to $\\boldsymbol{\\beta}$. As we\n", "did for OLS, this is most conveniently done by taking the negative\n", @@ -3268,9 +3081,7 @@ { "cell_type": "markdown", "id": "4e4ff218", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\frac{1}{2\\tau^2}\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_2^2,\n", @@ -3280,9 +3091,7 @@ { "cell_type": "markdown", "id": "00b53e65", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and replacing $1/2\\tau^2$ with $\\lambda$ we have" ] @@ -3290,9 +3099,7 @@ { "cell_type": "markdown", "id": "3d4d1abe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta})=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_2^2,\n", @@ -3302,9 +3109,7 @@ { "cell_type": "markdown", "id": "fad0d46d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is our Ridge cost function! Nice, isn't it?" ] @@ -3312,9 +3117,7 @@ { "cell_type": "markdown", "id": "974746aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Lasso and Bayes\n", "\n", @@ -3324,9 +3127,7 @@ { "cell_type": "markdown", "id": "0b0fece4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta})=\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\vert\\beta_j\\vert}{\\tau}\\right)}.\n", @@ -3336,9 +3137,7 @@ { "cell_type": "markdown", "id": "0f05f622", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Our posterior probability becomes then (omitting the normalization factor which is just a constant)" ] @@ -3346,9 +3145,7 @@ { "cell_type": "markdown", "id": "ed87949b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p(\\boldsymbol{\\beta}\\vert\\boldsymbol{D})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}\\prod_{j=0}^{p-1}\\exp{\\left(-\\frac{\\vert\\beta_j\\vert}{\\tau}\\right)}.\n", @@ -3358,9 +3155,7 @@ { "cell_type": "markdown", "id": "678ca6bc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Taking the negative\n", "logarithm of the posterior probability and leaving out the\n", @@ -3370,9 +3165,7 @@ { "cell_type": "markdown", "id": "642ac33f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta}=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\frac{1}{\\tau}\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -3382,9 +3175,7 @@ { "cell_type": "markdown", "id": "9e80553a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and replacing $1/\\tau$ with $\\lambda$ we have" ] @@ -3392,9 +3183,7 @@ { "cell_type": "markdown", "id": "25707056", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\beta}=\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n", @@ -3404,15 +3193,31 @@ { "cell_type": "markdown", "id": "95fed8fb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is our Lasso cost function!" ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/week36/week36.do.txt b/doc/src/week36/week36.do.txt index d8e1ec749..539f8e591 100644 --- a/doc/src/week36/week36.do.txt +++ b/doc/src/week36/week36.do.txt @@ -6,19 +6,18 @@ DATE: September 4-8, 2023 !split ===== Plans for week 36 ===== -o Material for the active learning sessions on Tuesday and Wednesday - o Summary from last week on discussion of SVD, Ridge and Lasso linear regression. - o Recommended Reading: Hastie et al chapter 3, see URL:"https://link.springer.com/book/10.1007/978-0-387-84858-7" - o Presentation and discussion of first project -o Material for the lecture on Thursday September 7 - o Linear Regression and links with Statistics, Resampling methods - o Recommended Reading: Hastie et al chapter 3, see URL:"https://link.springer.com/book/10.1007/978-0-387-84858-7" +* Material for the active learning sessions on Tuesday and Wednesday + * Summary from last week on discussion of SVD, Ridge and Lasso linear regression. + * Recommended Reading: Hastie et al chapter 3, see URL:"https://link.springer.com/book/10.1007/978-0-387-84858-7" + * Presentation and discussion of first project +* Material for the lecture on Thursday September 7 + * Linear Regression and links with Statistics, Resampling methods + * Recommended Reading: Hastie et al chapter 3, see URL:"https://link.springer.com/book/10.1007/978-0-387-84858-7" !split ===== Material for the active learning sessions Tuesday and Wednesday ===== -!split -===== Summary from last Week and discussion of SVD, Ridge and Lasso regression with examples ===== +The material here contains a summary from last Week and discussion of SVD, Ridge and Lasso regression with examples !split ===== Linear Regression and the SVD ===== @@ -233,7 +232,7 @@ C = SVDinv(A) print(np.abs(C-B)) !ec -As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by _Numpy_. +As you can see from this example, our own decomposition based on the SVD agrees with the pseudoinverse algorithm provided by _Numpy_. @@ -278,7 +277,9 @@ defining a new cost function to be optimized, that is which leads to the Ridge regression minimization problem where we require that $\vert\vert \bm{\beta}\vert\vert_2^2\le t$, where $t$ is -a finite number larger than zero. By defining +a finite number larger than zero. We do not include such a constraints in the discussions here. + +By defining !bt \[ @@ -448,34 +449,34 @@ Similarly, "Mehta et al's article":"https://arxiv.org/abs/1803.08823" is also re !split ===== Deriving the Lasso Regression Equations ===== -Using the matrix-vector expression for Lasso regression and dropping the parameter $1/n$ in front of the standard mean squared error equation, we have the following _cost_ function +Using the matrix-vector expression for Lasso regression, we have the following _cost_ function !bt \[ -C(\bm{X},\bm{\beta})=\left\{(\bm{y}-\bm{X}\bm{\beta})^T(\bm{y}-\bm{X}\bm{\beta})\right\}+\lambda\vert\vert\bm{\beta}\vert\vert_1, +C(\bm{X},\bm{\beta})=\frac{1}{n}\left\{(\bm{y}-\bm{X}\bm{\beta})^T(\bm{y}-\bm{X}\bm{\beta})\right\}+\lambda\vert\vert\bm{\beta}\vert\vert_1, \] !et -Taking the derivative with respect to $\bm{\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty) +Taking the derivative with respect to $\bm{\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicity) !bt \[ -\frac{d \vert \beta\vert}{d \bm{\beta}}=\mathrm{sgn}(\bm{\beta})=\left\{\begin{array}{cc} 1 & \beta > 0 \\-1 & \beta < 0, \end{array}\right. +\frac{d \vert \beta\vert}{d \beta}=\mathrm{sgn}(\beta)=\left\{\begin{array}{cc} 1 & \beta > 0 \\-1 & \beta < 0, \end{array}\right. \] !et we have that the derivative of the cost function is !bt \[ -\frac{\partial C(\bm{X},\bm{\beta})}{\partial \bm{\beta}}=-2\bm{X}^T(\bm{y}-\bm{X}\bm{\beta})+\lambda sgn(\bm{\beta})=0, +\frac{\partial C(\bm{X},\bm{\beta})}{\partial \bm{\beta}}=-\frac{2}{n}\bm{X}^T(\bm{y}-\bm{X}\bm{\beta})+\lambda sgn(\bm{\beta})=0, \] !et and reordering we have !bt \[ -\bm{X}^T\bm{X}\bm{\beta}+\lambda sgn(\bm{\beta})=2\bm{X}^T\bm{y}. +\bm{X}^T\bm{X}\bm{\beta}+\lambda sgn(\bm{\beta})=\bm{X}^T\bm{y}. \] !et -This equation does not lead to a nice analytical equation as in Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package "CVXOPT":"https://cvxopt.org/". We will discuss this later. +This equation does not lead to a nice analytical equation as in Ridge regression or ordinary least squares. We have absorbed the factor $2/n$ in a redefinition of the parameter $\lambda$. We will solve this type of problems using libraries like _scikit-learn_. @@ -812,7 +813,7 @@ for i in range(nlambdas): # and then make the prediction ypredictRidge = X @ Ridgebeta MSERidgePredict[i] = MSE(y,ypredictRidge) - RegLasso = linear_model.Lasso(lmb) + RegLasso = linear_model.Lasso(lmb,fit_intercept=False) RegLasso.fit(X,y) ypredictLasso = RegLasso.predict(X) print(RegLasso.coef_)