From da97fdfc3fb6ab3dbd391882726dd42ed0a85ec1 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Mon, 1 Sep 2025 11:15:58 +0200 Subject: [PATCH] update of p1 --- doc/pub/week35/ipynb/week35.ipynb | 2177 +++++++---------- .../Projects/2025/Project1/Project1.do.txt | 325 +-- 2 files changed, 988 insertions(+), 1514 deletions(-) diff --git a/doc/pub/week35/ipynb/week35.ipynb b/doc/pub/week35/ipynb/week35.ipynb index 0300d90b1..771c62f49 100644 --- a/doc/pub/week35/ipynb/week35.ipynb +++ b/doc/pub/week35/ipynb/week35.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "e5c865ad", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", @@ -15,9 +13,7 @@ { "cell_type": "markdown", "id": "725e6917", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "# Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo\n", @@ -28,9 +24,7 @@ { "cell_type": "markdown", "id": "7aab4a45", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Plans for week 35\n", "\n", @@ -50,9 +44,7 @@ { "cell_type": "markdown", "id": "dd9d9be5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "### Reading recommendations:\n", "\n", @@ -72,9 +64,7 @@ { "cell_type": "markdown", "id": "68a8aa98", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Reminder from last week\n", "\n", @@ -105,9 +95,7 @@ { "cell_type": "markdown", "id": "b2b56f10", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The equations for ordinary least squares\n", "\n", @@ -121,9 +109,7 @@ { "cell_type": "markdown", "id": "d0ffdd72", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_i=f(x_i)+\\epsilon_i,\n", @@ -133,9 +119,7 @@ { "cell_type": "markdown", "id": "f230ed73", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or in general" ] @@ -143,9 +127,7 @@ { "cell_type": "markdown", "id": "47c6c18d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y}=f(\\boldsymbol{x})+\\boldsymbol{\\epsilon},\n", @@ -155,9 +137,7 @@ { "cell_type": "markdown", "id": "77221535", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{\\epsilon}$ represents some noise which is normally assumed to\n", "be distributed via a normal probability distribution with zero mean\n", @@ -176,9 +156,7 @@ { "cell_type": "markdown", "id": "a6b3ef86", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\theta},\n", @@ -188,9 +166,7 @@ { "cell_type": "markdown", "id": "ab12e0e1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and in order to find the optimal parameters $\\theta_i$ we defined a function which\n", "gives a measure of the spread between the values $y_i$ (which\n", @@ -201,9 +177,7 @@ { "cell_type": "markdown", "id": "8806260c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The cost/loss function\n", "\n", @@ -213,9 +187,7 @@ { "cell_type": "markdown", "id": "c28506f9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", @@ -225,9 +197,7 @@ { "cell_type": "markdown", "id": "6ee2c6f3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or using the matrix $\\boldsymbol{X}$ and in a more compact matrix-vector notation as" ] @@ -235,9 +205,7 @@ { "cell_type": "markdown", "id": "9d693259", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right\\}.\n", @@ -247,9 +215,7 @@ { "cell_type": "markdown", "id": "4c8e76fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This function represents one of many possible ways to define the so-called cost function.\n", "\n", @@ -260,9 +226,7 @@ { "cell_type": "markdown", "id": "6ef26cfb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{2n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2,\n", @@ -272,9 +236,7 @@ { "cell_type": "markdown", "id": "a7d47918", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "since when taking the first derivative with respect to the unknown parameters $\\theta$, the factor of $2$ cancels out." ] @@ -282,9 +244,7 @@ { "cell_type": "markdown", "id": "3f18a933", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "\n", @@ -294,9 +254,7 @@ { "cell_type": "markdown", "id": "acfc82c7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right\\},\n", @@ -306,9 +264,7 @@ { "cell_type": "markdown", "id": "6cfbfe7b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "can be linked to the variance of the quantity $y_i$ if we interpret the latter as the mean value. \n", "When linking (see the discussions next week) with the maximum likelihood approach below, we will indeed interpret $y_i$ as a mean value" @@ -317,9 +273,7 @@ { "cell_type": "markdown", "id": "5c04831b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_{i}=\\langle y_i \\rangle = \\theta_0x_{i,0}+\\theta_1x_{i,1}+\\theta_2x_{i,2}+\\dots+\\theta_{n-1}x_{i,n-1}+\\epsilon_i,\n", @@ -329,9 +283,7 @@ { "cell_type": "markdown", "id": "63341f8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\langle y_i \\rangle$ is the mean value. Keep in mind also that\n", "till now we have treated $y_i$ as the exact value. Normally, the\n", @@ -348,9 +300,7 @@ { "cell_type": "markdown", "id": "68d06111", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}}={\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -361,9 +311,7 @@ { "cell_type": "markdown", "id": "08d49b08", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In practical terms it means we will require" ] @@ -371,9 +319,7 @@ { "cell_type": "markdown", "id": "5799e099", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\theta_j} = \\frac{\\partial }{\\partial \\theta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\theta_0x_{i,0}-\\theta_1x_{i,1}-\\theta_2x_{i,2}-\\dots-\\theta_{n-1}x_{i,n-1}\\right)^2\\right]=0,\n", @@ -383,9 +329,7 @@ { "cell_type": "markdown", "id": "cdbe32c8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which results in" ] @@ -393,9 +337,7 @@ { "cell_type": "markdown", "id": "a34a60de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\theta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_{ij}\\left(y_i-\\theta_0x_{i,0}-\\theta_1x_{i,1}-\\theta_2x_{i,2}-\\dots-\\theta_{n-1}x_{i,n-1}\\right)\\right]=0,\n", @@ -405,9 +347,7 @@ { "cell_type": "markdown", "id": "4380cb3d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or in a matrix-vector form as (multiplying away the factor $-2/n$, see derivation below)" ] @@ -415,9 +355,7 @@ { "cell_type": "markdown", "id": "0fa26221", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}^T} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right).\n", @@ -427,9 +365,7 @@ { "cell_type": "markdown", "id": "d5cec27d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "We can rewrite, see the derivations below," @@ -438,9 +374,7 @@ { "cell_type": "markdown", "id": "5a8454a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}^T} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right),\n", @@ -450,9 +384,7 @@ { "cell_type": "markdown", "id": "3f2c53c6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "as" ] @@ -460,9 +392,7 @@ { "cell_type": "markdown", "id": "1a38e0c7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{y} = \\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\theta},\n", @@ -472,9 +402,7 @@ { "cell_type": "markdown", "id": "f9d67e56", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and if the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ is invertible we have the solution" ] @@ -482,9 +410,7 @@ { "cell_type": "markdown", "id": "9605ae4f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}} =\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -494,9 +420,7 @@ { "cell_type": "markdown", "id": "de14b580", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We note also that since our design matrix is defined as $\\boldsymbol{X}\\in\n", "{\\mathbb{R}}^{n\\times p}$, the product $\\boldsymbol{X}^T\\boldsymbol{X} \\in\n", @@ -514,9 +438,7 @@ { "cell_type": "markdown", "id": "4054fa5c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Some useful matrix and vector expressions\n", "\n", @@ -541,9 +463,7 @@ { "cell_type": "markdown", "id": "61bd0198", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{y}=f(\\boldsymbol{x}).\n", @@ -553,9 +473,7 @@ { "cell_type": "markdown", "id": "bdb7e922", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Jacobian\n", "\n", @@ -565,9 +483,7 @@ { "cell_type": "markdown", "id": "54b6bbfa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{J}=\\frac{\\partial \\boldsymbol{y}}{\\partial \\boldsymbol{x}}=\\begin{bmatrix} \\frac{\\partial y_0}{\\partial x_0} & \\frac{\\partial y_0}{\\partial x_1} & \\frac{\\partial y_0}{\\partial x_2} & \\dots & \\dots & \\frac{\\partial y_0}{\\partial x_{n-1}} \\\\ \\frac{\\partial y_1}{\\partial x_0} & \\frac{\\partial y_1}{\\partial x_1} & \\frac{\\partial y_1}{\\partial x_2} & \\dots & \\dots & \\frac{\\partial y_1}{\\partial x_{n-1}} \\\\\n", @@ -581,9 +497,7 @@ { "cell_type": "markdown", "id": "fc3e4f2b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is an $m\\times n$ matrix. If $\\boldsymbol{x}$ is a scalar, then the\n", "Jacobian is only a single-column vector, or an $m\\times 1$ matrix. If\n", @@ -599,9 +513,7 @@ { "cell_type": "markdown", "id": "9624726e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Derivatives, example 1\n", "\n", @@ -611,9 +523,7 @@ { "cell_type": "markdown", "id": "e9579efb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "y_i = \\sum_{j=0}^{n-1}a_{ij}x_j,\n", @@ -623,9 +533,7 @@ { "cell_type": "markdown", "id": "d6da92cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\forall i=0,1,2,\\dots,m-1$. The individual matrix elements of $\\boldsymbol{A}$ are given by the symbol $a_{ij}$.\n", "It follows that the partial derivatives of $y_i$ with respect to $x_k$" @@ -634,9 +542,7 @@ { "cell_type": "markdown", "id": "214a54af", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial y_i }{\\partial x_k}= a_{ik} \\forall i=0,1,2,\\dots,m-1.\n", @@ -646,9 +552,7 @@ { "cell_type": "markdown", "id": "736edceb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "From this we have, using the definition of the Jacobian" ] @@ -656,9 +560,7 @@ { "cell_type": "markdown", "id": "70b72395", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\boldsymbol{y} }{\\partial \\boldsymbol{x}}= \\boldsymbol{A}.\n", @@ -668,9 +570,7 @@ { "cell_type": "markdown", "id": "d751f216", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 2\n", "\n", @@ -682,9 +582,7 @@ { "cell_type": "markdown", "id": "46b18e4b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{y}^T\\boldsymbol{A}\\boldsymbol{x},\n", @@ -694,9 +592,7 @@ { "cell_type": "markdown", "id": "dc835d90", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{y}$ a vector of length $m$, $\\boldsymbol{A}$ an $m\\times n$ matrix and $\\boldsymbol{x}$ a vector of length $n$. We assume also that $\\boldsymbol{A}$ does not depend on any of the two vectors.\n", "In order to find the derivative of $\\alpha$ with respect to the two vectors, we define an intermediate vector $\\boldsymbol{z}$. We define first\n", @@ -706,9 +602,7 @@ { "cell_type": "markdown", "id": "9dff0ac1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{z}^T\\boldsymbol{x},\n", @@ -718,9 +612,7 @@ { "cell_type": "markdown", "id": "243d40fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which means that (using our previous example and keeping track of our definition of the derivative of a scalar) we have" ] @@ -728,9 +620,7 @@ { "cell_type": "markdown", "id": "022a53ae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = \\frac{\\partial \\boldsymbol{z}^T\\boldsymbol{x}}{\\partial \\boldsymbol{x}}=\\boldsymbol{z}^T.\n", @@ -740,9 +630,7 @@ { "cell_type": "markdown", "id": "b321d425", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Note that the resulting vector elements are the same for $\\boldsymbol{z}^T$ and $\\boldsymbol{z}$, the only difference is that one is just the transpose of the other. We have the transposed here since we have used that the inner product of two vectors is a scalar.\n", "\n", @@ -752,9 +640,7 @@ { "cell_type": "markdown", "id": "edc20e3b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{y}} = \\boldsymbol{z}^T=\\boldsymbol{x}^T\\boldsymbol{A}^T.\n", @@ -764,9 +650,7 @@ { "cell_type": "markdown", "id": "a03ea760", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 3\n", "\n", @@ -778,9 +662,7 @@ { "cell_type": "markdown", "id": "89b04349", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{x}^T\\boldsymbol{A}\\boldsymbol{x},\n", @@ -790,9 +672,7 @@ { "cell_type": "markdown", "id": "cde4e746", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{x}$ a vector of length $n$.\n", "\n", @@ -802,9 +682,7 @@ { "cell_type": "markdown", "id": "b4d2c178", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\sum_{i=0}^{n-1}\\sum_{j=0}^{n-1}x_i a_{ij}x_j,\n", @@ -814,9 +692,7 @@ { "cell_type": "markdown", "id": "21a9b2ea", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "taking the derivative of $\\alpha$ with respect to a given component $x_k$ we get the two sums" ] @@ -824,9 +700,7 @@ { "cell_type": "markdown", "id": "b3cab956", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial x_k} = \\sum_{i=0}^{n-1}a_{ik}x_i+\\sum_{j=0}^{n-1}a_{kj}x_j,\n", @@ -836,9 +710,7 @@ { "cell_type": "markdown", "id": "47fbe8fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $\\forall k =0,1,2,\\dots,n-1$. We identify these sums as" ] @@ -846,9 +718,7 @@ { "cell_type": "markdown", "id": "254a6af2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = \\boldsymbol{x}^T\\left(\\boldsymbol{A}^T+\\boldsymbol{A}\\right).\n", @@ -858,9 +728,7 @@ { "cell_type": "markdown", "id": "f8b23553", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If the matrix $\\boldsymbol{A}$ is symmetric, that is $\\boldsymbol{A}=\\boldsymbol{A}^T$, we have" ] @@ -868,9 +736,7 @@ { "cell_type": "markdown", "id": "1dbfd08f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{x}} = 2\\boldsymbol{x}^T\\boldsymbol{A}.\n", @@ -880,9 +746,7 @@ { "cell_type": "markdown", "id": "23802261", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example 4\n", "\n", @@ -892,9 +756,7 @@ { "cell_type": "markdown", "id": "5e2f646b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\boldsymbol{y}^T\\boldsymbol{x},\n", @@ -904,9 +766,7 @@ { "cell_type": "markdown", "id": "0457dac9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where both $\\boldsymbol{y}$ and $\\boldsymbol{x}$ have the same length $n$, or if we\n", "wish to think of them as column vectors, they have dimensions $n\\times\n", @@ -919,9 +779,7 @@ { "cell_type": "markdown", "id": "795da8f4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\alpha = \\sum_{i=0}^{n-1}y_ix_i,\n", @@ -931,9 +789,7 @@ { "cell_type": "markdown", "id": "81262cf1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the partial derivative" ] @@ -941,9 +797,7 @@ { "cell_type": "markdown", "id": "fef60c8c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial z_k} = \\sum_{i=0}^{n-1}\\left(x_i\\frac{\\partial y_i}{\\partial z_k}+y_i\\frac{\\partial x_i}{\\partial z_k}\\right),\n", @@ -953,9 +807,7 @@ { "cell_type": "markdown", "id": "80836258", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for $\\forall k =0,1,2,\\dots,n-1$. We can rewrite the partial derivative in a more compact form as" ] @@ -963,9 +815,7 @@ { "cell_type": "markdown", "id": "7f130753", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{z}} = \\boldsymbol{x}^T\\frac{\\partial \\boldsymbol{y}}{\\partial \\boldsymbol{z}}+\\boldsymbol{y}^T\\frac{\\partial \\boldsymbol{x}}{\\partial \\boldsymbol{z}},\n", @@ -975,9 +825,7 @@ { "cell_type": "markdown", "id": "9c42dc95", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and if $\\boldsymbol{y}=\\boldsymbol{x}$ we have" ] @@ -985,9 +833,7 @@ { "cell_type": "markdown", "id": "72dc1ea4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\alpha}{\\partial \\boldsymbol{z}} = 2\\boldsymbol{x}^T\\frac{\\partial \\boldsymbol{x}}{\\partial \\boldsymbol{z}}.\n", @@ -997,9 +843,7 @@ { "cell_type": "markdown", "id": "9755f3e0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The mean squared error and its derivative\n", "\n", @@ -1009,9 +853,7 @@ { "cell_type": "markdown", "id": "e7505b7d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", @@ -1021,9 +863,7 @@ { "cell_type": "markdown", "id": "602afb09", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or using the design/feature matrix $\\boldsymbol{X}$ we have the more compact matrix-vector" ] @@ -1031,9 +871,7 @@ { "cell_type": "markdown", "id": "73328415", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right\\}.\n", @@ -1043,9 +881,7 @@ { "cell_type": "markdown", "id": "357f9fcb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We note that the design matrix $\\boldsymbol{X}$ does not depend on the unknown parameters defined by the vector $\\boldsymbol{\\theta}$.\n", "We are now interested in minimizing the cost function with respect to the unknown parameters $\\boldsymbol{\\theta}$.\n", @@ -1056,9 +892,7 @@ { "cell_type": "markdown", "id": "155beb6e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{w}=\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta},\n", @@ -1068,9 +902,7 @@ { "cell_type": "markdown", "id": "14dd6c43", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which depends on $\\boldsymbol{\\theta}$. We rewrite the cost function as" ] @@ -1078,9 +910,7 @@ { "cell_type": "markdown", "id": "65041e77", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta})=\\frac{1}{n}\\boldsymbol{w}^T\\boldsymbol{w},\n", @@ -1090,9 +920,7 @@ { "cell_type": "markdown", "id": "90a53e7e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with partial derivative" ] @@ -1100,9 +928,7 @@ { "cell_type": "markdown", "id": "104011e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}}=\\frac{2}{n}\\boldsymbol{w}^T\\frac{\\partial \\boldsymbol{w}}{\\partial \\boldsymbol{\\theta}},\n", @@ -1112,9 +938,7 @@ { "cell_type": "markdown", "id": "046656ab", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using that" ] @@ -1122,9 +946,7 @@ { "cell_type": "markdown", "id": "5d29e5be", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial \\boldsymbol{w}}{\\partial \\boldsymbol{\\theta}}=-\\boldsymbol{X},\n", @@ -1134,9 +956,7 @@ { "cell_type": "markdown", "id": "2da73e87", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we used the result from example two above. Inserting the last expression we obtain" ] @@ -1144,9 +964,7 @@ { "cell_type": "markdown", "id": "0f5cb23b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}}=-\\frac{2}{n}\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\boldsymbol{X},\n", @@ -1156,9 +974,7 @@ { "cell_type": "markdown", "id": "3f41a473", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or as" ] @@ -1166,9 +982,7 @@ { "cell_type": "markdown", "id": "5e2f3e3c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}^T}=-\\frac{2}{n}\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right).\n", @@ -1178,9 +992,7 @@ { "cell_type": "markdown", "id": "4d58e7cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Meet the Hessian Matrix\n", "\n", @@ -1194,9 +1006,7 @@ { "cell_type": "markdown", "id": "3f37e988", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial}{\\partial \\boldsymbol{\\theta}}\\frac{\\partial C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}^T} =\\frac{\\partial}{\\partial \\boldsymbol{\\theta}}\\left[-\\frac{2}{n}\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right]=\\frac{2}{n}\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -1206,9 +1016,7 @@ { "cell_type": "markdown", "id": "e259bd27", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The Hessian matrix plays an important role and is defined here as" ] @@ -1216,9 +1024,7 @@ { "cell_type": "markdown", "id": "143d5d28", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{H}=\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -1228,9 +1034,7 @@ { "cell_type": "markdown", "id": "af9f94d8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For ordinary least squares, it is inversely proportional (derivation\n", "next week) with the variance of the optimal parameters\n", @@ -1246,9 +1050,7 @@ { "cell_type": "markdown", "id": "387e3480", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpretations and optimizing our parameters\n", "\n", @@ -1258,9 +1060,7 @@ { "cell_type": "markdown", "id": "19eb19ee", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon} = \\boldsymbol{y}-\\boldsymbol{\\tilde{y}} = \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta},\n", @@ -1270,9 +1070,7 @@ { "cell_type": "markdown", "id": "dd32a1a4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and with" ] @@ -1280,9 +1078,7 @@ { "cell_type": "markdown", "id": "b7fc97aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)= 0,\n", @@ -1292,9 +1088,7 @@ { "cell_type": "markdown", "id": "92f090a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have" ] @@ -1302,9 +1096,7 @@ { "cell_type": "markdown", "id": "a07b14b0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{\\epsilon}=\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)= 0,\n", @@ -1314,9 +1106,7 @@ { "cell_type": "markdown", "id": "690f1feb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that the solution for $\\boldsymbol{\\theta}$ is the one which minimizes the residuals." ] @@ -1324,9 +1114,7 @@ { "cell_type": "markdown", "id": "98ce0e54", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example relevant for the exercises\n", "\n", @@ -1338,9 +1126,7 @@ { "cell_type": "markdown", "id": "8355044d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_i = \\theta_0+\\theta_1x_i+\\theta_2x_i^2+\\theta_3x_i^3+\\theta_4x_i^4.\n", @@ -1350,9 +1136,7 @@ { "cell_type": "markdown", "id": "a6b37202", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have five predictors/features. The first is the intercept $\\theta_0$. The other terms are $\\theta_i$ with $i=1,2,3,4$. Furthermore we have $n$ entries for each predictor. It means that our design matrix is an \n", "$n\\times p$ matrix $\\boldsymbol{X}$." @@ -1361,9 +1145,7 @@ { "cell_type": "markdown", "id": "9554ba3d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Own code for Ordinary Least Squares\n", "\n", @@ -1374,10 +1156,7 @@ "cell_type": "code", "execution_count": 1, "id": "5996cc7a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# matrix inversion to find theta\n", @@ -1401,9 +1180,7 @@ { "cell_type": "markdown", "id": "05513cf5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Alternatively, you can use the least squares functionality in **Numpy** as" ] @@ -1412,10 +1189,7 @@ "cell_type": "code", "execution_count": 2, "id": "6383f412", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "fit = np.linalg.lstsq(X, y, rcond =None)[0]\n", @@ -1425,9 +1199,7 @@ { "cell_type": "markdown", "id": "1201be66", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Adding error analysis and training set up\n", "\n", @@ -1439,10 +1211,7 @@ "cell_type": "code", "execution_count": 3, "id": "6f353ffc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def R2(y_data, y_model):\n", @@ -1452,9 +1221,7 @@ { "cell_type": "markdown", "id": "d8571bf0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and we would be using it as" ] @@ -1463,10 +1230,7 @@ "cell_type": "code", "execution_count": 4, "id": "203d6f02", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(R2(y,ytilde))" @@ -1475,9 +1239,7 @@ { "cell_type": "markdown", "id": "8e158e42", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can easily add our **MSE** score as" ] @@ -1486,10 +1248,7 @@ "cell_type": "code", "execution_count": 5, "id": "cc537d67", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def MSE(y_data,y_model):\n", @@ -1502,9 +1261,7 @@ { "cell_type": "markdown", "id": "dcfdabf6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and finally the relative error as" ] @@ -1513,10 +1270,7 @@ "cell_type": "code", "execution_count": 6, "id": "d9cd7659", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "def RelativeError(y_data,y_model):\n", @@ -1527,9 +1281,7 @@ { "cell_type": "markdown", "id": "a812e890", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Splitting our Data in Training and Test data\n", "\n", @@ -1548,22 +1300,33 @@ { "cell_type": "markdown", "id": "c16d6b48", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The complete code with a simple data set" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "id": "bd392aef", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 2.04908117 -0.13958616 4.2200579 2.12043941 -1.26158339]\n", + "Training R2\n", + "0.9953681781455221\n", + "Training MSE\n", + "0.009784636796208437\n", + "Test R2\n", + "0.9959617965662848\n", + "Test MSE\n", + "0.008116074243527612\n" + ] + } + ], "source": [ "%matplotlib inline\n", "\n", @@ -1612,21 +1375,16 @@ { "cell_type": "markdown", "id": "e01305b4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Making your own test-train splitting" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "c274659b", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# equivalently in numpy\n", @@ -1648,9 +1406,7 @@ { "cell_type": "markdown", "id": "40c22ab9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "But since **scikit-learn** has its own function for doing this and since\n", "it interfaces easily with **tensorflow** and other libraries, we\n", @@ -1660,9 +1416,7 @@ { "cell_type": "markdown", "id": "14c0cb14", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Reducing the number of degrees of freedom, overarching view\n", "\n", @@ -1689,9 +1443,7 @@ { "cell_type": "markdown", "id": "d0adb995", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Preprocessing our data\n", "\n", @@ -1714,9 +1466,7 @@ { "cell_type": "markdown", "id": "c8acfb81", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Functionality in Scikit-Learn\n", "\n", @@ -1734,9 +1484,7 @@ { "cell_type": "markdown", "id": "25284826", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More preprocessing\n", "\n", @@ -1761,9 +1509,7 @@ { "cell_type": "markdown", "id": "a673748d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Frequently used scaling functions\n", "\n", @@ -1774,9 +1520,7 @@ { "cell_type": "markdown", "id": "82b9b725", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n", @@ -1786,9 +1530,7 @@ { "cell_type": "markdown", "id": "ac08b34d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard deviation, respectively, of the feature $x_j$.\n", "This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one." @@ -1797,9 +1539,7 @@ { "cell_type": "markdown", "id": "8e4281c3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example of own Standard scaling\n", "\n", @@ -1811,13 +1551,419 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "id": "81ebdd6e", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
01234
0-1.7497650.3426801.153036-0.2524360.981321
10.5142190.221180-1.070043-0.1894960.255001
2-0.4580270.435163-0.5835950.8168470.672721
3-0.104411-0.5312801.029733-0.438136-1.118318
41.6189821.541605-0.251879-0.8424360.184519
50.9370820.7310001.361556-0.3262380.055676
60.222400-1.443217-0.7563520.8164540.750445
7-0.4559471.189622-1.690617-1.356399-1.232435
8-0.544439-0.6681720.007315-0.6129391.299748
9-1.733096-0.9833100.357508-1.6135791.470714
\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4\n", + "0 -1.749765 0.342680 1.153036 -0.252436 0.981321\n", + "1 0.514219 0.221180 -1.070043 -0.189496 0.255001\n", + "2 -0.458027 0.435163 -0.583595 0.816847 0.672721\n", + "3 -0.104411 -0.531280 1.029733 -0.438136 -1.118318\n", + "4 1.618982 1.541605 -0.251879 -0.842436 0.184519\n", + "5 0.937082 0.731000 1.361556 -0.326238 0.055676\n", + "6 0.222400 -1.443217 -0.756352 0.816454 0.750445\n", + "7 -0.455947 1.189622 -1.690617 -1.356399 -1.232435\n", + "8 -0.544439 -0.668172 0.007315 -0.612939 1.299748\n", + "9 -1.733096 -0.983310 0.357508 -1.613579 1.470714" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 -0.175300\n", + "1 0.083527\n", + "2 -0.044334\n", + "3 -0.399836\n", + "4 0.331939\n", + "dtype: float64\n", + "0 1.069584\n", + "1 0.965548\n", + "2 1.018232\n", + "3 0.793167\n", + "4 0.918992\n", + "dtype: float64\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
01234
0-1.5744650.2591531.1973700.1474000.649382
10.6895190.137652-1.0257090.210340-0.076938
2-0.2827270.351636-0.5392611.2166830.340782
30.070889-0.6148081.074067-0.038300-1.450257
41.7942821.458078-0.207545-0.442600-0.147420
51.1123830.6474731.4058900.073598-0.276263
60.397700-1.526744-0.7120181.2162900.418506
7-0.2806471.106095-1.646283-0.956563-1.564374
8-0.369139-0.7516990.051649-0.2131030.967809
9-1.557795-1.0668370.401842-1.2137431.138775
\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4\n", + "0 -1.574465 0.259153 1.197370 0.147400 0.649382\n", + "1 0.689519 0.137652 -1.025709 0.210340 -0.076938\n", + "2 -0.282727 0.351636 -0.539261 1.216683 0.340782\n", + "3 0.070889 -0.614808 1.074067 -0.038300 -1.450257\n", + "4 1.794282 1.458078 -0.207545 -0.442600 -0.147420\n", + "5 1.112383 0.647473 1.405890 0.073598 -0.276263\n", + "6 0.397700 -1.526744 -0.712018 1.216290 0.418506\n", + "7 -0.280647 1.106095 -1.646283 -0.956563 -1.564374\n", + "8 -0.369139 -0.751699 0.051649 -0.213103 0.967809\n", + "9 -1.557795 -1.066837 0.401842 -1.213743 1.138775" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
01234
00.00.00.00.00.0
10.00.00.00.00.0
20.00.00.00.00.0
30.00.00.00.00.0
40.00.00.00.00.0
50.00.00.00.00.0
60.00.00.00.00.0
70.00.00.00.00.0
80.00.00.00.00.0
90.00.00.00.00.0
\n", + "
" + ], + "text/plain": [ + " 0 1 2 3 4\n", + "0 0.0 0.0 0.0 0.0 0.0\n", + "1 0.0 0.0 0.0 0.0 0.0\n", + "2 0.0 0.0 0.0 0.0 0.0\n", + "3 0.0 0.0 0.0 0.0 0.0\n", + "4 0.0 0.0 0.0 0.0 0.0\n", + "5 0.0 0.0 0.0 0.0 0.0\n", + "6 0.0 0.0 0.0 0.0 0.0\n", + "7 0.0 0.0 0.0 0.0 0.0\n", + "8 0.0 0.0 0.0 0.0 0.0\n", + "9 0.0 0.0 0.0 0.0 0.0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import sklearn.linear_model as skl\n", "from sklearn.metrics import mean_squared_error\n", @@ -1847,9 +1993,7 @@ { "cell_type": "markdown", "id": "beb0eda1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives." ] @@ -1857,9 +2001,7 @@ { "cell_type": "markdown", "id": "d2f901fc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Min-Max Scaling\n", "\n", @@ -1872,9 +2014,7 @@ { "cell_type": "markdown", "id": "3d254833", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "x_j^{(i)} \\rightarrow (b-a)\\frac{x_j^{(i)} - \\min(x_j)}{\\max(x_j) - \\min(x_j)} - a\n", @@ -1884,9 +2024,7 @@ { "cell_type": "markdown", "id": "4e7c28a9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively." ] @@ -1894,9 +2032,7 @@ { "cell_type": "markdown", "id": "763af26f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Testing the Means Squared Error as function of Complexity\n", "\n", @@ -1908,12 +2044,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "id": "cea37f94", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.random.seed()\n", @@ -1927,9 +2060,7 @@ { "cell_type": "markdown", "id": "54451e45", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $y$ is the function we want to fit with a given polynomial.\n", "\n", @@ -1938,13 +2069,23 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 23, "id": "498277ed", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1DUlEQVR4nO3deXycZbn4/881SzKZrDNJuqZtWmkLhe6xLLIjCIoUFaT85NCjfEVEUDhHRfSlcvTw/YrLQVFEOcpBcKGKImURVKBSThFIS+kGXSilTfdM1pkks96/P55n0jTNMkkmM9PM9X69+so869xPp50r93bdYoxBKaWUcmS7AEoppXKDBgSllFKABgSllFI2DQhKKaUADQhKKaVsrmwXYCiqqqpMbW1ttouhlFLHjbVr1zYaY6pTOfe4Cgi1tbXU19dnuxhKKXXcEJF3Uz1Xm4yUUkoBGhCUUkrZNCAopZQCjrM+BKXU8S8ajdLQ0EBXV1e2izKmeDweampqcLvdw76HBgSlVEY1NDRQWlpKbW0tIpLt4owJxhgCgQANDQ1Mnz592PfRJiOlVEZ1dXVRWVmpwSCNRITKysoR17o0ICilMk6DQfql4+907AcEY+Af34Udf892SZRSKqeN/YAgAmt+DNs1ICilIBAIsGDBAhYsWMCECROYPHly93YkEhn0+lWrVrFmzZo+jz344INUV1d332/BggVs2bIl3Y8wavKjU7nIBx2BbJdCKZUDKisrWb9+PQB33HEHJSUlfPGLX0z5+lWrVlFSUsIZZ5zR5/GrrrqKn/zkJ/1eH4vFcLlc/W6net1oGPs1BABvJXQ2ZbsUSqkctXbtWs455xwWL17MBz7wAfbv3w/APffcw5w5c5g3bx7Lli1j165d/OxnP+Puu+9mwYIFrF69OqX7r1q1irPOOovLLruMOXPmHLPd1dXFJz/5SebOncvChQt54YUXAKvGcdlll3H++edzwQUXjNrzJ+VHDcHr1xqCUjnoP57YzJZ9bWm955xJZXzzwyenfL4xhptvvpnHH3+c6upqVqxYwde+9jUeeOABvvOd7/DOO+9QWFhIS0sLFRUV3HDDDQPWKlasWMFLL73Uvf3yyy8DsG7dOjZt2sT06dNZtWrVUds/+MEPEBE2btzIW2+9xUUXXcS2bdu6r9uwYQN+v38EfyupGfMBwRhDU6KEsuA2hj9dQyk1VoXDYTZt2sSFF14IQDweZ+LEiQDMmzePT3ziE1x++eVcfvnlKd2vvyajJUuWHDVHoOf2Sy+9xM033wzAiSeeyLRp07oDwoUXXpiRYAB5EBAAntwRZpmrMdvFUEr1MpTf5EeLMYaTTz65+zf5np566ilefPFFnnjiCe688042btw47PcpLi4ecDvV60bTmO9DEBHCBT4KEx0QG3wEgVIqvxQWFnL48OHugBCNRtm8eTOJRII9e/Zw3nnncdddd9Ha2kowGKS0tJT29va0luGss87iN7/5DQDbtm1j9+7dzJ49O63vkYoxHxAA4oUV1gvtWFZK9eJwOHj00Ue57bbbmD9/PgsWLGDNmjXE43Guueaa7o7ez3/+81RUVPDhD3+Yxx57rN9O5RUrVhw17LS/Iao93XjjjSQSCebOnctVV13Fgw8+SGFh4Wg87oDEGJPxNx2uuro6M5wFcn567/e58fC34bMvw/g5o1AypVSq3nzzTU466aRsF2NM6uvvVkTWGmPqUrk+L2oIzmK7Q0ZHGimlVL/yIiC4S63lRI0GBKWU6ldeBARPmRUQuloPZ7kkSimVu/IiIJT6xwHQoQFBKaX6lRcBwVdWRsgUEmnTgKCUUv3Ji4lplSUFNFMKIe1DUEqp/uRNQDhoSvB16DwEpfJdIBDoThR34MABnE4n1dVWP+Orr75KQUFBv9fW19fz0EMPcc8996T8frW1tZSWluJ0OgE4++yzh3R9JqUUEETkYuBHgBP4hTHmO72OFwIPAYuBAHCVMWaXiFQCjwLvBR40xtzU45rFwINAEfA08AUzSpMi/N4C3qKUqq7m0bi9Uuo4Mlj664HSTNfV1VFXl9KQ/qO88MILVFVV9Xt8uCmx4/F4d6BJh0H7EETECdwLXALMAa4Wkd6zu64Dmo0xJwB3A3fZ+7uArwN9pQW8D/g0MNP+c/FwHiAVLqeDkLOcgogGBKXUsf71X/+VG264gVNPPZUvf/nLvPrqq5x++uksXLiQM844g61btwJWGutLL70UsILJpz71Kc4991xmzJgx5N/6zz33XG655Rbq6ur40Y9+dMz2c889x8KFC5k7dy6f+tSnCIfDgFXjuO2221i0aBF/+MMf0vr3kEoNYQmwwxizE0BEHgGWAj2XAVoK3GG/fhT4iYiIMSYEvCQiJ/S8oYhMBMqMMf+0tx8CLgf+MvxHGViXuwJPrHW0bq+UGo6/fAUODD9hXJ8mzIVLvjP4eb00NDSwZs0anE4nbW1trF69GpfLxd///ne++tWv8sc//vGYa9566y1eeOEF2tvbmT17Np/97Gdxu4/Nq3zeeed1/ya/fPlybr31VgAikQjJ7AtPPPFE93ZXVxczZ87kueeeY9asWVx77bXcd9993HLLLYBVy1m3bt2Qn3EwqQSEycCeHtsNwKn9nWOMiYlIK1AJ9JdidLJ9n573nNzXiSJyPXA9wNSpU1Mobt9ihRUUR4IQj4EzL7pOlFJDcOWVV3Z/abe2trJ8+XK2b9+OiBCNRvu85kMf+hCFhYUUFhYybtw4Dh48SE1NzTHn9ddkdNVVV/W5vXXrVqZPn86sWbMAK4jce++93QGh93XpkvPfjMaY+4H7wcplNOz7FPmhHehshpLqdBVPKTUSw/hNfrT0TDP99a9/nfPOO4/HHnuMXbt2ce655/Z5Tc8EdE6nk1gsNuz37Gs71evSJZV5CHuBKT22a+x9fZ4jIi6gHKtzeaB79gyjfd0zrRzeSuuFpq9QSg2itbWVyZOtRosHH3ww4+8/e/Zsdu3axY4dOwB4+OGHOeecc0b9fVMJCK8BM0VkuogUAMuAlb3OWQkst19fATw/0IghY8x+oE1EThMRAa4FHh9y6YfAVWJV1yJBXShHKTWwL3/5y9x+++0sXLhwyL/19+W8887rTod97bXXDnq+x+Phf/7nf7jyyiuZO3cuDoeDG264YcTlGExK6a9F5IPAD7GGnT5gjLlTRL4F1BtjVoqIB3gYWAg0Act6dELvAsqAAqAFuMgYs0VE6jgy7PQvwM2DDTsdbvprgGf+9iwX/+/Habr0Afx1HxvWPZRSI6fpr0fPSNNfp9SHYIx5GmuuQM993+jxugu4sp9ra/vZXw+cksr7p0NRhZ3PqOUwmVmdVCmlji95kcsIoNQ/HoBwu+YzUkqpvuRNQKisKKfTFBBr1z4EpbLteFqp8XiRjr/TvAkIVSWFNFOii+QolWUej4dAIKBBIY2MMQQCATwez4juk/PzENLFW+DkXcpwaT4jpbKqpqaGhoYGDh/W5tt08ng8fU6KG4q8CQgiQshZRlVYA4JS2eR2u5k+fXq2i6H6kDdNRgBdrgo8Uc1npJRSfcmrgBAtrKA4rgFBKaX6klcBIe7xU2KCkIhnuyhKKZVz8iogiNePA4Pp1H4EpdTx4Z87A/xpXcPgJ6ZBXgUEZ6mVz6i96VCWS6KUUqn507oGvvOXtzLyXnkVEArtgNDWdDDLJVFKqdQ0hSJUlhQOfmIa5FVAKCq38hmFWrSGoJQ6PjQGI1SVFGTkvfIqIJT6rYDQ1aoTYpRSx4dAKExlsQaEtCuvnABATNdEUEodJ5qCEfzF2mSUdr7yCsLGRSKk+YyUUrmvMxInFIlTqU1G6edyOWmVMqSzKdtFUUqpQQVCYQDtQxgtQUcpri4NCEqp3NcUigBok9Fo6XRVUKj5jJRSx4FA0AoI2mQ0SsIFFXhjGhCUUrmvMWg3GWkNYXRY+Yzasl0MpZQaVHeTkdYQRkmRjwrTTlckmu2SKKXUgAKhCIUuB8UFzoy8X94FBGdJFU4xNDfpXASlVG5rDIapKilERDLyfnkXENyllQC0BTSfkVIqtzWFIvgzNEsZ8jAgeOx8RsFmDQhKqdwWCEYyNsII8jAglFaMB6CzVRPcKaVyWyAYpjJDI4wgDwNCWaUVECLtmr5CKZW7jDE0hrSGMKqKyq01ERIh7VRWSuWuUCROJJbIWKZTSDEgiMjFIrJVRHaIyFf6OF4oIivs46+ISG2PY7fb+7eKyAd67L9VRDaLyCYR+Z2IeNLyRIM9i6ecGE5Mh6avUErlroA9KS1Ti+NACgFBRJzAvcAlwBzgahGZ0+u064BmY8wJwN3AXfa1c4BlwMnAxcBPRcQpIpOBzwN1xphTAKd93ugToV3KNJ+RUiqnNSbTVuRYDWEJsMMYs9MYEwEeAZb2Omcp8Cv79aPABWINnF0KPGKMCRtj3gF22PcDcAFFIuICvMC+kT1K6jpc5RRENH2FUip3JWcp51ofwmRgT4/tBntfn+cYY2JAK1DZ37XGmL3A94HdwH6g1Rjz177eXESuF5F6Eak/fDg9K52F3eUUxVrSci+llBoNOdlkNBpExIdVe5gOTAKKReSavs41xtxvjKkzxtRVV1en5f2jHh8l8TYSCZOW+ymlVLoFQrnZZLQXmNJju8be1+c5dhNQORAY4Nr3A+8YYw4bY6LAn4AzhvMAw2GK/PiknbYuzWeklMpNgWCE4gInHndm8hhBagHhNWCmiEwXkQKszt+Vvc5ZCSy3X18BPG+MMfb+ZfYopOnATOBVrKai00TEa/c1XAC8OfLHSY2juJIKgjS2hzP1lkopNSSBUDijzUVgdewOyBgTE5GbgGexRgM9YIzZLCLfAuqNMSuBXwIPi8gOoAl7xJB93u+BLUAM+JwxJg68IiKPAuvs/a8D96f/8frmLqnCLXGamxthfGmm3lYppVKW6bQVkEJAADDGPA083WvfN3q87gKu7OfaO4E7+9j/TeCbQylsuhSWW30R7c2HsboxlFIqtwRCESZXZGR6Vre8m6kMUGwnuOts0QR3SqnclOk8RpCvAcFnBYRwu6avUErlHmMMTRnOYwR5GhBcJVY+o5gGBKVUDmrrjBFLmIyuhQB5GhDw+q2fms9IKZWDGkPWCMiqDI8yys+AUFhOAgcOzWeklMpBgWDm01ZAvgYEh4OQswx3uCXbJVFKqWM02TUEbTLKkC5XOUXRlmwXQymljpHMdKpNRhkSLaygJNFGVzSe7aIopdRRkk1GPq/WEDIiYeczSqaYVUqpXBEIhSnzuChwZfYrOm8DAkV+fBKkMaj5jJRSuSUQimS8uQjyOCC4Sqvw0U5AE9wppXJMIBjO+AgjyOOAUFhaTaHEaG5tyXZRlFLqKIFgJOMjjCCPA4K3IpnP6FCWS6KUUkez0lZok1HGFJZZ6Ss6W9OzLKdSSqVDPGFo6ohQpTWEDPJWAhALaj4jpVTuaO6IYEzmJ6VBPgeEIiufUSIUyHJBlFLqiORQeG0yyiS7hiCdms9IKZU7kkPhdZRRJhVVkEBwhZuzXRKllOoWyFLaCsjngOBwEnaVUhRtIZEw2S6NUkoBR5qMtA8hwyIFFZTTTltXNNtFUUopwJqUJpL5PEaQ5wEh7vHjo13TVyilckZjKILfW4DTIRl/77wOCEfyGWmCO6VUbmjK0ixlyPOA4CipxCft3Z04SimVbYFQdvIYQZ4HhIKSKnxoxlOlVO4IBLOTtgLAlZV3zRGe8nE4JEJra2u2i6KUUoCV+rpSm4wyz1FszVbubNN8Rkqp7IvEErR2Rqkszk4NIa8DQnK2crRdA4JSKvuaO5JpK3K4hiAiF4vIVhHZISJf6eN4oYissI+/IiK1PY7dbu/fKiIf6LG/QkQeFZG3RORNETk9LU80FHY+o3hQ01copbKvO21FrjYZiYgTuBe4BJgDXC0ic3qddh3QbIw5AbgbuMu+dg6wDDgZuBj4qX0/gB8BzxhjTgTmA2+O/HGGqDufkSa4U0plXzYT20FqNYQlwA5jzE5jTAR4BFja65ylwK/s148CF4iI2PsfMcaEjTHvADuAJSJSDpwN/BLAGBMxxrSM+GmGymvVEJxdms9IKZV9ySHwudxkNBnY02O7wd7X5znGmBjQClQOcO104DDwPyLyuoj8QkSK+3pzEbleROpFpP7w4TS39Rf5APDGWumKxtN7b6WUGqKcbzIaJS5gEXCfMWYhEAKO6ZsAMMbcb4ypM8bUVVdXp7cUTjcRVykVEiQQ0slpSqnsagpFcDmEMo87K++fSkDYC0zpsV1j7+vzHBFxAeVAYIBrG4AGY8wr9v5HsQJExsU8PvzSTkAnpymlsixgp61wZCGPEaQWEF4DZorIdBEpwOokXtnrnJXAcvv1FcDzxhhj719mj0KaDswEXjXGHAD2iMhs+5oLgC0jfJZhMR4fPjR9hVIq+wKhcNbyGEEKM5WNMTERuQl4FnACDxhjNovIt4B6Y8xKrM7hh0VkB9CEFTSwz/s91pd9DPicMSbZWH8z8Bs7yOwEPpnmZ0uJo7gKn7zNm1pDUEplWSAUycrCOEkppa4wxjwNPN1r3zd6vO4Cruzn2juBO/vYvx6oG0JZR4W7tBKfvKE1BKVU1gWCEaZO9Wbt/fN7pjLgKqm2m4y0hqCUyq5AMJy1tBWgAQG8PoolTGt7e7ZLopTKY13ROKFIPGtzEEADQvds5XBbY5YLopTKZ8mh79magwAaELrzGUWDmr5CKZU9yWbrbKWtAA0IR/IZdWhAUEplT7KGkM1hpxoQeuQzSiRMlgujlMpXyZGOVdqHkEV2DaGcNlo7o1kujFIqX2mTUS6w+xAqCBII6dBTpVR2NIUiFLgcFBc4Bz95lGhAcBUQc5fgl3YadXKaUipLGoMRqooLsFYOyA4NCEDC46NCgt2pZ5VSKtMCoXBWm4tAAwIA4q3ErwnulFJZlMx0mk0aEABXSSU+TYGtlMqiplAkq7OUQQMCYNUQKh1BGnWRHKVUFhhjaAyGs5rpFDQgWLx+fARpbNcaglIq80KROOFYQpuMcoK3kmI6aA2Gsl0SpVQeagpmP48RaECwFPkAiLZr+gqlVOY12nOgtMkoF9izlROaz0gplQXJEY7aZJQL7HxGhZFWuqLxQU5WSqn0agol01ZoQMg+u4bgk/bujINKKZUpjd19CNpklH12PiOdi6CUyoZAMIK3wElRFvMYgQYEi91k5KNd01copTKuKRTOenMRaECwuItIuIrwSVAT3CmlMi4QimS9uQg0IBzhrcQvms9IKZV5jcFIVhfGSdKAYHN4/VQ6QtqHoJTKuKZQOOtDTkEDwhHeSqodmgJbKZVZxhgCwUjWU1+DBoQjvH58jqAOO1VKZVRbZ4xYwmQ9bQWkGBBE5GIR2SoiO0TkK30cLxSRFfbxV0Sktsex2+39W0XkA72uc4rI6yLy5IifZKS8lZSbNu1UVkplVCBHJqVBCgFBRJzAvcAlwBzgahGZ0+u064BmY8wJwN3AXfa1c4BlwMnAxcBP7fslfQF4c6QPkRZFfooTQZrbO7JdEqVUHkm2Shwvo4yWADuMMTuNMRHgEWBpr3OWAr+yXz8KXCDWwqBLgUeMMWFjzDvADvt+iEgN8CHgFyN/jDTozmfURCJhslwYpVS+SA5kOS5qCMBkYE+P7QZ7X5/nGGNiQCtQOci1PwS+DCQGenMRuV5E6kWk/vDhwykUd5jsyWmlpp3WzujovY9SSvVwvNUQ0k5ELgUOGWPWDnauMeZ+Y0ydMaauurp69AplBwQ/7d1tekopNdpyJdMppBYQ9gJTemzX2Pv6PEdEXEA5EBjg2vcBl4nILqwmqPNF5NfDKH/69MhndLhdO5aVUoP7+5aD7GkaWb9jIBimzOOiwJX9QZ+plOA1YKaITBeRAqxO4pW9zlkJLLdfXwE8b4wx9v5l9iik6cBM4FVjzO3GmBpjTK19v+eNMdek4XmGrzvjaVBrCEqpQXVEYnzm12v50XPbR3SfxlBuzEEAcA12gjEmJiI3Ac8CTuABY8xmEfkWUG+MWQn8EnhYRHYATVhf8tjn/R7YAsSAzxljcnPBgR4J7jR9hVJqMJv3tRFPGNa+2zyi+zQFIzkxBwFSCAgAxpingad77ftGj9ddwJX9XHsncOcA914FrEqlHKPK7cW4PPhjmgJbKTW4N/a0APBOY4jGYHjYy18GQmGmVxWnsWTDl/1Gq1whghT5Ge/u4LDWEJRSg9jQ0IpDrNcjqSUEghH8OTDCCDQgHM1byTinJrhTSg1uQ0ML58yqpsDloH5X07DuEU8YmjtyI9MpaEA4mteHX/MZKaUG0doRZVegg7paP/Mml1M/zBpCS0eEhCFn+hA0IPTkraTC6KppSqmBbdjbAsC8mnLqav1s2ttKV3To42WSv3z6c2SUkQaEnor8lCbadJSRUmpAGxpaAZg3uYK6aT6icdO9byiS3zVVWkPIQd5KiuJtdIQjw4r2Sqn8sKGhhdpKL+VeN4un+QCof3fo/QhHMp1qDSH3eP0IhjJC2o+glOrXhoZW5tVUAOArLuA91cXU7xp6P0KyhpALie1AA8LR7NnKfmmnsV37EZRSxzrU3sX+1i7m1ZR376ub5mftu81DzpQcCEUQAZ9XA0LusfMZVaDpK5RSfduwx+ormD+lontfXa2P1s4obx8ODulegWAYn7cAZ3JCQ5ZpQOgpmfFU2nXlNKVUnzY0tOAQOHlSWfe+ulrru2Oow08DOZS2AjQgHM17JOOpDj1VSvXljYZWZo4rxVtwJPNPbaWXyuKCIfcjNIUiOZH2OkkDQk92H4I1W1lrCEqpoxlj2NDQclT/AYCIsHiab8gjjRpDw8+BNBo0IPRUUAIONxMLOjR9hVLqGA3NnTR3RJnXo/8g6b21ft4NdHB4CANSAsFIzowwAg0IRxMBbyXjXTrsVCl1rOTks/m9aggAi2ut+QhrU6wlROMJWjuj2mSU07x+Kh2hIUV5pVR+2NDQQoHTwYkTyo45dsqkcgpdjpT7EZqTaylrk1EO81Zai+RoDUEp1csbDS2cNLG0z+UuC1wO5tdUpDzSqDHH0laABoRjFfkoNW00hSJDmmRijMFaNVQpNRYlEoZNe9u6Zyj3ZXGtj017W+mMDJ76JjnXKZeajFJaMS2veCspjrUSTxhaO6P47A8rFk9wsD3MvpZO9jZ3srfF+pPc3tfSydTKYp68+cycmWSilEqfnY1BguHYMSOMenpvrY/7VhneaGjhtBmVA96vKQebjDQg9Ob1UxhrQ0jwpUffoK0zxt6WTg60dRHvVWPwFxcwqcLDjOpiZo4v4emNB/jHtkOcf+L4LBVeKTVa3uhjhnJvi6YmO5abBw0I3U1GOTTKSANCb95KHCbOJE+EN/e3M7miiCXT/UyuKGJSRRGTfUX2a89RE1Oi8QT1u57nV2ve1YCg1Bi0oaEFb4GT91SX9HtOhbeAmeNKUlpBLRAM43QIZR53Oos5IhoQerPzGf3vzQug8j0pX+Z2OvjEqdO4++/beKcxlDOLZiul0uONhlZOmVQ+aJNwXa2PpzbsJ5EwOAY4NzlLeaBzMk07lXuzZyvTMfTc5lefOgW3U3j45XfTXCilVDZF4wm27G8bsP8gafE0P21dMbYfGjjRXWOO5TECDQjHsvMZ0Tn0gDCu1MMlp0zkD2v3EArH0lwwpVS2bD3QTiSW6HOGcm/vrU1twZxAKJxTs5RBA8KxkgGhIzCsy5efMY32rhh/Xr83jYVSSmXTQDOUe5vq91JVUsjaQSaoNYUiVBbnzggj0IBwrKJkQBh6DQGsUQYnTyrjoTXv6rwEpcaIDQ0tVHjdTPV7Bz1XRKib5ht0glqu5TECDQjH8pSDOIddQxARlp9ey9aD7bzyzvCCilIqt7zR0MrcyeWIpNYBXFfrY3dTB4fauvo83hWNEwzHcirTKWhAOJaI1Ww0jD6EpMsWTKLC6+ahl3elr1xKqazojMTZdrCd+QPMUO5t8bRkP0LftYTkpLRcmqUMKQYEEblYRLaKyA4R+UofxwtFZIV9/BURqe1x7HZ7/1YR+YC9b4qIvCAiW0Rks4h8IW1PlA7eymHXEAA8bidX1U3h2c0HOdDa928ISqnjw5b9VuaCuSn0HySdPKkcj7v/RHfJ9VaOu1FGIuIE7gUuAeYAV4vInF6nXQc0G2NOAO4G7rKvnQMsA04GLgZ+at8vBvy7MWYOcBrwuT7umT1FfugY2spHvV1z2jQSxvDbV3QIqlLHs+4ZykOoISQT3fWXCrvRzmOUS2krILUawhJghzFmpzEmAjwCLO11zlLgV/brR4ELxGpsWwo8YowJG2PeAXYAS4wx+40x6wCMMe3Am8DkkT9Omnj9I6ohAEzxe7ngxHH89tXdhGODJ7pSSuWmDQ0tjCstZEK5Z0jX1dX62Lyvrc9Ed03Haw0B64t6T4/tBo798u4+xxgTA1qBylSutZuXFgKv9PXmInK9iNSLSP3hw4dTKG4ajLAPIena02tpDEZ4ZtOBNBRKKZUNG/a2DpjhtD910/zEEob1e1qOORboriEcfwFh1IhICfBH4BZjTFtf5xhj7jfG1Blj6qqrqzNTsGQfwgiHjZ55QhUzqor51Zpd6SmXUiqj2rqi7DwcSmn+QW/JRHd95TUKBCMUuByUFOZW9qBUAsJeYEqP7Rp7X5/niIgLKAcCA10rIm6sYPAbY8yfhlP4UVPkh0QMwu0juo3DIfzL6dNYt7uFjfbEFqXU8WOT/f82lRnKvZV73cweX9rnSKNAyEpbkeow1kxJJSC8BswUkekiUoDVSbyy1zkrgeX26yuA5401K2slsMwehTQdmAm8avcv/BJ40xjzX+l4kLTqzmc0sn4EgI8trsFb4NQhqEodh95IBoTJQ68hgLVgzrrdzccsthUI5l7aCkgh26kxJiYiNwHPAk7gAWPMZhH5FlBvjFmJ9eX+sIjsAJqwggb2eb8HtmCNLPqcMSYuImcC/wJsFJH19lt91RjzdJqfb3iKq6yf9ywEh9OaqNb90wEO17H7xAlTlsDl91n7bWUeNx9dNJnf1zfw1Q+e1L3gjlIq921oaGGq3zvs/7d103z89pXdbDvUftQ6zIEcTFsBKaa/tr+on+617xs9XncBV/Zz7Z3Anb32vQTkVl2pp+lnw/v/AyJBSMTBxO2fiV7bcUgkrJ9drbBhBUycD6d/7qjbXXt6Lb/+525W1O/hhnNST6mtlMquDQ2tLJhaMezr66ZZqXDqdzUfHRCCEU4YYF2FbMmtHo1c4S6CM28Z2jXGwO+WwXPfhtmXgH9G96FZ40s5bYafh19+l0+fNUOX2FTqONAYDLO3pZPlZ0wb9j2m+IuoLi2kflcT15xm3ccYk5OZTkFTV6SPCFx6NzjdsPLzx4xQWn56LXtbOnn+rUNZKqBSaig2NLQADGvIaZKI8N7aoxPddUTidEUTOTcpDTQgpFfZJLjo27BrNax98KhDF84Zz8Ryj3YuK5UBxhi2HRzZKMENDa2IwCnD7FBOWjzNT0NzJwftRHfJtBW5lscINCCk36LlVh/EX78OrQ3du11OB584dSqrtzfy9uGBV1JSSo3Mf6/eyUV3v8gf1zYMfnI/NjS0ckJ1yYjnCtQlE93ZeY2Sk9KqtMkoD4jAh++xOpqfvPWopqNlS6ZS4HToEptKjaKWjgg/eX4HAP/xxOZ+U1APxBjDhoaWETUXJc2ZVEaR29m9gtqRxHbaZJQf/NPh/K/D9r/Cht93764qKeRD8ybyx7UNBHWJTaVGxU9XvU17OMZ9n1hEOJbga3/eNOTFqva1dtEYjDB/ysiaiwDcTgfzp5QfU0PQJqN8cupnoGYJPHMbBI90JF97+jTawzEee12X2FQq3fa2dPLgml18dGENl8ydyL9fNIu/bTnIyjf2Dek+G+z8Q+moIQC8t9bPlv1thMIxAvZaCDrKKJ84nLD0JxAJwdNf6t69YEoF82rKeWjNLl1iU6k0+8FftwLwbxfNAuC6M2ewYEoFd6zczOH2cMr3eaOhFbdTOGliaVrKtXiaj3jC8MaeFgLBCN4CJ96C3Bv1rwFhNFXPhnO+DFv+DG8+AVjD0K49vZbth4K8vHPkqTGUUpYt+9p47PW9fPKMWiZXFAHgdAjfu2IeoXCcb67clPK9NjS0MHtCKYUu5+Anp2DRNB8i1gpqgWA4J5uLQAPC6HvfLTBhLjz179BptSFeOm8iPq+bh9YM3rkcTxj2t3ZSv6uJx9fv5Xev7iak/Q9KHeOuZ96izOPmxnNPOGr/zPGlfOH9M3l64wGe2rB/0PskEoaNDcNLed2fMs+RRHeBUCQn5yCAzlQefU43LL0X7j8Pnv0aXP5TPG4ny5ZM5ef/eJud9hDUvS2d7G3uZF9LJw32670tnRxo7SLWKzHWitf28OAn30uFNzd/y1Aq09bsaOQf2w7z1Q+eSLnXfczxz5w9g2c2HeAbj2/itBn+Ab+Q3wmEaA/HhpXyeiCLp/l4fP0+anxF3TWYXKMBIRMmzrdSYaz+AZzyUTjh/XziVCsgnP+Dfxx1qkNgQpmHyb4i6qb5mOwrYlKF9Q+oxlfEtoNBblmxnqt+/k8evm4J48qGtoqTUmNNImH4zjNvMancw7Wn1/Z5jsvp4HtXzuPDP36JO57Ywo+vXtjv/ZKp6tNZQwCrY/k3r+xm68F25qU52KSLBoRMOfvLVj/CE7fAjS9T4yvle1fMZ39rZ/cX/mRfERPKPLic/bfknTCulAqvm0//qp4rfvYyv77uVKZWejP3HErlmKc27mdDQyvfv3I+Hnf/bf4nTijj5vNn8l9/28al8ybygZMn9HneGw0teNwOZo5Lb/K5xfYENWPAn4NzEED7EDLH7bGajlob4O//AVhrJdx0/kw+uqiGU2dUUuPzDhgMks54TxW/+fRptHVFueJna9h6YGRT9JU6XkViCb737FZOnFDKRxYOviz7Z899D3MmlvG1xzbR0hHp85wNDa2cMqk8pf+LQ1HjK2J8mRUIcnGWMmhAyKwpS+DUG+C1/4Z314zoVgumVPCHz5yOCHz85y/z+u5jV2VSaqz77Svvsrupg9suOTGlLMJuu+mopSPCt57YcszxWDzB5n3p7VBOEpHudNi5OAcBNCBk3gVfh4pp8PhNEO0c0a1mji/l0RvOoMLr5hO/eIWXtjemqZBK5b72rij3PL+D02dUcu6s1NdbP3lSOTee+x7+9Ppenn/r4FHHth0M0hVNpGWGcl+SzUbaZKQsBcVw2T3Q9Das+n8jvt0Uv5c/fOZ0pvq9fOrB13hm04E0FFKp3Hf/iztpCkW4/YMnDnlt4pvOn8ns8aXc/qeNtHZGu/cnU17PHWGG0/5cfMoEzp5VPewlOUebdipnw4xzYdG1sObH1vrNhaXWEpzisJfldPRYmtNx9DGXB6a9D1xHqpzjyjysuP50Pvngq9z4m7Xc9bF5XFk3JXvPp9QoO9TWxS9Wv8Ol8yYOq3mnwGU1HX3kp2v4zye38L0r5wPWDOVSj4vayuI0l9gyqaKIhz61ZFTunQ4aELLlov+EXS/B374x+Lm9lU+BM2+FhdeAy6p6lnvd/Pr/nMpnHl7Llx7dQFtXjOvOnJ7mQis1cs9s2k9jMMJV752Ce5gdtz98bjvReIIvXjR72OWYV1PB9WfP4L5Vb/OheRM5d/Y4O8NpOY48XdVQA0K2eMrhxlegq+XIes2m57rNpte2fbx1L/zvD+Gpf4MXv28FhkXXgtuDt8DFL5bXceuK9Xz7yS20dkS49cJZQ65OKzUa2rqifPPxzd2JHX/9z3f51tJTWDLdP6T7vH04yIrX9nDNqVOprRrZb/JfuGAmf9tykNv/tJEnbj6TrQfa+fTZMwa/cIzSgJBNrgIoGTe0ayYthBM/BDtXwT++C3/5kjXh7X2fh8WfpLDAy4+vXkRp4UbueX4HrZ1Rvvnhk4/6jccYQ2c0TnNHlOZQhJaOKM0dEVo6IjR3RGnpiDJzfAkfnj9pxIuDKAXw2q4mbnlkPQfaurj1/bOYPaGEbz/5Jh//+ct8dOFkbv/gSVSXptbR+t1n3sLjcnDzBTNHXC6P28n3rpjHx+5bw6cfqieWMGmfoXw8keMp42ZdXZ2pr6/PdjFyyzur4R93Wct2FlfDGTdD3XWYgmL+31/e4v4Xd7JwagWFLkf3F39zR5RILNHvLT1uB13RBN4CJ5fNn8TVS6Yyr6ZcaxpqyKLxBPc8t517X9hBjc/LD5ctYNFUa6RNRyTGvS/s4P4Xd+JxOfn3i2ZxzWnTBhz/v/bdJj5238v824Wz+HwaAkLS/336Te5/cScAa75yPpNyNLXEcIjIWmNMXUrnakAYI959GV78Lrz9PBT54YybMO/9P/zi1QB/XNdAmcdNudeNz+vG5y2gwluAz+s+5me5102B08Hre1r43Su7eXLDfjqjceZMLOPqJVNYunAyZZ5jc8Uo1duuxhBfWLGeN/a0cMXiGu647OQ+a5w7Dwf55srNrN7eyJyJZXz78lO6h2f2ZIzhyp+9zLtNHfzjS+emNX10VzTOB3+0mmA4xitfvWBM/fKjASGf7XnNCgzb/wqeCjjtRli8HNxea3lPcQD2T5GjX3fvO6KtK8rj6/fxu1d2s2V/Gx63g0vnWbWGRVMrxtR/HJUexhj+UN/AHU9sxu108H8/MpcPzZs46DV/2XSAbz+5hf2tXXy8robbLj7xqCR0f918gOsfXsudHzmFT5w6Le3l3tvSSXMowik5OiR0uDQgKNi7zup03vrUMC4WKK+x+ismL4bJizAT57Ox0fC7V3ezcv0+QpE4s8eXsmzJFD66sKbPDJPq+NIYDFO/q5lxZYWcPKlsWGsBtHREuP1PG/nLpgOcPqOS/7pqPhPLU29+CYVj3PP8dn65+h28BU6+dPGJ/H9LpmKM4QM/fBFj4Nlbzx726KR8pAFBHXFgI7zzoj1KyQDmyGuTsLfN0ccScWjaCfvWQfMu+0YCVbNg8mLC4+ezKjiVn28tYt3eDgpdDi6cM57K4gIicUMkliAaP/InEjdEY7224wncTiuB2OwJpZw4oZRZ40up8RVprSNDOiNxXtvVxEs7Glm9vZE397d1HytwOjh5chkLp/hYNK2ChVN9TCr3DPjZrNnRyL/9/g0CoTBfvGg2nz5rxrCHb+441M7X/7yZl3cGmDu5nNNm+Pnv1e/ws2sWcfEpA9c21NE0IKj0CQVg3+uwd60VIPauhdBh65izgA7/SWwwM/hb0wTaTSEuhwOXQ3A4BKfDgcvpwOkQXPa2M7nfIYTiTja2etjY5iVAGXGclBS6mDXeChKzx5cye0IZsyeUDmuFKWMMsYTB5ZDjNshEYgkOtHaxt6WT1s4o1aWFTCz3MK60cMjJ1xIJw+Z9bazecZiXtjdS/24zkVgCt1NYPM3HWTOrOW2Gn8PtYV7f3cK63c1saGglbA9AGF9WyMIpPhZOrWDRNB9zJ5fjcTsJx+L84K/buP/FnbynupgfLVuYlmYXYwxPbNjPfz65hUPtYRZNreCPnz3juP0ssyXtAUFELgZ+BDiBXxhjvtPreCHwELAYCABXGWN22cduB64D4sDnjTHPpnLPvmhAyAHGWBlbuwPEOti3HiIjy7hqxEFXgZ9mRyX7ExXsCpewJ1bOIePjoKkg4h1PxbgpuL1lRGIJwjFDOBonHDeEYwnrTzRBJG697rLPMQgOl5vK4kL8JQX4iwvxe934iwupLCnA5y3AX1xAZYn10+8toLjQRWc0TigcoyMSIxS2XociyZ8xOsJxgsnjkTgClBW5KfW4KPW4KfO4KPMc2bZ+uigucHX/1myMoSkUYV+L9YW/L/mntZO9LV3sa+mkMRimr/+iDoHq0kImlHmYUO5hYnkRE8o9PbY9jC/zcLg9zEs7GnlpeyNr3m6kucNK03DihFLOPKGKM2dWsWS6v98O2mg8wVv721m3u5nXdzfz+p4W3g10AOByCHMmldEZibP9UJB/OW0aX/3gSRQVpGfZyaT2rii/e3U37z9pPDOq05uSOh+kNSCIiBPYBlwINACvAVcbY7b0OOdGYJ4x5gYRWQZ8xBhzlYjMAX4HLAEmAX8HZtmXDXjPvmhAyFGJBLTsgridE6b731SPf1t97Yt2QfAAtCf/7IfgQWjfj2k/CKHDCCOvwSZwEnYU0SlFhCgkmPDQliikLVFIBx5CppAQRd2vOynEcOxvocl9PY+5nA4KXQ5ixkF7VAgnHERwEcNJFJf1xziJ2a/j4sRdUIi7oJDmjlj38N/kPQvcDsaXWV/s48uKGFdWyPgy68u+1OMmEIpxoD3KwWCEA+0R9rdFOdAWYV9bhLZwgjgOEtiDBXoYV1rImTOrOGtmFe87oYpxpcNfWKkxGGa9XYN4fXcLgVCY2y4+kQtOGj/se6rRM5SAkMq4rSXADmPMTvvmjwBLgZ5f3kuBO+zXjwI/EatetxR4xBgTBt4RkR32/Ujhnup44XCAP72zOwWsABM81CNo7D+SIbZ3gOlv2yRwRDspioQoirTjj4QgEoJwkEQ4SCK8HxMOItEOXLHQ8AqbnNIxlFGQEfv8vq4J2X8GX/73CAF6fMcbhIQ9okzEgSRAtgJbkyf3vLZ38JNeI9Kkxz6hCuH9Iry/5/GnBJ4a5J7HlLlH4Oo+X3qdLsceG4rBytFn89MQ32dYTVhDvMZbCZ/6yzDeZ2hS+Sc8GdjTY7sBOLW/c4wxMRFpBSrt/f/sdW1yFYvB7gmAiFwPXA8wderUFIqrxgynG8onW39GgYNe6X4TCYh29EhLPkgNp+e+RBwSUSuIxaMQj0AiZv1M7kvY++P2/v6CWb/7EkcGBCTifac2SVjniInjTJ7f0zEtAqaP4+aogHrUvp4DEXoOUBj0nr2ZHqcNFNR7HxuKwcrRxz2H/D7DKNdwnsVTNvRrhiHn8xIYY+4H7gerySjLxVFjmcMBhSXWH6XyUCrDFPYCPXMp19j7+jxHRFxAOVbncn/XpnJPpZRSGZRKQHgNmCki00WkAFgGrOx1zkpguf36CuB5Y/VWrwSWiUihiEwHZgKvpnhPpZRSGTRok5HdJ3AT8CzWENEHjDGbReRbQL0xZiXwS+Bhu9O4CesLHvu832N1FseAzxlj4gB93TP9j6eUUipVOjFNKaXGsKEMO9WEIEoppQANCEoppWwaEJRSSgEaEJRSStmOq05lETkMvDvMy6uAxjQW53iSz88O+f38+uz5K/n804wx1alccFwFhJEQkfpUe9rHmnx+dsjv59dnz89nh+E9vzYZKaWUAjQgKKWUsuVTQLg/2wXIonx+dsjv59dnz19Dfv686UNQSik1sHyqISillBqABgSllFJAHgQEEblYRLaKyA4R+Uq2y5NpIrJLRDaKyHoRGdOZAUXkARE5JCKbeuzzi8jfRGS7/dOXzTKOpn6e/w4R2Wt//utF5IPZLONoEZEpIvKCiGwRkc0i8gV7/5j//Ad49iF/9mO6D0FEnMA24EKsZTpfA642xuTN2s0isguoM8aM+Qk6InI2EAQeMsacYu/7LtBkjPmO/QuBzxhzWzbLOVr6ef47gKAx5vvZLNtoE5GJwERjzDoRKQXWApcD/8oY//wHePaPM8TPfqzXEJYAO4wxO40xEeARYGmWy6RGiTHmRaz1OHpaCvzKfv0rrP8oY1I/z58XjDH7jTHr7NftwJtY67eP+c9/gGcfsrEeECYDe3psNzDMv6jjmAH+KiJrReT6bBcmC8YbY/bbrw8A47NZmCy5SUQ22E1KY67JpDcRqQUWAq+QZ59/r2eHIX72Yz0gKDjTGLMIuAT4nN2skJfsZV3Hbhtp3+4D3gMsAPYDP8hqaUaZiJQAfwRuMca09Tw21j//Pp59yJ/9WA8Ie4EpPbZr7H15wxiz1/55CHgMqxktnxy021iTba2HslyejDLGHDTGxI0xCeC/GcOfv4i4sb4Qf2OM+ZO9Oy8+/76efTif/VgPCK8BM0VkuogUYK31vDLLZcoYESm2O5kQkWLgImDTwFeNOSuB5fbr5cDjWSxLxiW/DG0fYYx+/iIiWGu7v2mM+a8eh8b859/fsw/nsx/To4wA7KFWPwScwAPGmDuzW6LMEZEZWLUCABfw27H8/CLyO+BcrLS/B4FvAn8Gfg9MxUqd/nFjzJjseO3n+c/FajIwwC7gMz3a1McMETkTWA1sBBL27q9itaWP6c9/gGe/miF+9mM+ICillErNWG8yUkoplSINCEoppQANCEoppWwaEJRSSgEaEJRSStk0ICillAI0ICillLL9/5VTPU1hQAChAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -1955,11 +2096,14 @@ "\n", "\n", "np.random.seed(2018)\n", - "n = 50\n", - "maxdegree = 5\n", + "n = 100\n", + "maxdegree = 25\n", "# Make data set.\n", - "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", - "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "x = np.linspace(-1, 1, n).reshape(-1, 1)\n", + "#y = 1./(1+25*x*x)\n", + "y = (np.tanh(9*x)+1)/9.\n", + "#y = (1./3.)*np.exp(-(x-0.5)**2)\n", + "#y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", "TestError = np.zeros(maxdegree)\n", "TrainError = np.zeros(maxdegree)\n", "polydegree = np.zeros(maxdegree)\n", @@ -1987,9 +2131,7 @@ { "cell_type": "markdown", "id": "dbabb429", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Mathematical Interpretation of Ordinary Least Squares\n", "\n", @@ -2001,9 +2143,7 @@ { "cell_type": "markdown", "id": "fd4d35d3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -2013,9 +2153,7 @@ { "cell_type": "markdown", "id": "72475f65", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The **hat** over $\\boldsymbol{\\theta}$ means we have the optimal parameters after minimization of the cost function.\n", "\n", @@ -2025,9 +2163,7 @@ { "cell_type": "markdown", "id": "fe531c24", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}=\\boldsymbol{X}\\hat{\\boldsymbol{\\theta}} = \\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -2037,9 +2173,7 @@ { "cell_type": "markdown", "id": "a3430e87", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We now define a matrix" ] @@ -2047,9 +2181,7 @@ { "cell_type": "markdown", "id": "02ea8f17", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{A}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T.\n", @@ -2059,9 +2191,7 @@ { "cell_type": "markdown", "id": "cba7fe69", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can rewrite" ] @@ -2069,9 +2199,7 @@ { "cell_type": "markdown", "id": "68a5ef20", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}=\\boldsymbol{X}\\hat{\\boldsymbol{\\theta}} = \\boldsymbol{A}\\boldsymbol{y}.\n", @@ -2081,9 +2209,7 @@ { "cell_type": "markdown", "id": "e0e90274", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{A}$ has the important property that $\\boldsymbol{A}^2=\\boldsymbol{A}$. This is the definition of a projection matrix.\n", "We can then interpret our optimal model $\\tilde{\\boldsymbol{y}}$ as being represented by an orthogonal projection of $\\boldsymbol{y}$ onto a space defined by the column vectors of $\\boldsymbol{X}$. In our case here the matrix $\\boldsymbol{A}$ is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix." @@ -2092,9 +2218,7 @@ { "cell_type": "markdown", "id": "bcc38362", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Residual Error\n", "\n", @@ -2104,9 +2228,7 @@ { "cell_type": "markdown", "id": "1f2209b1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon}=\\boldsymbol{y}-\\tilde{\\boldsymbol{y}}=\\left[\\boldsymbol{I}-\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\right]\\boldsymbol{y}.\n", @@ -2116,9 +2238,7 @@ { "cell_type": "markdown", "id": "5b38d0d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The residual errors are then the projections of $\\boldsymbol{y}$ onto the orthogonal component of the space defined by the column vectors of $\\boldsymbol{X}$." ] @@ -2126,9 +2246,7 @@ { "cell_type": "markdown", "id": "381bb9f1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple case\n", "\n", @@ -2138,9 +2256,7 @@ { "cell_type": "markdown", "id": "8c247e6e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{X}\\boldsymbol{X}^T = \\boldsymbol{I}.\n", @@ -2150,9 +2266,7 @@ { "cell_type": "markdown", "id": "cb0b8048", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the matrix $\\boldsymbol{A}$ becomes" ] @@ -2160,9 +2274,7 @@ { "cell_type": "markdown", "id": "9dbf0514", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{A}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T)=\\boldsymbol{I},\n", @@ -2172,9 +2284,7 @@ { "cell_type": "markdown", "id": "d5d0f41c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and we have the obvious case" ] @@ -2182,9 +2292,7 @@ { "cell_type": "markdown", "id": "a8f9a174", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\epsilon}=\\boldsymbol{y}-\\tilde{\\boldsymbol{y}}=0.\n", @@ -2194,9 +2302,7 @@ { "cell_type": "markdown", "id": "f3ff3d0d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This serves also as a useful test of our codes." ] @@ -2204,9 +2310,7 @@ { "cell_type": "markdown", "id": "b1cfea2c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The singular value decomposition\n", "\n", @@ -2244,9 +2348,7 @@ { "cell_type": "markdown", "id": "02c449e0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linear Regression Problems\n", "\n", @@ -2261,9 +2363,7 @@ { "cell_type": "markdown", "id": "7ccce190", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -2284,9 +2384,7 @@ { "cell_type": "markdown", "id": "5cfffb71", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The columns of $\\boldsymbol{X}$ are linearly dependent. We see this easily since the \n", "the first column is the row-wise sum of the other two columns. The rank (more correct,\n", @@ -2301,9 +2399,7 @@ { "cell_type": "markdown", "id": "8c175ba9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\begin{align*}\n", @@ -2320,9 +2416,7 @@ { "cell_type": "markdown", "id": "3733595c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see easily that $\\mbox{det}(\\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \\times (-1) - 1 \\times (-1) = 0$. Hence, $\\mathbf{X}$ is singular and its inverse is undefined.\n", "This is equivalent to saying that the matrix $\\boldsymbol{X}$ has at least an eigenvalue which is zero." @@ -2331,9 +2425,7 @@ { "cell_type": "markdown", "id": "3d6b26a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Fixing the singularity\n", "\n", @@ -2343,9 +2435,7 @@ { "cell_type": "markdown", "id": "1c19da0b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", "
\n", @@ -2361,9 +2451,7 @@ { "cell_type": "markdown", "id": "97d8029f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "has linearly dependent column vectors, we will not be able to compute the inverse\n", "of $\\boldsymbol{X}^T\\boldsymbol{X}$ and we cannot find the parameters (estimators) $\\theta_i$. \n", @@ -2377,9 +2465,7 @@ { "cell_type": "markdown", "id": "176790e6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^{T} \\boldsymbol{X} \\rightarrow \\boldsymbol{X}^{T} \\boldsymbol{X}+\\lambda \\boldsymbol{I},\n", @@ -2389,9 +2475,7 @@ { "cell_type": "markdown", "id": "5b10a10c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{I}$ is the identity matrix. When we discuss **Ridge** regression this is actually what we end up evaluating. The parameter $\\lambda$ is called a hyperparameter. More about this later." ] @@ -2399,9 +2483,7 @@ { "cell_type": "markdown", "id": "fd6aaf02", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ridge and LASSO Regression\n", "\n", @@ -2412,9 +2494,7 @@ { "cell_type": "markdown", "id": "73287deb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right\\}.\n", @@ -2424,9 +2504,7 @@ { "cell_type": "markdown", "id": "a2748589", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or we can state it as" ] @@ -2434,9 +2512,7 @@ { "cell_type": "markdown", "id": "bf616cae", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -2447,9 +2523,7 @@ { "cell_type": "markdown", "id": "268336e2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have used the definition of a norm-2 vector, that is" ] @@ -2457,9 +2531,7 @@ { "cell_type": "markdown", "id": "b3b06dab", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", @@ -2469,9 +2541,7 @@ { "cell_type": "markdown", "id": "c35ba695", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "By minimizing the above equation with respect to the parameters\n", "$\\boldsymbol{\\theta}$ we could then obtain an analytical expression for the\n", @@ -2482,9 +2552,7 @@ { "cell_type": "markdown", "id": "907b685c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -2495,9 +2563,7 @@ { "cell_type": "markdown", "id": "b7739af7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to the Ridge regression minimization problem where we\n", "require that $\\vert\\vert \\boldsymbol{\\theta}\\vert\\vert_2^2\\le t$, where $t$ is\n", @@ -2507,9 +2573,7 @@ { "cell_type": "markdown", "id": "28f1e119", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\theta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\theta}\\vert\\vert_1,\n", @@ -2519,9 +2583,7 @@ { "cell_type": "markdown", "id": "b2eb3706", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have a new optimization equation" ] @@ -2529,9 +2591,7 @@ { "cell_type": "markdown", "id": "3d6e929d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -2542,9 +2602,7 @@ { "cell_type": "markdown", "id": "506c9ea1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", "\n", @@ -2554,9 +2612,7 @@ { "cell_type": "markdown", "id": "1f8a7545", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", @@ -2566,9 +2622,7 @@ { "cell_type": "markdown", "id": "96e67a6c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Ridge Regression Equations\n", "\n", @@ -2578,9 +2632,7 @@ { "cell_type": "markdown", "id": "a8e5496a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\theta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta})\\right\\}+\\lambda\\boldsymbol{\\theta}^T\\boldsymbol{\\theta},\n", @@ -2590,9 +2642,7 @@ { "cell_type": "markdown", "id": "af2cdec9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and \n", "taking the derivatives with respect to $\\boldsymbol{\\theta}$ we obtain then\n", @@ -2604,9 +2654,7 @@ { "cell_type": "markdown", "id": "d5d7fa3b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -2616,9 +2664,7 @@ { "cell_type": "markdown", "id": "7475fc98", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that" ] @@ -2626,9 +2672,7 @@ { "cell_type": "markdown", "id": "6256388d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i=0}^{p-1} \\theta_i^2 \\leq t,\n", @@ -2638,9 +2682,7 @@ { "cell_type": "markdown", "id": "f86d8ef4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $t$ a finite positive number. \n", "\n", @@ -2650,9 +2692,7 @@ { "cell_type": "markdown", "id": "a21b725b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+n\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -2662,9 +2702,7 @@ { "cell_type": "markdown", "id": "c12315f7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In many textbooks the $1/n$ term is often omitted. Note that a library like **Scikit-Learn** does not include the $1/n$ factor in the setup of the cost function.\n", "\n", @@ -2674,9 +2712,7 @@ { "cell_type": "markdown", "id": "da1bdc50", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -2686,9 +2722,7 @@ { "cell_type": "markdown", "id": "02beeb43", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n", "\n", @@ -2709,10 +2743,7 @@ "cell_type": "code", "execution_count": 12, "id": "f1d32998", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -2785,9 +2816,7 @@ { "cell_type": "markdown", "id": "1b7bf147", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", @@ -2797,9 +2826,7 @@ { "cell_type": "markdown", "id": "91e26b57", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Basic math of the SVD\n", "\n", @@ -2812,9 +2839,7 @@ { "cell_type": "markdown", "id": "4e52a538", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\lambda_1,\\boldsymbol{u}_1),\\dots, (\\lambda_n,\\boldsymbol{u}_n),\n", @@ -2824,9 +2849,7 @@ { "cell_type": "markdown", "id": "f296d012", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the eigenvalues are given by the diagonal matrix" ] @@ -2834,9 +2857,7 @@ { "cell_type": "markdown", "id": "eb96aff5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\\mathrm{Diag}(\\lambda_1, \\dots,\\lambda_n).\n", @@ -2846,9 +2867,7 @@ { "cell_type": "markdown", "id": "095933fc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{X}$ can be written in terms of an orthogonal/unitary transformation $\\boldsymbol{U}$" ] @@ -2856,9 +2875,7 @@ { "cell_type": "markdown", "id": "485459fb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -2868,9 +2885,7 @@ { "cell_type": "markdown", "id": "dc378439", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{I}$ or $\\boldsymbol{U}\\boldsymbol{U}^{\\dagger}=\\boldsymbol{I}$.\n", "\n", @@ -2880,9 +2895,7 @@ { "cell_type": "markdown", "id": "c10033aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\begin{bmatrix} \n", @@ -2895,9 +2908,7 @@ { "cell_type": "markdown", "id": "d710a13a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is not diagonalizable, it is a so-called [defective matrix](https://en.wikipedia.org/wiki/Defective_matrix). It is easy to see that the condition\n", "$\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{X}^T\\boldsymbol{X}$ is not fulfilled." @@ -2906,9 +2917,7 @@ { "cell_type": "markdown", "id": "1463c042", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The SVD, a Fantastic Algorithm\n", "\n", @@ -2926,9 +2935,7 @@ { "cell_type": "markdown", "id": "0f238692", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\n", @@ -2938,9 +2945,7 @@ { "cell_type": "markdown", "id": "79337cdd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "As an example, the above defective matrix can be decomposed as" ] @@ -2948,9 +2953,7 @@ { "cell_type": "markdown", "id": "43d19beb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X} = \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& 1 \\\\ 1& -1\\\\ \\end{bmatrix} \\begin{bmatrix} 2& 0 \\\\ 0& 0\\\\ \\end{bmatrix} \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& -1 \\\\ 1& 1\\\\ \\end{bmatrix}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -2960,9 +2963,7 @@ { "cell_type": "markdown", "id": "fc9ba98c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with eigenvalues $\\sigma_1=2$ and $\\sigma_2=0$. \n", "The SVD exits always! \n", @@ -2989,9 +2990,7 @@ { "cell_type": "markdown", "id": "e1c49876", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Economy-size SVD\n", "\n", @@ -3016,9 +3015,7 @@ { "cell_type": "markdown", "id": "ab5ab11f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Codes for the SVD" ] @@ -3027,10 +3024,7 @@ "cell_type": "code", "execution_count": 13, "id": "2cd13083", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -3067,9 +3061,7 @@ { "cell_type": "markdown", "id": "eabd32cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The matrix $\\boldsymbol{X}$ has columns that are linearly dependent. The first\n", "column is the row-wise sum of the other two columns. The rank of a\n", @@ -3084,9 +3076,7 @@ { "cell_type": "markdown", "id": "d7f8324f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Note about SVD Calculations\n", "\n", @@ -3107,9 +3097,7 @@ { "cell_type": "markdown", "id": "d3ab6303", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Mathematics of the SVD and implications\n", "\n", @@ -3121,9 +3109,7 @@ { "cell_type": "markdown", "id": "e325098f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -3140,9 +3126,7 @@ { "cell_type": "markdown", "id": "4a3be35c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can SVD decompose our matrix as" ] @@ -3150,9 +3134,7 @@ { "cell_type": "markdown", "id": "e6385438", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -3162,9 +3144,7 @@ { "cell_type": "markdown", "id": "fd0d8edb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{U}$ is an orthogonal matrix of dimension $n\\times n$, meaning that $\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{U}^T\\boldsymbol{U}=\\boldsymbol{I}_n$. Here $\\boldsymbol{I}_n$ is the unit matrix of dimension $n \\times n$.\n", "\n", @@ -3176,9 +3156,7 @@ { "cell_type": "markdown", "id": "c743c53c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sigma_0 > \\sigma_1 > \\sigma_2 > \\dots > \\sigma_{p-1} > 0.\n", @@ -3188,9 +3166,7 @@ { "cell_type": "markdown", "id": "c85f493d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "All values beyond $p-1$ are all zero." ] @@ -3198,9 +3174,7 @@ { "cell_type": "markdown", "id": "040c3b45", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Example Matrix\n", "\n", @@ -3210,9 +3184,7 @@ { "cell_type": "markdown", "id": "1909dccd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\n", @@ -3227,9 +3199,7 @@ { "cell_type": "markdown", "id": "49352aa0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The singular values are $\\sigma_0=2$ and $\\sigma_1=1$. It is common to rewrite the matrix $\\boldsymbol{\\Sigma}$ as" ] @@ -3237,9 +3207,7 @@ { "cell_type": "markdown", "id": "13063947", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}=\n", @@ -3253,9 +3221,7 @@ { "cell_type": "markdown", "id": "9764fa95", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where" ] @@ -3263,9 +3229,7 @@ { "cell_type": "markdown", "id": "be4c1a31", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\tilde{\\Sigma}}=\n", @@ -3279,9 +3243,7 @@ { "cell_type": "markdown", "id": "956fd3f2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "contains only the singular values. Note also (and we will use this below) that" ] @@ -3289,9 +3251,7 @@ { "cell_type": "markdown", "id": "0d96a41b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}=\n", @@ -3305,9 +3265,7 @@ { "cell_type": "markdown", "id": "1f9345dd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is a $2\\times 2 $ matrix while" ] @@ -3315,9 +3273,7 @@ { "cell_type": "markdown", "id": "c28e9976", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T=\n", @@ -3332,9 +3288,7 @@ { "cell_type": "markdown", "id": "59da3beb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is a $3\\times 3 $ matrix. The last row and column of this last matrix\n", "contain only zeros. This will have important consequences for our SVD\n", @@ -3344,9 +3298,7 @@ { "cell_type": "markdown", "id": "8eb5c7ad", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Setting up the Matrix to be inverted\n", "\n", @@ -3356,9 +3308,7 @@ { "cell_type": "markdown", "id": "ef46c055", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", @@ -3368,9 +3318,7 @@ { "cell_type": "markdown", "id": "9b25659c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using the orthogonality of the matrix $\\boldsymbol{U}$ we have" ] @@ -3378,9 +3326,7 @@ { "cell_type": "markdown", "id": "010f41c8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -3390,9 +3336,7 @@ { "cell_type": "markdown", "id": "06758e16", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We define $\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}=\\tilde{\\boldsymbol{\\Sigma}}^2$ which is a diagonal matrix containing only the singular values squared. It has dimensionality $p \\times p$.\n", "\n", @@ -3402,9 +3346,7 @@ { "cell_type": "markdown", "id": "bdbf298a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", @@ -3414,9 +3356,7 @@ { "cell_type": "markdown", "id": "a4272077", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and using our SVD decomposition of $\\boldsymbol{X}$ we have" ] @@ -3424,9 +3364,7 @@ { "cell_type": "markdown", "id": "cb8c73ef", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\left(\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^{2}(\\boldsymbol{V}^T\\right)^{-1}\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{y},\n", @@ -3436,9 +3374,7 @@ { "cell_type": "markdown", "id": "8967ae8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which gives us, using the orthogonality of the matrix $\\boldsymbol{V}$," ] @@ -3446,9 +3382,7 @@ { "cell_type": "markdown", "id": "fb30d808", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{y}_{\\mathrm{OLS}}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_i\\boldsymbol{u}^T_i\\boldsymbol{y},\n", @@ -3458,9 +3392,7 @@ { "cell_type": "markdown", "id": "53969156", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is not the same as $\\tilde{y}_{\\mathrm{OLS}}=\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}$, which due to the orthogonality of $\\boldsymbol{U}$ would have given us that the model equals the output.\n", "\n", @@ -3475,9 +3407,7 @@ { "cell_type": "markdown", "id": "763f5e94", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Further properties (important for our analyses later)\n", "\n", @@ -3487,9 +3417,7 @@ { "cell_type": "markdown", "id": "af22af28", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -3499,9 +3427,7 @@ { "cell_type": "markdown", "id": "63027cc0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we now multiply from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" ] @@ -3509,9 +3435,7 @@ { "cell_type": "markdown", "id": "6d4f7a03", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}.\n", @@ -3521,9 +3445,7 @@ { "cell_type": "markdown", "id": "7fcfc0cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This means the vectors $\\boldsymbol{v}_i$ of the orthogonal matrix $\\boldsymbol{V}$ are the eigenvectors of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$\n", "with eigenvalues given by the singular values squared, that is" @@ -3532,9 +3454,7 @@ { "cell_type": "markdown", "id": "f5d44c36", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", @@ -3544,9 +3464,7 @@ { "cell_type": "markdown", "id": "623c17e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Similarly, if we use the SVD decomposition for the matrix $\\boldsymbol{X}\\boldsymbol{X}^T$, we have" ] @@ -3554,9 +3472,7 @@ { "cell_type": "markdown", "id": "883abde1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T.\n", @@ -3566,9 +3482,7 @@ { "cell_type": "markdown", "id": "7393a1b9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we now multiply from the right with $\\boldsymbol{U}$ (using the orthogonality of $\\boldsymbol{U}$) we get" ] @@ -3576,9 +3490,7 @@ { "cell_type": "markdown", "id": "4dd69c12", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}\\boldsymbol{X}^T\\right)\\boldsymbol{U}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T.\n", @@ -3588,9 +3500,7 @@ { "cell_type": "markdown", "id": "e71fd75d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This means the vectors $\\boldsymbol{u}_i$ of the orthogonal matrix $\\boldsymbol{U}$ are the eigenvectors of the matrix $\\boldsymbol{X}\\boldsymbol{X}^T$\n", "with eigenvalues given by the singular values squared, that is" @@ -3599,9 +3509,7 @@ { "cell_type": "markdown", "id": "4dabe6e8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}\\boldsymbol{X}^T\\right)\\boldsymbol{u}_i=\\boldsymbol{u}_i\\sigma_i^2.\n", @@ -3611,9 +3519,7 @@ { "cell_type": "markdown", "id": "96201b0a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "**Important note**: we have defined our design matrix $\\boldsymbol{X}$ to be an\n", "$n\\times p$ matrix. In most supervised learning cases we have that $n\n", @@ -3629,9 +3535,7 @@ { "cell_type": "markdown", "id": "940c56fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Meet the Covariance Matrix\n", "\n", @@ -3645,9 +3549,7 @@ { "cell_type": "markdown", "id": "ea6a1fb1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial^2 C(\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}\\partial \\boldsymbol{\\theta}^T} =\\frac{2}{n}\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -3657,9 +3559,7 @@ { "cell_type": "markdown", "id": "991316eb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).\n", "\n", @@ -3669,9 +3569,7 @@ { "cell_type": "markdown", "id": "93aa9e83", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{H}=\\boldsymbol{X}^T\\boldsymbol{X}.\n", @@ -3681,9 +3579,7 @@ { "cell_type": "markdown", "id": "6a6c2628", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The Hessian matrix for ordinary least squares is also proportional to\n", "the covariance matrix. This means also that we can use the SVD to find\n", @@ -3694,9 +3590,7 @@ { "cell_type": "markdown", "id": "db4254c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Introducing the Covariance and Correlation functions\n", "\n", @@ -3710,9 +3604,7 @@ { "cell_type": "markdown", "id": "251c4816", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -3724,9 +3616,7 @@ { "cell_type": "markdown", "id": "3726ee5e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where for example" ] @@ -3734,9 +3624,7 @@ { "cell_type": "markdown", "id": "e0f575a7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", @@ -3746,9 +3634,7 @@ { "cell_type": "markdown", "id": "e548131c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "With this definition and recalling that the variance is defined as" ] @@ -3756,9 +3642,7 @@ { "cell_type": "markdown", "id": "d60225e8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{var}[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", @@ -3768,9 +3652,7 @@ { "cell_type": "markdown", "id": "89d9aae5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we can rewrite the covariance matrix as" ] @@ -3778,9 +3660,7 @@ { "cell_type": "markdown", "id": "5f4ae97f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -3792,9 +3672,7 @@ { "cell_type": "markdown", "id": "76b9235f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "**Note:** we have used $1/n$ in the above definitions of the *sample* variance and covariance. We assume then that we can calculate the exact mean value. \n", "What you will find in essentially all statistics texts are equations\n", @@ -3809,9 +3687,7 @@ { "cell_type": "markdown", "id": "290a9403", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Covariance and Correlation Matrix\n", "\n", @@ -3825,9 +3701,7 @@ { "cell_type": "markdown", "id": "832b31ab", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{\\mathrm{var}[\\boldsymbol{x}] \\mathrm{var}[\\boldsymbol{y}]}}.\n", @@ -3837,9 +3711,7 @@ { "cell_type": "markdown", "id": "10a81bbb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The correlation function is then given by values $\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]\n", "\\in [-1,1]$. This avoids eventual problems with too large values. We\n", @@ -3850,9 +3722,7 @@ { "cell_type": "markdown", "id": "1dd8962b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & \\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -3864,9 +3734,7 @@ { "cell_type": "markdown", "id": "47f1db97", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the above example this is the function we constructed using **pandas**." ] @@ -3874,9 +3742,7 @@ { "cell_type": "markdown", "id": "833510b2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Function and Design/Feature Matrix\n", "\n", @@ -3887,9 +3753,7 @@ { "cell_type": "markdown", "id": "4f43985b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -3906,9 +3770,7 @@ { "cell_type": "markdown", "id": "81c592cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors/features $p$ refering to the column numbers and the\n", "entries $n$ being the row elements.\n", @@ -3918,9 +3780,7 @@ { "cell_type": "markdown", "id": "d1a2a046", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix} \\boldsymbol{x}_0 & \\boldsymbol{x}_1 & \\boldsymbol{x}_2 & \\dots & \\dots & \\boldsymbol{x}_{p-1}\\end{bmatrix},\n", @@ -3930,9 +3790,7 @@ { "cell_type": "markdown", "id": "e59142de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with a given vector" ] @@ -3940,9 +3798,7 @@ { "cell_type": "markdown", "id": "4729dbde", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{x}_i^T = \\begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \\dots & \\dots x_{n-1,i}\\end{bmatrix}.\n", @@ -3952,9 +3808,7 @@ { "cell_type": "markdown", "id": "0cbc86c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "With these definitions, we can now rewrite our $2\\times 2$\n", "correlation/covariance matrix in terms of a moe general design/feature\n", @@ -3965,9 +3819,7 @@ { "cell_type": "markdown", "id": "ca29148b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -3984,9 +3836,7 @@ { "cell_type": "markdown", "id": "660a2ac1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the correlation matrix" ] @@ -3994,9 +3844,7 @@ { "cell_type": "markdown", "id": "5a435a0a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -4013,9 +3861,7 @@ { "cell_type": "markdown", "id": "764e2e96", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Covariance Matrix Examples\n", "\n", @@ -4031,9 +3877,7 @@ { "cell_type": "markdown", "id": "d1789d6c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{W} = \\begin{bmatrix} x_0 & x_1 & x_2 & \\dots & x_{n-2} & x_{n-1} \\\\\n", @@ -4045,9 +3889,7 @@ { "cell_type": "markdown", "id": "c6d6fec5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which in turn is converted into into the $2\\times 2$ covariance matrix\n", "$\\boldsymbol{C}$ via the Numpy function **np.cov()**. We note that we can also calculate\n", @@ -4060,10 +3902,7 @@ "cell_type": "code", "execution_count": 14, "id": "e6914cd6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Importing various packages\n", @@ -4081,9 +3920,7 @@ { "cell_type": "markdown", "id": "3a53a2a1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Matrix\n", "\n", @@ -4098,10 +3935,7 @@ "cell_type": "code", "execution_count": 15, "id": "ec053f1e", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -4130,9 +3964,7 @@ { "cell_type": "markdown", "id": "521e11ff", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see that the matrix elements along the diagonal are one as they\n", "should be and that the matrix is symmetric. Furthermore, diagonalizing\n", @@ -4144,9 +3976,7 @@ { "cell_type": "markdown", "id": "0aef5f95", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Correlation Matrix with Pandas\n", "\n", @@ -4157,10 +3987,7 @@ "cell_type": "code", "execution_count": 16, "id": "3dbd8c2a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -4182,9 +4009,7 @@ { "cell_type": "markdown", "id": "45dc0cd3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Rewriting the Covariance and/or Correlation Matrix\n", "\n", @@ -4194,9 +4019,7 @@ { "cell_type": "markdown", "id": "85285784", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}= \\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}].\n", @@ -4206,9 +4029,7 @@ { "cell_type": "markdown", "id": "1acb278d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "To see this let us simply look at a design matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{2\\times 2}$" ] @@ -4216,9 +4037,7 @@ { "cell_type": "markdown", "id": "061cf852", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -4233,9 +4052,7 @@ { "cell_type": "markdown", "id": "255b8070", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we then compute the expectation value (note the $1/n$ factor instead of $1/(n-1)$)" ] @@ -4243,9 +4060,7 @@ { "cell_type": "markdown", "id": "cfff2c26", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}=\\frac{1}{n}\\begin{bmatrix}\n", @@ -4258,9 +4073,7 @@ { "cell_type": "markdown", "id": "2d8af4a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is just" ] @@ -4268,9 +4081,7 @@ { "cell_type": "markdown", "id": "333e3603", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]=\\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] \\\\\n", @@ -4282,9 +4093,7 @@ { "cell_type": "markdown", "id": "e2eb99a0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we wrote $$\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]$$ to indicate that this is the covariance of the vectors $\\boldsymbol{x}$ of the design/feature matrix $\\boldsymbol{X}$.\n", "\n", @@ -4294,9 +4103,7 @@ { "cell_type": "markdown", "id": "e4d380d0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Linking with the SVD\n", "\n", @@ -4306,9 +4113,7 @@ { "cell_type": "markdown", "id": "28a2dd25", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", @@ -4318,9 +4123,7 @@ { "cell_type": "markdown", "id": "cceb8b2c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $p\\times p$, with $p$ corresponding to the singular values, we defined earlier the matrix" ] @@ -4328,9 +4131,7 @@ { "cell_type": "markdown", "id": "6faf5750", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma} = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix},\n", @@ -4340,9 +4141,7 @@ { "cell_type": "markdown", "id": "d60d582e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where the tilde-matrix $\\tilde{\\boldsymbol{\\Sigma}}$ is a matrix of dimension $p\\times p$ containing only the singular values $\\sigma_i$, that is" ] @@ -4350,9 +4149,7 @@ { "cell_type": "markdown", "id": "84c6c9a6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{\\Sigma}}=\\begin{bmatrix} \\sigma_0 & 0 & 0 & \\dots & 0 & 0 \\\\\n", @@ -4367,9 +4164,7 @@ { "cell_type": "markdown", "id": "42c17867", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning we can write" ] @@ -4377,9 +4172,7 @@ { "cell_type": "markdown", "id": "5abca958", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2\\boldsymbol{V}^T.\n", @@ -4389,9 +4182,7 @@ { "cell_type": "markdown", "id": "8840539e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" ] @@ -4399,9 +4190,7 @@ { "cell_type": "markdown", "id": "bb62c206", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2.\n", @@ -4411,9 +4200,7 @@ { "cell_type": "markdown", "id": "b07a0443", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## What does it mean?\n", "\n", @@ -4425,9 +4212,7 @@ { "cell_type": "markdown", "id": "21dbb160", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", @@ -4437,9 +4222,7 @@ { "cell_type": "markdown", "id": "94e2b39d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In other words, each non-zero singular value of $\\boldsymbol{X}$ is a positive\n", "square root of an eigenvalue of $\\boldsymbol{X}^T\\boldsymbol{X}$. It means also that\n", @@ -4459,9 +4242,7 @@ { "cell_type": "markdown", "id": "c0c57304", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{X}]=\\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X},\n", @@ -4471,9 +4252,7 @@ { "cell_type": "markdown", "id": "fba325af", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "meaning that every squared non-singular value of $\\boldsymbol{X}$ divided by $n$ (\n", "the number of samples) are the eigenvalues of the covariance\n", @@ -4486,9 +4265,7 @@ { "cell_type": "markdown", "id": "2f93e108", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## And finally $\\boldsymbol{X}\\boldsymbol{X}^T$\n", "\n", @@ -4498,9 +4275,7 @@ { "cell_type": "markdown", "id": "0980890f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{U}^T.\n", @@ -4510,9 +4285,7 @@ { "cell_type": "markdown", "id": "083e0c4f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Since the matrices here have dimension $n\\times n$, we have" ] @@ -4520,9 +4293,7 @@ { "cell_type": "markdown", "id": "018f833f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\boldsymbol{0}\\\\ \\end{bmatrix}=\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix},\n", @@ -4532,9 +4303,7 @@ { "cell_type": "markdown", "id": "e24970b9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "leading to" ] @@ -4542,9 +4311,7 @@ { "cell_type": "markdown", "id": "717244d4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\boldsymbol{U}^T.\n", @@ -4554,9 +4321,7 @@ { "cell_type": "markdown", "id": "4af271cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying with $\\boldsymbol{U}$ from the right gives us the eigenvalue problem" ] @@ -4564,9 +4329,7 @@ { "cell_type": "markdown", "id": "fd37dbba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U}=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}.\n", @@ -4576,9 +4339,7 @@ { "cell_type": "markdown", "id": "4994c663", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that the eigenvalues of $\\boldsymbol{X}\\boldsymbol{X}^T$ are again given by\n", "the non-zero singular values plus now a series of zeros. The column\n", @@ -4593,9 +4354,7 @@ { "cell_type": "markdown", "id": "619fc071", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Back to Ridge and LASSO Regression\n", "\n", @@ -4606,9 +4365,7 @@ { "cell_type": "markdown", "id": "4aa4b3cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\right)\\right\\}.\n", @@ -4618,9 +4375,7 @@ { "cell_type": "markdown", "id": "30146bc3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "or we can state it as" ] @@ -4628,9 +4383,7 @@ { "cell_type": "markdown", "id": "a6931d07", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -4641,9 +4394,7 @@ { "cell_type": "markdown", "id": "28849f6b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have used the definition of a norm-2 vector, that is" ] @@ -4651,9 +4402,7 @@ { "cell_type": "markdown", "id": "189c09b8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", @@ -4663,9 +4412,7 @@ { "cell_type": "markdown", "id": "2cc85459", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "By minimizing the above equation with respect to the parameters\n", "$\\boldsymbol{\\theta}$ we could then obtain an analytical expression for the\n", @@ -4676,9 +4423,7 @@ { "cell_type": "markdown", "id": "63cc18f2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -4689,9 +4434,7 @@ { "cell_type": "markdown", "id": "26ec85d7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to the Ridge regression minimization problem where we\n", "require that $\\vert\\vert \\boldsymbol{\\theta}\\vert\\vert_2^2\\le t$, where $t$ is\n", @@ -4701,9 +4444,7 @@ { "cell_type": "markdown", "id": "d10bf29f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\theta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\theta}\\vert\\vert_1,\n", @@ -4713,9 +4454,7 @@ { "cell_type": "markdown", "id": "d80c1950", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have a new optimization equation" ] @@ -4723,9 +4462,7 @@ { "cell_type": "markdown", "id": "8c4ef7d9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "{\\displaystyle \\min_{\\boldsymbol{\\theta}\\in\n", @@ -4736,9 +4473,7 @@ { "cell_type": "markdown", "id": "d220a4fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", "\n", @@ -4748,9 +4483,7 @@ { "cell_type": "markdown", "id": "a754dfcb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", @@ -4760,9 +4493,7 @@ { "cell_type": "markdown", "id": "84ce95ee", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Ridge regression, as discussed above, is nothing but the standard OLS with a\n", "modified diagonal term added to $\\boldsymbol{X}^T\\boldsymbol{X}$. The consequences, in\n", @@ -4777,9 +4508,7 @@ { "cell_type": "markdown", "id": "eb70c3ba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\theta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n", @@ -4789,9 +4518,7 @@ { "cell_type": "markdown", "id": "b78198ec", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For Ridge regression this becomes" ] @@ -4799,9 +4526,7 @@ { "cell_type": "markdown", "id": "42500af7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\theta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n", @@ -4811,9 +4536,7 @@ { "cell_type": "markdown", "id": "c27c4731", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$." ] @@ -4821,9 +4544,7 @@ { "cell_type": "markdown", "id": "3a231959", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Interpreting the Ridge results\n", "\n", @@ -4833,9 +4554,7 @@ { "cell_type": "markdown", "id": "708b2568", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n", @@ -4845,9 +4564,7 @@ { "cell_type": "markdown", "id": "3f444e5e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n", "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n", @@ -4861,9 +4578,7 @@ { "cell_type": "markdown", "id": "28a39069", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More interpretations\n", "\n", @@ -4873,9 +4588,7 @@ { "cell_type": "markdown", "id": "3b7f1ec3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n", @@ -4885,9 +4598,7 @@ { "cell_type": "markdown", "id": "58c302ac", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the standard OLS results in" ] @@ -4895,9 +4606,7 @@ { "cell_type": "markdown", "id": "dfbc40a2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\theta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{n-1}\\boldsymbol{u}_i\\boldsymbol{u}_i^T\\boldsymbol{y},\n", @@ -4907,9 +4616,7 @@ { "cell_type": "markdown", "id": "27a3dfa6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -4917,9 +4624,7 @@ { "cell_type": "markdown", "id": "eab4118d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\theta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\theta}^{\\mathrm{OLS}},\n", @@ -4929,9 +4634,7 @@ { "cell_type": "markdown", "id": "1b6c78a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n", "the Ridge estimator converges to zero when the hyperparameter goes to\n", @@ -4946,9 +4649,7 @@ { "cell_type": "markdown", "id": "0c7d1aa4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Deriving the Lasso Regression Equations\n", "\n", @@ -4958,9 +4659,7 @@ { "cell_type": "markdown", "id": "66be627d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{X},\\boldsymbol{\\theta})=\\frac{1}{n}\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\theta}\\vert\\vert_1,\n", @@ -4970,9 +4669,7 @@ { "cell_type": "markdown", "id": "8890e9c1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Taking the derivative with respect to $\\boldsymbol{\\theta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)" ] @@ -4980,9 +4677,7 @@ { "cell_type": "markdown", "id": "dd17e6a0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{d \\vert \\theta\\vert}{d \\theta}=\\mathrm{sgn}(\\theta)=\\left\\{\\begin{array}{cc} 1 & \\theta > 0 \\\\-1 & \\theta < 0, \\end{array}\\right.\n", @@ -4992,9 +4687,7 @@ { "cell_type": "markdown", "id": "e4901607", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have that the derivative of the cost function is" ] @@ -5002,9 +4695,7 @@ { "cell_type": "markdown", "id": "2eb73708", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\theta})}{\\partial \\boldsymbol{\\theta}}=-\\frac{2}{n}\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\theta})+\\lambda sgn(\\boldsymbol{\\theta})=0,\n", @@ -5014,9 +4705,7 @@ { "cell_type": "markdown", "id": "1e01a932", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and reordering we have" ] @@ -5024,9 +4713,7 @@ { "cell_type": "markdown", "id": "32808d90", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\theta}+\\frac{n}{2}\\lambda sgn(\\boldsymbol{\\theta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -5036,9 +4723,7 @@ { "cell_type": "markdown", "id": "55548599", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can redefine $\\lambda$ to absorb the constant $n/2$ and we rewrite the last equation as" ] @@ -5046,9 +4731,7 @@ { "cell_type": "markdown", "id": "b5750cba", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\theta}+\\lambda sgn(\\boldsymbol{\\theta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -5058,9 +4741,7 @@ { "cell_type": "markdown", "id": "22aeaf24", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss how to code LASSO regression next week, when we have introduced gradient methods." ] @@ -5068,9 +4749,7 @@ { "cell_type": "markdown", "id": "5b553c45", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Material for exercises week 35" ] @@ -5078,9 +4757,7 @@ { "cell_type": "markdown", "id": "9b726931", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Important technicalities: More on Rescaling data\n", "\n", @@ -5137,10 +4814,7 @@ "cell_type": "code", "execution_count": 17, "id": "767365c7", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", @@ -5164,9 +4838,7 @@ { "cell_type": "markdown", "id": "3682c045", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Let us try to understand what this may imply mathematically when we\n", "subtract the mean values, also known as *zero centering*. For\n", @@ -5178,9 +4850,7 @@ { "cell_type": "markdown", "id": "d31a8d9d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\theta_0, \\theta_1, ... , \\theta_{p-1}) = \\frac{1}{n}\\sum_{i=0}^{n} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij}\\theta_j\\right)^2,.\n", @@ -5190,9 +4860,7 @@ { "cell_type": "markdown", "id": "16680525", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Recall also that we use the squared value. This expression can lead to an\n", "increased penalty for higher differences between predicted and\n", @@ -5207,9 +4875,7 @@ { "cell_type": "markdown", "id": "e2b809f7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C}{\\partial \\theta_j} = 0,\n", @@ -5219,9 +4885,7 @@ { "cell_type": "markdown", "id": "58a1d22e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "for all $j$. For $\\theta_0$ we have" ] @@ -5229,9 +4893,7 @@ { "cell_type": "markdown", "id": "409ab1ff", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial C}{\\partial \\theta_0} = -\\frac{2}{n}\\sum_{i=0}^{n-1} \\left(y_i - \\theta_0 - \\sum_{j=1}^{p-1} X_{ij} \\theta_j\\right).\n", @@ -5241,9 +4903,7 @@ { "cell_type": "markdown", "id": "caebebfa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Multiplying away the constant $2/n$, we obtain" ] @@ -5251,9 +4911,7 @@ { "cell_type": "markdown", "id": "8f1afa7f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i=0}^{n-1} \\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} \\sum_{j=1}^{p-1} X_{ij} \\theta_j.\n", @@ -5263,9 +4921,7 @@ { "cell_type": "markdown", "id": "85775b22", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Let us specialize first to the case where we have only two parameters $\\theta_0$ and $\\theta_1$.\n", "Our result for $\\theta_0$ simplifies then to" @@ -5274,9 +4930,7 @@ { "cell_type": "markdown", "id": "08dd11f6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "n\\theta_0 = \\sum_{i=0}^{n-1}y_i - \\sum_{i=0}^{n-1} X_{i1} \\theta_1.\n", @@ -5286,9 +4940,7 @@ { "cell_type": "markdown", "id": "5d2e98fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We obtain then" ] @@ -5296,9 +4948,7 @@ { "cell_type": "markdown", "id": "42da16f4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\theta_1\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1}.\n", @@ -5308,9 +4958,7 @@ { "cell_type": "markdown", "id": "8c434ec5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we define" ] @@ -5318,9 +4966,7 @@ { "cell_type": "markdown", "id": "23217d9c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mu_{\\boldsymbol{x}_1}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{i1},\n", @@ -5330,9 +4976,7 @@ { "cell_type": "markdown", "id": "85202a44", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and the mean value of the outputs as" ] @@ -5340,9 +4984,7 @@ { "cell_type": "markdown", "id": "595571b3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mu_y=\\frac{1}{n}\\sum_{i=0}^{n-1}y_i,\n", @@ -5352,9 +4994,7 @@ { "cell_type": "markdown", "id": "507ad25e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "we have" ] @@ -5362,9 +5002,7 @@ { "cell_type": "markdown", "id": "c1af1141", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\theta_0 = \\mu_y - \\theta_1\\mu_{\\boldsymbol{x}_1}.\n", @@ -5374,9 +5012,7 @@ { "cell_type": "markdown", "id": "22b1a4c2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the general case with more parameters than $\\theta_0$ and $\\theta_1$, we have" ] @@ -5384,9 +5020,7 @@ { "cell_type": "markdown", "id": "344d7aaa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\frac{1}{n}\\sum_{i=0}^{n-1}\\sum_{j=1}^{p-1} X_{ij}\\theta_j.\n", @@ -5396,9 +5030,7 @@ { "cell_type": "markdown", "id": "d1d8e7a0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can rewrite the latter equation as" ] @@ -5406,9 +5038,7 @@ { "cell_type": "markdown", "id": "9a0b634c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\theta_0 = \\frac{1}{n}\\sum_{i=0}^{n-1}y_i - \\sum_{j=1}^{p-1} \\mu_{\\boldsymbol{x}_j}\\theta_j,\n", @@ -5418,9 +5048,7 @@ { "cell_type": "markdown", "id": "aad7ad10", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have defined" ] @@ -5428,9 +5056,7 @@ { "cell_type": "markdown", "id": "b0c4c38f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mu_{\\boldsymbol{x}_j}=\\frac{1}{n}\\sum_{i=0}^{n-1} X_{ij},\n", @@ -5440,9 +5066,7 @@ { "cell_type": "markdown", "id": "b52a1b70", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "the mean value for all elements of the column vector $\\boldsymbol{x}_j$.\n", "\n", @@ -5452,9 +5076,7 @@ { "cell_type": "markdown", "id": "90e55e2a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{\\theta}) = (\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta})^T(\\boldsymbol{\\tilde{y}} - \\tilde{X}\\boldsymbol{\\theta}).\n", @@ -5464,9 +5086,7 @@ { "cell_type": "markdown", "id": "01b093ef", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "If we minimize with respect to $\\boldsymbol{\\theta}$ we have then" ] @@ -5474,9 +5094,7 @@ { "cell_type": "markdown", "id": "18810909", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X})^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}},\n", @@ -5486,9 +5104,7 @@ { "cell_type": "markdown", "id": "112af953", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\boldsymbol{\\tilde{y}} = \\boldsymbol{y} - \\overline{\\boldsymbol{y}}$\n", "and $\\tilde{X}_{ij} = X_{ij} - \\frac{1}{n}\\sum_{k=0}^{n-1}X_{kj}$.\n", @@ -5499,9 +5115,7 @@ { "cell_type": "markdown", "id": "dc42bdf3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\hat{\\boldsymbol{\\theta}} = (\\tilde{X}^T\\tilde{X} + \\lambda I)^{-1}\\tilde{X}^T\\boldsymbol{\\tilde{y}}.\n", @@ -5511,9 +5125,7 @@ { "cell_type": "markdown", "id": "9c9fd729", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "What does this mean? And why do we insist on all this? Let us look at some examples.\n", "\n", @@ -5525,10 +5137,7 @@ "cell_type": "code", "execution_count": 18, "id": "94867b33", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -5622,9 +5231,7 @@ { "cell_type": "markdown", "id": "5bd0d190", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The intercept is the value of our output/target variable\n", "when all our features are zero and our function crosses the $y$-axis (for a one-dimensional case). \n", @@ -5643,9 +5250,7 @@ { "cell_type": "markdown", "id": "a06d4663", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=0}^{p-1}\\theta_j^2,\n", @@ -5655,9 +5260,7 @@ { "cell_type": "markdown", "id": "01ab58d2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "but when we take out the intercept, this equation becomes" ] @@ -5665,9 +5268,7 @@ { "cell_type": "markdown", "id": "dca995a5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_2^2 = \\lambda \\sum_{j=1}^{p-1}\\theta_j^2.\n", @@ -5677,9 +5278,7 @@ { "cell_type": "markdown", "id": "3694dc1d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "For Lasso regression we have" ] @@ -5687,9 +5286,7 @@ { "cell_type": "markdown", "id": "5104f166", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\lambda \\vert\\vert \\boldsymbol{\\theta} \\vert\\vert_1 = \\lambda \\sum_{j=1}^{p-1}\\vert\\theta_j\\vert.\n", @@ -5699,9 +5296,7 @@ { "cell_type": "markdown", "id": "91921fb2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "It means that, when scaling the design matrix and the outputs/targets,\n", "by subtracting the mean values, we have an optimization problem which\n", @@ -5717,10 +5312,7 @@ "cell_type": "code", "execution_count": 19, "id": "9af3375b", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -5793,9 +5385,7 @@ { "cell_type": "markdown", "id": "e4a67fbf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The results here agree when we force **Scikit-Learn**'s Ridge function to include the first column in our design matrix.\n", "We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.\n", @@ -5807,10 +5397,7 @@ "cell_type": "code", "execution_count": 20, "id": "07d02e73", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -5896,9 +5483,7 @@ { "cell_type": "markdown", "id": "20eff4f1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We see here, when compared to the code which includes explicitely the\n", "intercept column, that our MSE value is actually smaller. This is\n", @@ -5909,7 +5494,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/Projects/2025/Project1/Project1.do.txt b/doc/src/Projects/2025/Project1/Project1.do.txt index e2163e568..0b19bccd1 100644 --- a/doc/src/Projects/2025/Project1/Project1.do.txt +++ b/doc/src/Projects/2025/Project1/Project1.do.txt @@ -1,4 +1,4 @@ -TITLE: Project 1 on Machine Learning, deadline October 7 (midnight), 2024 +TITLE: Project 1 on Machine Learning, deadline October 6 (midnight), 2025 AUTHOR: "Data Analysis and Machine Learning FYS-STK3155/FYS4155":"http://www.uio.no/studier/emner/matnat/fys/FYS3155/index-eng.html" at University of Oslo, Norway DATE: September 2 @@ -21,7 +21,7 @@ also be discussed during the various lab sessions. Please do ask us if you are i When using codes and material from other sources, you should refer to these in the bibliography of your report, indicating wherefrom you for example got the code, whether this is from the lecture notes, softwares like -Scikit-Learn, TensorFlow, PyTorch or other sources. These should +Scikit-Learn, TensorFlow, PyTorch or other sources such AI software. These should always be cited correctly. How to cite some of the libraries is often indicated from their corresponding GitHub sites or websites, see for example how to cite Scikit-Learn at URL:"https://scikit-learn.org/dev/about.html". @@ -44,114 +44,30 @@ regression methods, including the Ordinary Least Squares (OLS) method. In addition to the scientific part, in this course we want also to give you an experience in writing scientific reports. - - -_A small recommendation when developing the codes here_. Instead of -jumping on to the two-dimensional function described below, we -recommend to do the code development and testing with a simpler -one-dimensional function, similar to those discussed in the exercises -of weeks 35 and 36. A simple test, as discussed during the lectures the first -three weeks is to set the design matrix equal to the identity -matrix. Then your model should give a mean square error which is exactly equal to zero. -When you are sure that your codes function well, you can then replace -the one-dimensional test function with the two-dimensional _Franke_ function -discussed here. - -The Franke function serves as a stepping stone towards the analysis of -real topographic data. The latter is the last part of this project. - - -=== Description of two-dimensional function === - -We will first study how to fit polynomials to a specific -two-dimensional function called "Franke's -function":"http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf". This -is a function which has been widely used when testing various -interpolation and fitting algorithms. Furthermore, after having -established the model and the method, we will employ resamling -techniques such as cross-validation and/or bootstrap in order to perform a -proper assessment of our models. We will also study in detail the -so-called Bias-Variance trade off. - - -The Franke function, which is a weighted sum of four exponentials reads as follows +We will first study how to fit polynomials to specific +one-dimensional functions. We will start with a function given by Runge's function (see URL:"https://en.wikipedia.org/wiki/Runge%27s_phenomenon" for a discussion). The one-dimensional function we will study first is !bt -\begin{align*} -f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\ -&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }. -\end{align*} +\[ +f(x) = \frac{1}{1+25x^2}. +\] !et -The function will be defined for $x,y\in [0,1]$. In a sense, our data are thus scaled to a particular domain for the input values. -Our first step will -be to perform an OLS regression analysis of this function, trying out -a polynomial fit with an $x$ and a $y$ dependence of the form $[x, y, -x^2, y^2, xy, \dots]$. We will also include bootstrap first as a -resampling technique. After that we will include the cross-validation -technique. -We can -use a uniform distribution to set up the arrays of values for $x$ and -$y$, or as in the example below just a set of fixed values for $x$ and -$y$ with a given step size. We will fit a function (for example a -polynomial) of $x$ and $y$. Thereafter we will repeat much of the +Our first step will be to perform an OLS regression analysis of this +function, trying out a polynomial fit with an $x$ dependence of the +form $[x,x^2,\dots]$. We can use a uniform distribution to set up the +arrays of values for $x \in [-5,5]$, or alternatively use a fixed step size. +Thereafter we will repeat much of the same procedure using the Ridge and Lasso regression methods, introducing thus a dependence on the bias (penalty) $\lambda$. -Finally we are going to use (real) digital terrain data and try to -reproduce these data using the same methods. We will also try to go -beyond the second-order polynomials metioned above and explore -which polynomial fits the data best. +We will also include bootstrap first as a +resampling technique. After that we will include the cross-validation +technique. -The Python code for the Franke function is included here (it performs also a three-dimensional plot of it) -!bc pycod -from mpl_toolkits.mplot3d import Axes3D -import matplotlib.pyplot as plt -from matplotlib import cm -from matplotlib.ticker import LinearLocator, FormatStrFormatter -import numpy as np -from random import random, seed - -fig = plt.figure() -ax = fig.add_subplot(projection = '3d') -# Make data. -x = np.arange(0, 1, 0.05) -y = np.arange(0, 1, 0.05) -x, y = np.meshgrid(x,y) - - -def FrankeFunction(x,y): - term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) - term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) - term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) - term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) - return term1 + term2 + term3 + term4 - - -z = FrankeFunction(x, y) - -# Plot the surface. -surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm, - linewidth=0, antialiased=False) - -# Customize the z axis. -ax.set_zlim(-0.10, 1.40) -ax.zaxis.set_major_locator(LinearLocator(10)) -ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f')) - -# Add a color bar which maps values to colors. -fig.colorbar(surf, shrink=0.5, aspect=5) - -plt.show() - -!ec - - -If you wish to compare your results with other on the Franke function or other popular functions tested with linear regression, see the list in Figure 1 of the article by Cook et al at URL:"https://arxiv.org/abs/2401.11694". - === Part a) : Ordinary Least Square (OLS) on the Franke function === We will generate our own dataset for a function @@ -191,8 +107,8 @@ where we have defined the mean value of $\bm{y}$ as \] !et -Plot the resulting scores (MSE and R$^2$) as functions of the polynomial degree (here up to polymial degree five). -Plot also the parameters $\beta$ as you increase the order of the polynomial. Comment your results. +Plot the resulting scores (MSE and R$^2$) as functions of the polynomial degree (here up to polymial degree 20). +Plot also the parameters $\theta$ as you increase the order of the polynomial. Comment your results. Your code has to include a scaling/centering of the data (for example by subtracting the mean value), and @@ -211,23 +127,27 @@ data and say test data. An accepted rule of thumb is to use approximately $2/3$ to $4/5$ of the data as training data. -You can easily reuse the solutions to your exercises from week 35 and week 36. +You can easily reuse the solutions to your exercises from week 35. See also the lecture slides from week 35 and week 36. On scaling, we recommend reading the following section from the scikit-learn software description, see URL:"https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html#plot-all-scaling-standard-scaler-section". -=== Part b): Adding Ridge regression for the Franke function === +=== Part b): Adding Ridge regression for the Runge function === Write your own code for the Ridge method, either using matrix inversion or the singular value decomposition as done in the previous -exercise. The lecture notes from week 35 and 36 contain more information. Furthermore, the numerical exercise from week 36 is something you can reuse here. +exercise. The lecture notes from week 35 and 36 contain more information. Furthermore, the exercise from week 36 is something you can reuse here. Perform the same analysis as you did in the previous exercise but now for different values of $\lambda$. Compare and analyze your results with those obtained in part a) with the ordinary least squares method. Study the dependence on $\lambda$. +Add to the + + + === Part c): Adding Lasso for the Franke function === This exercise is essentially a repeat of the previous two ones, but now @@ -239,65 +159,6 @@ model fits the data best. -=== Part d): Paper and pencil part === - - -This exercise deals with various mean values and variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of "Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer":"https://www.springer.com/gp/book/9780387848570"). The exercise is also part of the weekly exercises for week 37. - -The assumption we have made is -that there exists a continuous function $f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim N(0, \sigma^2)$ -which describes our data -!bt -\[ -\bm{y} = f(\bm{x})+\bm{\varepsilon} -\] -!et - -We then approximate this function $f(\bm{x})$ with our model $\bm{\tilde{y}}$ from the solution of the linear regression equations (ordinary least squares OLS), that is our -function $f$ is approximated by $\bm{\tilde{y}}$ where we minimized $(\bm{y}-\bm{\tilde{y}})^2$, with -!bt -\[ -\bm{\tilde{y}} = \bm{X}\bm{\beta}. -\] -!et -The matrix $\bm{X}$ is the so-called design or feature matrix. - - -Show that the expectation value of $\bm{y}$ for a given element $i$ -!bt -\[ -\mathbb{E}(y_i) =\sum_{j}x_{ij} \beta_j=\mathbf{X}_{i, \ast} \, \bm{\beta}, -\] -!et -and that -its variance is -!bt -\[ -\mbox{Var}(y_i) = \sigma^2. -\] -!et -Hence, $y_i \sim N( \mathbf{X}_{i, \ast} \, \bm{\beta}, \sigma^2)$, that is $\bm{y}$ follows a normal distribution with -mean value $\bm{X}\bm{\beta}$ and variance $\sigma^2$. - -With the OLS expressions for the optimal parameters $\bm{\hat{\beta}}$ show that -!bt -\[ -\mathbb{E}(\bm{\hat{\beta}}) = \bm{\beta}. -\] -!et -Show finally that the variance of $\bm{\beta}$ is -!bt -\[ -\mbox{Var}(\bm{\hat{\beta}}) = \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}. -\] -!et - - -We can use the last expression when we define a so-called confidence interval for the parameters $\beta$. -A given parameter $\beta_j$ is given by the diagonal matrix element of the above matrix. - - - === Part e): Bias-variance trade-off and resampling techniques === @@ -388,80 +249,18 @@ You can follow the code example in the jupyter-book at URL:"https://compphysics. The aim here is to implement another widely popular resampling technique, the so-called cross-validation method. -Implement the $k$-fold cross-validation algorithm (write your own -code or use the functionality of _Scikit-Learn_) and evaluate again the MSE function resulting +Implement the $k$-fold cross-validation algorithm (feel free to use the functionality of _Scikit-Learn_ or write your own code) and evaluate again the MSE function resulting from the test folds. Compare the MSE you get from your cross-validation code with the one you got from your _bootstrap_ code. Comment your results. Try $5-10$ folds. -In addition to using the ordinary least squares method, you should include both Ridge and Lasso regression. +In addition to using the ordinary least squares method, you should include both Ridge and Lasso regression in the analysis. === Part g): Analysis of real data === -With our codes functioning and having been tested properly on a -simpler function we are now ready to look at real data. We will -essentially repeat in this exercise what was done in exercises a-f. However, we -need first to download the data and prepare properly the inputs to our -codes. We are going to download digital terrain data from the website -URL:"https://earthexplorer.usgs.gov/", - -Or, if you prefer, we have placed selected datafiles at URL:"https://github.com/CompPhysics/MachineLearning/tree/master/doc/Projects/2023/Project1/DataFiles" - -In order to obtain data for a specific region, you need to register as -a user (free) at this website and then decide upon which area you want -to fetch the digital terrain data from. In order to be able to read -the data properly, you need to specify that the format should be _SRTM -Arc-Second Global_ and download the data as a _GeoTIF_ file. The -files are then stored in *tif* format which can be imported into a -Python program using - -!bc pycod -scipy.misc.imread -!ec - -Here is a simple part of a Python code which reads and plots the data -from such files - -!bc pycod -import numpy as np -from imageio import imread -import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D -from matplotlib import cm - -# Load the terrain -terrain1 = imread('SRTM_data_Norway_1.tif') -# Show the terrain -plt.figure() -plt.title('Terrain over Norway 1') -plt.imshow(terrain1, cmap='gray') -plt.xlabel('X') -plt.ylabel('Y') -plt.show() -!ec - -If you should have problems in downloading the digital terrain data, -we provide two examples under the data folder of project 1. One is -from a region close to Stavanger in Norway and the other Møsvatn -Austfjell, again in Norway. -Feel free to produce your own terrain data. - - -Alternatively, if you would like to use another data set, feel free to do so. This could be data close to your reseach area or simply a data set you found interesting. See for example "kaggle.com":"https://www.kaggle.com/datasets" for examples. - - -Our final part deals with the parameterization of your digital terrain -data (or your own data). We will apply all three methods for linear regression, the same type (or higher order) of polynomial -approximation and cross-validation as resampling technique to evaluate which -model fits the data best. - -At the end, you should present a critical evaluation of your results -and discuss the applicability of these regression methods to the type -of data presented here (either the terrain data we propose or other data sets). - @@ -548,3 +347,75 @@ encompass communities of developers in the thousands or more. And the number of code developers and contributors keeps increasing. +"Franke's +function":"http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf". This +is a function which has been widely used when testing various +interpolation and fitting algorithms. Furthermore, after having +established the model and the method, we will employ resamling +techniques such as cross-validation and/or bootstrap in order to perform a +proper assessment of our models. We will also study in detail the +so-called Bias-Variance trade off. + + +The Franke function, which is a weighted sum of four exponentials reads as follows +!bt +\begin{align*} +f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\ +&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }. +\end{align*} +!et + +The function will be defined for $x,y\in [0,1]$. In a sense, our data are thus scaled to a particular domain for the input values. + + +Finally we are going to use (real) digital terrain data and try to +reproduce these data using the same methods. We will also try to go +beyond the second-order polynomials metioned above and explore +which polynomial fits the data best. + + +The Python code for the Franke function is included here (it performs also a three-dimensional plot of it) +!bc pycod +from mpl_toolkits.mplot3d import Axes3D +import matplotlib.pyplot as plt +from matplotlib import cm +from matplotlib.ticker import LinearLocator, FormatStrFormatter +import numpy as np +from random import random, seed + +fig = plt.figure() +ax = fig.add_subplot(projection = '3d') +# Make data. +x = np.arange(0, 1, 0.05) +y = np.arange(0, 1, 0.05) +x, y = np.meshgrid(x,y) + + +def FrankeFunction(x,y): + term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) + term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) + term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) + term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) + return term1 + term2 + term3 + term4 + + +z = FrankeFunction(x, y) + +# Plot the surface. +surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm, + linewidth=0, antialiased=False) + +# Customize the z axis. +ax.set_zlim(-0.10, 1.40) +ax.zaxis.set_major_locator(LinearLocator(10)) +ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f')) + +# Add a color bar which maps values to colors. +fig.colorbar(surf, shrink=0.5, aspect=5) + +plt.show() + +!ec + + +If you wish to compare your results with other on the Franke function or other popular functions tested with linear regression, see the list in Figure 1 of the article by Cook et al at URL:"https://arxiv.org/abs/2401.11694".