update book
This commit is contained in:
Morten Hjorth-Jensen
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
details.dropdown .summary-title{padding-right:3em !important;-moz-user-select:none;-ms-user-select:none;-webkit-user-select:none;user-select:none}details.dropdown:hover{cursor:pointer}details.dropdown .summary-content{cursor:default}details.dropdown summary{list-style:none;padding:1em}details.dropdown summary .octicon.no-title{vertical-align:middle}details.dropdown[open] summary .octicon.no-title{visibility:hidden}details.dropdown summary::-webkit-details-marker{display:none}details.dropdown summary:focus{outline:none}details.dropdown summary:hover .summary-up svg,details.dropdown summary:hover .summary-down svg{opacity:1}details.dropdown .summary-up svg,details.dropdown .summary-down svg{display:block;opacity:.6}details.dropdown .summary-up,details.dropdown .summary-down{pointer-events:none;position:absolute;right:1em;top:.75em}details.dropdown[open] .summary-down{visibility:hidden}details.dropdown:not([open]) .summary-up{visibility:hidden}details.dropdown.fade-in[open] summary~*{-moz-animation:panels-fade-in .5s ease-in-out;-webkit-animation:panels-fade-in .5s ease-in-out;animation:panels-fade-in .5s ease-in-out}details.dropdown.fade-in-slide-down[open] summary~*{-moz-animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out;-webkit-animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out;animation:panels-fade-in .5s ease-in-out, panels-slide-down .5s ease-in-out}@keyframes panels-fade-in{0%{opacity:0}100%{opacity:1}}@keyframes panels-slide-down{0%{transform:translate(0, -10px)}100%{transform:translate(0, 0)}}.octicon{display:inline-block;fill:currentColor;vertical-align:text-top}.tabbed-content{box-shadow:0 -.0625rem var(--tabs-color-overline),0 .0625rem var(--tabs-color-underline);display:none;order:99;padding-bottom:.75rem;padding-top:.75rem;width:100%}.tabbed-content>:first-child{margin-top:0 !important}.tabbed-content>:last-child{margin-bottom:0 !important}.tabbed-content>.tabbed-set{margin:0}.tabbed-set{border-radius:.125rem;display:flex;flex-wrap:wrap;margin:1em 0;position:relative}.tabbed-set>input{opacity:0;position:absolute}.tabbed-set>input:checked+label{border-color:var(--tabs-color-label-active);color:var(--tabs-color-label-active)}.tabbed-set>input:checked+label+.tabbed-content{display:block}.tabbed-set>input:focus+label{outline-style:auto}.tabbed-set>input:not(.focus-visible)+label{outline:none;-webkit-tap-highlight-color:transparent}.tabbed-set>label{border-bottom:.125rem solid transparent;color:var(--tabs-color-label-inactive);cursor:pointer;font-size:var(--tabs-size-label);font-weight:700;padding:1em 1.25em .5em;transition:color 250ms;width:auto;z-index:1}html .tabbed-set>label:hover{color:var(--tabs-color-label-active)}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
:root {
|
||||
--tabs-color-label-active: hsla(231, 99%, 66%, 1);
|
||||
--tabs-color-label-inactive: rgba(178, 206, 245, 0.62);
|
||||
--tabs-color-overline: rgb(207, 236, 238);
|
||||
--tabs-color-underline: rgb(207, 236, 238);
|
||||
--tabs-size-label: 1rem;
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b941c35",
|
||||
"id": "8e6632a0",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -14,7 +14,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dc05b096",
|
||||
"id": "82705c4f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -27,7 +27,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2cf07405",
|
||||
"id": "921bf331",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -46,7 +46,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3c139edb",
|
||||
"id": "adff65d5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -58,7 +58,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aad4cfac",
|
||||
"id": "70418b3d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6682282f",
|
||||
"id": "11a3cf73",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -83,7 +83,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89e2f4c4",
|
||||
"id": "04a06b51",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -99,7 +99,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b06d4e53",
|
||||
"id": "408db3d9",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -120,7 +120,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "63796480",
|
||||
"id": "37fb732c",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -140,12 +140,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "80748600",
|
||||
"id": "d861e1e3",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Fill in the necessary details.\n",
|
||||
"Fill in the necessary details. Do we need to center the $y$-values? \n",
|
||||
"\n",
|
||||
"After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n",
|
||||
"and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This makes the optimization landscape\n",
|
||||
@@ -156,7 +156,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "92751e5f",
|
||||
"id": "b3e774d0",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -168,7 +168,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aedfbd7a",
|
||||
"id": "d5dc7708",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -179,7 +179,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "5d1288fa",
|
||||
"id": "4c9c86ac",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -187,7 +187,9 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set regularization parameter, either a single value or a vector of values\n",
|
||||
"lambda = ?\n",
|
||||
"# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called \"anonymous functions\" or \"lambda functions.\"\n",
|
||||
"lam = ?\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y\n",
|
||||
"I = np.eye(n_features)\n",
|
||||
@@ -200,7 +202,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "628f5e89",
|
||||
"id": "eeae00fd",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -214,7 +216,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f115ba4e",
|
||||
"id": "e1c215d5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -226,7 +228,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a9b5189c",
|
||||
"id": "587dd3dc",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -238,7 +240,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a3969ff6",
|
||||
"id": "bfa34697",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -258,7 +260,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "34d87303",
|
||||
"id": "49245f55",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -273,19 +275,8 @@
|
||||
"# Initialize weights for gradient descent\n",
|
||||
"theta = np.zeros(n_features)\n",
|
||||
"\n",
|
||||
"# Arrays to store history for plotting\n",
|
||||
"cost_history = np.zeros(num_iters)\n",
|
||||
"\n",
|
||||
"# Gradient descent loop\n",
|
||||
"m = n_samples # number of data points\n",
|
||||
"for t in range(num_iters):\n",
|
||||
" # Compute prediction error\n",
|
||||
" error = X_norm.dot(theta) - y_centered \n",
|
||||
" # Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring\n",
|
||||
" cost_OLS = ?\n",
|
||||
" cost_Ridge = ?\n",
|
||||
" # You could add a history for both methods (optional)\n",
|
||||
" cost_history[t] = ?\n",
|
||||
" # Compute gradients for OSL and Ridge\n",
|
||||
" grad_OLS = ?\n",
|
||||
" grad_Ridge = ?\n",
|
||||
@@ -302,31 +293,33 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "989f70bb",
|
||||
"id": "f3f43f2c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"### 4a)\n",
|
||||
"\n",
|
||||
"Discuss the results as function of the learning rate parameters and the number of iterations."
|
||||
"Write first a gradient descent code for OLS only using the above template.\n",
|
||||
"Discuss the results as function of the learning rate parameters and the number of iterations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "370b2dad",
|
||||
"id": "9ba303be",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"### 4b)\n",
|
||||
"\n",
|
||||
"Write then a similar code for Ridge regression using the above template.\n",
|
||||
"Try to add a stopping parameter as function of the number iterations and the difference between the new and old $\\theta$ values. How would you define a stopping criterion?"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ef197cd7",
|
||||
"id": "78362c6c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -352,7 +345,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "4ccc2f65",
|
||||
"id": "8be1cebe",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -381,7 +374,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "00e279ef",
|
||||
"id": "e2693666",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -395,7 +388,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c910b3f4",
|
||||
"id": "bc954d12",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -407,7 +400,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89e6e040",
|
||||
"id": "6534b610",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1da77599",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercises week 37\n",
|
||||
"## September 15-19\n",
|
||||
"\n",
|
||||
"## Resampling and the Bias-Variance Trade-off"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e9f27b0e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Learning goals\n",
|
||||
"\n",
|
||||
"After completing these exercises, you will know how to\n",
|
||||
"- Derive expectation and variances values related to linear regression\n",
|
||||
"- Compute expectation and variances values related to linear regression\n",
|
||||
"- Compute and evaluate the trade-off between bias and variance of a model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "984af8e3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercises are also a part of project 1 and can be reused in the theory part of the project.\n",
|
||||
"\n",
|
||||
"For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n",
|
||||
"\n",
|
||||
"We assume that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$ which describes our data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c16f7d0e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9fcf981a",
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"We further assume that this continous function can be modeled with a linear model $\\mathbf{\\tilde{y}}$ of some features $\\mathbf{X}$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4189366",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\boldsymbol{y} = \\boldsymbol{\\tilde{y}} + \\boldsymbol{\\varepsilon} = \\boldsymbol{X}\\boldsymbol{\\beta} +\\boldsymbol{\\varepsilon}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f4fca21b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We therefore get that our data $\\boldsymbol{y}$ has an expectation value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$, that is $\\boldsymbol{y}$ follows a normal distribution with mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5de0c7e6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 1: Expectation values for ordinary least squares expressions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d878c699",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "08b7007d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\boldsymbol{\\beta}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "46e93394",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Show that the variance of $\\boldsymbol{\\hat{\\beta}_{OLS}}$ is"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "be1b65be",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbf{Var}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d2143684",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can use the last expression when we define a [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval) for the parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$. \n",
|
||||
"A given parameter ${\\boldsymbol{\\hat{\\beta}_{OLS}}}_j$ is given by the diagonal matrix element of the above matrix."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f5c2dc22",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 2: Expectation values for Ridge regression"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3893e3e7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{Ridge}}$ show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "79dc571f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "028209a1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We see that $\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big] \\not= \\mathbb{E} \\big[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}\\big ]$ for any $\\lambda > 0$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "65f6f914",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Why do we say that Ridge regression gives a biased estimate? Is this a problem?"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b4e721fc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**c)** Show that the variance is"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "090eb1e1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbf{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6b8e8697",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We see that if the parameter $\\lambda$ goes to infinity then the variance of the Ridge parameters $\\boldsymbol{\\beta}$ goes to zero."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "74bc300b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 3: Deriving the expression for the Bias-Variance Trade-off"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eeb86010",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.\n",
|
||||
"\n",
|
||||
"The parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ are found by optimizing the mean squared error via the so-called cost function"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "522a0d1d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "831db06c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a\n",
|
||||
"term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function $f$ with the model ${\\bf y}$.\n",
|
||||
"show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8cc52b3c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8cb50416",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"with"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e49bdbb4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eca5554a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"and"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b1054343",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathrm{var}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\tilde{\\boldsymbol{y}}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "70fbfcd7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Explain what the terms mean and discuss their interpretations."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b8f8b9d1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 4: Computing the Bias and Variance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9e012430",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Before you compute the bias and variance of a real model for different complexities, let's for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.\n",
|
||||
"\n",
|
||||
"**a)** Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"id": "b5bf581c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"n = 100\n",
|
||||
"bootstraps = 1000\n",
|
||||
"\n",
|
||||
"predictions = np.random.rand(bootstraps, n) * 10 + 10\n",
|
||||
"targets = np.random.rand(bootstraps, n)\n",
|
||||
"\n",
|
||||
"mse = ...\n",
|
||||
"bias = ...\n",
|
||||
"variance = ..."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7b1dc621",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Change the prediction values in some way to increase the bias while decreasing the variance.\n",
|
||||
"\n",
|
||||
"**c)** Change the prediction values in some way to increase the variance while decreasing the bias."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8da63362",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**d)** Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "dd5855e4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures # use the fit_transform method of the created object!\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"from sklearn.metrics import mean_squared_error\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.utils import resample"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7e35fa37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"n = 100\n",
|
||||
"bootstraps = 1000\n",
|
||||
"\n",
|
||||
"x = np.linspace(-3, 3, n)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1)\n",
|
||||
"\n",
|
||||
"biases = []\n",
|
||||
"variances = []\n",
|
||||
"mses = []\n",
|
||||
"\n",
|
||||
"#for p in range(1, 5):\n",
|
||||
"# predictions = ...\n",
|
||||
"# targets = ...\n",
|
||||
"# for b in range(bootstraps):\n",
|
||||
"# x_sample, y_sample = ...\n",
|
||||
"# X = ...\n",
|
||||
"# X_train, X_test, y_train, y_test = ...\n",
|
||||
"#\n",
|
||||
"# predictions[b, :] = \n",
|
||||
"# targets[b, :] = \n",
|
||||
"# \n",
|
||||
"# biases.append(...)\n",
|
||||
"# variances.append(...)\n",
|
||||
"# mses.append(...)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "253b8461",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**e)** Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).\n",
|
||||
"\n",
|
||||
"**f)** Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -228,6 +228,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -62,7 +62,7 @@
|
||||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek37';</script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
<link rel="next" title="Project 1 on Machine Learning, deadline October 6 (midnight), 2025" href="project1.html" />
|
||||
<link rel="next" title="Week 37: Statistical interpretations and Resampling Methods" href="week37.html" />
|
||||
<link rel="prev" title="Week 36: Linear Regression and Gradient descent" href="week36.html" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@@ -470,7 +472,7 @@ y_centered = ?
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p>Fill in the necessary details.</p>
|
||||
<p>Fill in the necessary details. Do we need to center the <span class="math notranslate nohighlight">\(y\)</span>-values?</p>
|
||||
<p>After this preprocessing, each column of <span class="math notranslate nohighlight">\(\boldsymbol{X}_{\mathrm{norm}}\)</span> has mean zero and standard deviation <span class="math notranslate nohighlight">\(1\)</span>
|
||||
and <span class="math notranslate nohighlight">\(\boldsymbol{y}_{\mathrm{centered}}\)</span> has mean 0. This makes the optimization landscape
|
||||
nicer and ensures the regularization penalty <span class="math notranslate nohighlight">\(\lambda \sum_j
|
||||
@@ -487,7 +489,9 @@ same scale).</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span># Set regularization parameter, either a single value or a vector of values
|
||||
lambda = ?
|
||||
# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called "anonymous functions" or "lambda functions."
|
||||
lam = ?
|
||||
|
||||
|
||||
# Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y
|
||||
I = np.eye(n_features)
|
||||
@@ -533,19 +537,8 @@ num_iters = 1000
|
||||
# Initialize weights for gradient descent
|
||||
theta = np.zeros(n_features)
|
||||
|
||||
# Arrays to store history for plotting
|
||||
cost_history = np.zeros(num_iters)
|
||||
|
||||
# Gradient descent loop
|
||||
m = n_samples # number of data points
|
||||
for t in range(num_iters):
|
||||
# Compute prediction error
|
||||
error = X_norm.dot(theta) - y_centered
|
||||
# Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring
|
||||
cost_OLS = ?
|
||||
cost_Ridge = ?
|
||||
# You could add a history for both methods (optional)
|
||||
cost_history[t] = ?
|
||||
# Compute gradients for OSL and Ridge
|
||||
grad_OLS = ?
|
||||
grad_Ridge = ?
|
||||
@@ -564,11 +557,13 @@ print("Gradient Descent Ridge coefficients:", theta_gdRidge)
|
||||
</div>
|
||||
<section id="id2">
|
||||
<h3>4a)<a class="headerlink" href="#id2" title="Link to this heading">#</a></h3>
|
||||
<p>Discuss the results as function of the learning rate parameters and the number of iterations.</p>
|
||||
<p>Write first a gradient descent code for OLS only using the above template.
|
||||
Discuss the results as function of the learning rate parameters and the number of iterations</p>
|
||||
</section>
|
||||
<section id="id3">
|
||||
<h3>4b)<a class="headerlink" href="#id3" title="Link to this heading">#</a></h3>
|
||||
<p>Try to add a stopping parameter as function of the number iterations and the difference between the new and old <span class="math notranslate nohighlight">\(\theta\)</span> values. How would you define a stopping criterion?</p>
|
||||
<p>Write then a similar code for Ridge regression using the above template.
|
||||
Try to add a stopping parameter as function of the number iterations and the difference between the new and old <span class="math notranslate nohighlight">\(\theta\)</span> values. How would you define a stopping criterion?</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="exercise-5-ridge-regression-and-a-new-synthetic-dataset">
|
||||
@@ -667,11 +662,11 @@ should be in the same ballpark. Which method (OLS or Ridge) gives the best resu
|
||||
</div>
|
||||
</a>
|
||||
<a class="right-next"
|
||||
href="project1.html"
|
||||
href="week37.html"
|
||||
title="next page">
|
||||
<div class="prev-next-info">
|
||||
<p class="prev-next-subtitle">next</p>
|
||||
<p class="prev-next-title">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</p>
|
||||
<p class="prev-next-title">Week 37: Statistical interpretations and Resampling Methods</p>
|
||||
</div>
|
||||
<i class="fa-solid fa-angle-right"></i>
|
||||
</a>
|
||||
|
||||
@@ -0,0 +1,690 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
|
||||
<html lang="en" data-content_root="./" >
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<title>Exercises week 37 — Applied Data Analysis and Machine Learning</title>
|
||||
|
||||
|
||||
|
||||
<script data-cfasync="false">
|
||||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||||
</script>
|
||||
|
||||
<!-- Loaded before other Sphinx assets -->
|
||||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||||
|
||||
|
||||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||||
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||||
|
||||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||||
|
||||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||||
<script>let toggleHintShow = 'Click to show';</script>
|
||||
<script>let toggleHintHide = 'Click to hide';</script>
|
||||
<script>let toggleOpenOnPrint = 'true';</script>
|
||||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek38';</script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
<link rel="next" title="Project 1 on Machine Learning, deadline October 6 (midnight), 2025" href="project1.html" />
|
||||
<link rel="prev" title="Week 37: Statistical interpretations and Resampling Methods" href="week37.html" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
</head>
|
||||
|
||||
|
||||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||||
|
||||
|
||||
|
||||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||||
|
||||
<div id="pst-scroll-pixel-helper"></div>
|
||||
|
||||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||||
|
||||
|
||||
<input type="checkbox"
|
||||
class="sidebar-toggle"
|
||||
id="pst-primary-sidebar-checkbox"/>
|
||||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||||
|
||||
<input type="checkbox"
|
||||
class="sidebar-toggle"
|
||||
id="pst-secondary-sidebar-checkbox"/>
|
||||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||||
|
||||
<div class="search-button__wrapper">
|
||||
<div class="search-button__overlay"></div>
|
||||
<div class="search-button__search-container">
|
||||
<form class="bd-search d-flex align-items-center"
|
||||
action="search.html"
|
||||
method="get">
|
||||
<i class="fa-solid fa-magnifying-glass"></i>
|
||||
<input type="search"
|
||||
class="form-control"
|
||||
name="q"
|
||||
id="search-input"
|
||||
placeholder="Search this book..."
|
||||
aria-label="Search this book..."
|
||||
autocomplete="off"
|
||||
autocorrect="off"
|
||||
autocapitalize="off"
|
||||
spellcheck="false"/>
|
||||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||||
</form></div>
|
||||
</div>
|
||||
|
||||
<div class="pst-async-banner-revealer d-none">
|
||||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||||
</div>
|
||||
|
||||
|
||||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||||
</header>
|
||||
|
||||
|
||||
<div class="bd-container">
|
||||
<div class="bd-container__inner bd-page-width">
|
||||
|
||||
|
||||
|
||||
<div class="bd-sidebar-primary bd-sidebar">
|
||||
|
||||
|
||||
|
||||
<div class="sidebar-header-items sidebar-primary__section">
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||||
<div class="sidebar-primary-item">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a class="navbar-brand logo" href="intro.html">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||||
|
||||
|
||||
</a></div>
|
||||
<div class="sidebar-primary-item">
|
||||
|
||||
<script>
|
||||
document.write(`
|
||||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="fa-solid fa-magnifying-glass"></i>
|
||||
<span class="search-button__default-text">Search</span>
|
||||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||||
</button>
|
||||
`);
|
||||
</script></div>
|
||||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||||
<div class="bd-toc-item navbar-nav active">
|
||||
|
||||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||||
<li class="toctree-l1">
|
||||
<a class="reference internal" href="intro.html">
|
||||
Applied Data Analysis and Machine Learning
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||||
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||||
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||||
<ul class="current nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</nav></div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||||
</div>
|
||||
|
||||
<div id="rtd-footer-container"></div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<main id="main-content" class="bd-main" role="main">
|
||||
|
||||
|
||||
|
||||
<div class="sbt-scroll-pixel-helper"></div>
|
||||
|
||||
<div class="bd-content">
|
||||
<div class="bd-article-container">
|
||||
|
||||
<div class="bd-header-article d-print-none">
|
||||
<div class="header-article-items header-article__inner">
|
||||
|
||||
<div class="header-article-items__start">
|
||||
|
||||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<span class="fa-solid fa-bars"></span>
|
||||
</button></div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div class="header-article-items__end">
|
||||
|
||||
<div class="header-article-item">
|
||||
|
||||
<div class="article-header-buttons">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="dropdown dropdown-download-buttons">
|
||||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||||
<i class="fas fa-download"></i>
|
||||
</button>
|
||||
<ul class="dropdown-menu">
|
||||
|
||||
|
||||
|
||||
<li><a href="_sources/exercisesweek38.ipynb" target="_blank"
|
||||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||||
title="Download source file"
|
||||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||||
>
|
||||
|
||||
|
||||
<span class="btn__icon-container">
|
||||
<i class="fas fa-file"></i>
|
||||
</span>
|
||||
<span class="btn__text-container">.ipynb</span>
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
<li>
|
||||
<button onclick="window.print()"
|
||||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||||
title="Print to PDF"
|
||||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||||
>
|
||||
|
||||
|
||||
<span class="btn__icon-container">
|
||||
<i class="fas fa-file-pdf"></i>
|
||||
</span>
|
||||
<span class="btn__text-container">.pdf</span>
|
||||
</button>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<button onclick="toggleFullScreen()"
|
||||
class="btn btn-sm btn-fullscreen-button"
|
||||
title="Fullscreen mode"
|
||||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||||
>
|
||||
|
||||
|
||||
<span class="btn__icon-container">
|
||||
<i class="fas fa-expand"></i>
|
||||
</span>
|
||||
|
||||
</button>
|
||||
|
||||
|
||||
|
||||
<script>
|
||||
document.write(`
|
||||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||||
</button>
|
||||
`);
|
||||
</script>
|
||||
|
||||
|
||||
<script>
|
||||
document.write(`
|
||||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||||
</button>
|
||||
`);
|
||||
</script>
|
||||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<span class="fa-solid fa-list"></span>
|
||||
</button>
|
||||
</div></div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div id="jb-print-docs-body" class="onlyprint">
|
||||
<h1>Exercises week 37</h1>
|
||||
<!-- Table of contents -->
|
||||
<div id="print-main-content">
|
||||
<div id="jb-print-toc">
|
||||
|
||||
<div>
|
||||
<h2> Contents </h2>
|
||||
</div>
|
||||
<nav aria-label="Page">
|
||||
<ul class="visible nav section-nav flex-column">
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#september-15-19">September 15-19</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#resampling-and-the-bias-variance-trade-off">Resampling and the Bias-Variance Trade-off</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-goals">Learning goals</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-expectation-values-for-ordinary-least-squares-expressions">Exercise 1: Expectation values for ordinary least squares expressions</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-expectation-values-for-ridge-regression">Exercise 2: Expectation values for Ridge regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-deriving-the-expression-for-the-bias-variance-trade-off">Exercise 3: Deriving the expression for the Bias-Variance Trade-off</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-computing-the-bias-and-variance">Exercise 4: Computing the Bias and Variance</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div id="searchbox"></div>
|
||||
<article class="bd-article">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="exercises-week-37">
|
||||
<h1>Exercises week 37<a class="headerlink" href="#exercises-week-37" title="Link to this heading">#</a></h1>
|
||||
<section id="september-15-19">
|
||||
<h2>September 15-19<a class="headerlink" href="#september-15-19" title="Link to this heading">#</a></h2>
|
||||
</section>
|
||||
<section id="resampling-and-the-bias-variance-trade-off">
|
||||
<h2>Resampling and the Bias-Variance Trade-off<a class="headerlink" href="#resampling-and-the-bias-variance-trade-off" title="Link to this heading">#</a></h2>
|
||||
</section>
|
||||
<section id="learning-goals">
|
||||
<h2>Learning goals<a class="headerlink" href="#learning-goals" title="Link to this heading">#</a></h2>
|
||||
<p>After completing these exercises, you will know how to</p>
|
||||
<ul class="simple">
|
||||
<li><p>Derive expectation and variances values related to linear regression</p></li>
|
||||
<li><p>Compute expectation and variances values related to linear regression</p></li>
|
||||
<li><p>Compute and evaluate the trade-off between bias and variance of a model</p></li>
|
||||
</ul>
|
||||
<p>This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of <a class="reference external" href="https://www.springer.com/gp/book/9780387848570">Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer</a>). The exercises are also a part of project 1 and can be reused in the theory part of the project.</p>
|
||||
<p>For more discussions on Ridge regression and calculation of expectation values, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.</p>
|
||||
<p>We assume that there exists a continuous function <span class="math notranslate nohighlight">\(f(\boldsymbol{x})\)</span> and a normal distributed error <span class="math notranslate nohighlight">\(\boldsymbol{\varepsilon}\sim N(0, \sigma^2)\)</span> which describes our data</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon}
|
||||
\]</div>
|
||||
<p>We further assume that this continous function can be modeled with a linear model <span class="math notranslate nohighlight">\(\mathbf{\tilde{y}}\)</span> of some features <span class="math notranslate nohighlight">\(\mathbf{X}\)</span>.</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{y} = \boldsymbol{\tilde{y}} + \boldsymbol{\varepsilon} = \boldsymbol{X}\boldsymbol{\beta} +\boldsymbol{\varepsilon}
|
||||
\]</div>
|
||||
<p>We therefore get that our data <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> has an expectation value <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{\beta}\)</span> and variance <span class="math notranslate nohighlight">\(\sigma^2\)</span>, that is <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> follows a normal distribution with mean value <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{\beta}\)</span> and variance <span class="math notranslate nohighlight">\(\sigma^2\)</span>.</p>
|
||||
</section>
|
||||
<section id="exercise-1-expectation-values-for-ordinary-least-squares-expressions">
|
||||
<h2>Exercise 1: Expectation values for ordinary least squares expressions<a class="headerlink" href="#exercise-1-expectation-values-for-ordinary-least-squares-expressions" title="Link to this heading">#</a></h2>
|
||||
<p><strong>a)</strong> With the expressions for the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span> show that</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbb{E}(\boldsymbol{\hat{\beta}_{OLS}}) = \boldsymbol{\beta}.
|
||||
\]</div>
|
||||
<p><strong>b)</strong> Show that the variance of <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span> is</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{Var}(\boldsymbol{\hat{\beta}_{OLS}}) = \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}.
|
||||
\]</div>
|
||||
<p>We can use the last expression when we define a <a class="reference external" href="https://en.wikipedia.org/wiki/Confidence_interval">confidence interval</a> for the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span>.
|
||||
A given parameter <span class="math notranslate nohighlight">\({\boldsymbol{\hat{\beta}_{OLS}}}_j\)</span> is given by the diagonal matrix element of the above matrix.</p>
|
||||
</section>
|
||||
<section id="exercise-2-expectation-values-for-ridge-regression">
|
||||
<h2>Exercise 2: Expectation values for Ridge regression<a class="headerlink" href="#exercise-2-expectation-values-for-ridge-regression" title="Link to this heading">#</a></h2>
|
||||
<p><strong>a)</strong> With the expressions for the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{Ridge}}\)</span> show that</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbb{E} \big[ \hat{\boldsymbol{\beta}}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\boldsymbol{\beta}
|
||||
\]</div>
|
||||
<p>We see that <span class="math notranslate nohighlight">\(\mathbb{E} \big[ \hat{\boldsymbol{\beta}}^{\mathrm{Ridge}} \big] \not= \mathbb{E} \big[\hat{\boldsymbol{\beta}}^{\mathrm{OLS}}\big ]\)</span> for any <span class="math notranslate nohighlight">\(\lambda > 0\)</span>.</p>
|
||||
<p><strong>b)</strong> Why do we say that Ridge regression gives a biased estimate? Is this a problem?</p>
|
||||
<p><strong>c)</strong> Show that the variance is</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{Var}[\hat{\boldsymbol{\beta}}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T}\mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}
|
||||
\]</div>
|
||||
<p>We see that if the parameter <span class="math notranslate nohighlight">\(\lambda\)</span> goes to infinity then the variance of the Ridge parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> goes to zero.</p>
|
||||
</section>
|
||||
<section id="exercise-3-deriving-the-expression-for-the-bias-variance-trade-off">
|
||||
<h2>Exercise 3: Deriving the expression for the Bias-Variance Trade-off<a class="headerlink" href="#exercise-3-deriving-the-expression-for-the-bias-variance-trade-off" title="Link to this heading">#</a></h2>
|
||||
<p>The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.</p>
|
||||
<p>The parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span> are found by optimizing the mean squared error via the so-called cost function</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
C(\boldsymbol{X},\boldsymbol{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]
|
||||
\]</div>
|
||||
<p><strong>a)</strong> Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a
|
||||
term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function <span class="math notranslate nohighlight">\(f\)</span> with the model <span class="math notranslate nohighlight">\({\bf y}\)</span>.
|
||||
show that</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathrm{Bias}[\tilde{y}]+\mathrm{var}[\tilde{y}]+\sigma^2,
|
||||
\]</div>
|
||||
<p>with</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathrm{Bias}[\tilde{y}]=\mathbb{E}\left[\left(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right],
|
||||
\]</div>
|
||||
<p>and</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathrm{var}[\tilde{y}]=\mathbb{E}\left[\left(\tilde{\boldsymbol{y}}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]\right)^2\right]=\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2.
|
||||
\]</div>
|
||||
<p><strong>b)</strong> Explain what the terms mean and discuss their interpretations.</p>
|
||||
</section>
|
||||
<section id="exercise-4-computing-the-bias-and-variance">
|
||||
<h2>Exercise 4: Computing the Bias and Variance<a class="headerlink" href="#exercise-4-computing-the-bias-and-variance" title="Link to this heading">#</a></h2>
|
||||
<p>Before you compute the bias and variance of a real model for different complexities, let’s for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.</p>
|
||||
<p><strong>a)</strong> Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error.</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
|
||||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||||
<span class="n">bootstraps</span> <span class="o">=</span> <span class="mi">1000</span>
|
||||
|
||||
<span class="n">predictions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">bootstraps</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">+</span> <span class="mi">10</span>
|
||||
<span class="n">targets</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">bootstraps</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span>
|
||||
|
||||
<span class="n">mse</span> <span class="o">=</span> <span class="o">...</span>
|
||||
<span class="n">bias</span> <span class="o">=</span> <span class="o">...</span>
|
||||
<span class="n">variance</span> <span class="o">=</span> <span class="o">...</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>b)</strong> Change the prediction values in some way to increase the bias while decreasing the variance.</p>
|
||||
<p><strong>c)</strong> Change the prediction values in some way to increase the variance while decreasing the bias.</p>
|
||||
<p><strong>d)</strong> Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model.</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">PolynomialFeatures</span> <span class="c1"># use the fit_transform method of the created object!</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.utils</span> <span class="kn">import</span> <span class="n">resample</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||||
<span class="n">bootstraps</span> <span class="o">=</span> <span class="mi">1000</span>
|
||||
|
||||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span>
|
||||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">)</span>
|
||||
|
||||
<span class="n">biases</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">variances</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">mses</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="c1">#for p in range(1, 5):</span>
|
||||
<span class="c1"># predictions = ...</span>
|
||||
<span class="c1"># targets = ...</span>
|
||||
<span class="c1"># for b in range(bootstraps):</span>
|
||||
<span class="c1"># x_sample, y_sample = ...</span>
|
||||
<span class="c1"># X = ...</span>
|
||||
<span class="c1"># X_train, X_test, y_train, y_test = ...</span>
|
||||
<span class="c1">#</span>
|
||||
<span class="c1"># predictions[b, :] = </span>
|
||||
<span class="c1"># targets[b, :] = </span>
|
||||
<span class="c1"># </span>
|
||||
<span class="c1"># biases.append(...)</span>
|
||||
<span class="c1"># variances.append(...)</span>
|
||||
<span class="c1"># mses.append(...)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p><strong>e)</strong> Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).</p>
|
||||
<p><strong>f)</strong> Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting).</p>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
<script type="text/x-thebe-config">
|
||||
{
|
||||
requestKernel: true,
|
||||
binderOptions: {
|
||||
repo: "binder-examples/jupyter-stacks-datascience",
|
||||
ref: "master",
|
||||
},
|
||||
codeMirrorConfig: {
|
||||
theme: "abcdef",
|
||||
mode: "python"
|
||||
},
|
||||
kernelOptions: {
|
||||
name: "python3",
|
||||
path: "./."
|
||||
},
|
||||
predefinedOutput: true
|
||||
}
|
||||
</script>
|
||||
<script>kernelName = 'python3'</script>
|
||||
|
||||
</article>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<footer class="prev-next-footer d-print-none">
|
||||
|
||||
<div class="prev-next-area">
|
||||
<a class="left-prev"
|
||||
href="week37.html"
|
||||
title="previous page">
|
||||
<i class="fa-solid fa-angle-left"></i>
|
||||
<div class="prev-next-info">
|
||||
<p class="prev-next-subtitle">previous</p>
|
||||
<p class="prev-next-title">Week 37: Statistical interpretations and Resampling Methods</p>
|
||||
</div>
|
||||
</a>
|
||||
<a class="right-next"
|
||||
href="project1.html"
|
||||
title="next page">
|
||||
<div class="prev-next-info">
|
||||
<p class="prev-next-subtitle">next</p>
|
||||
<p class="prev-next-title">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</p>
|
||||
</div>
|
||||
<i class="fa-solid fa-angle-right"></i>
|
||||
</a>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||||
|
||||
|
||||
<div class="sidebar-secondary-item">
|
||||
<div class="page-toc tocsection onthispage">
|
||||
<i class="fa-solid fa-list"></i> Contents
|
||||
</div>
|
||||
<nav class="bd-toc-nav page-toc">
|
||||
<ul class="visible nav section-nav flex-column">
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#september-15-19">September 15-19</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#resampling-and-the-bias-variance-trade-off">Resampling and the Bias-Variance Trade-off</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-goals">Learning goals</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-expectation-values-for-ordinary-least-squares-expressions">Exercise 1: Expectation values for ordinary least squares expressions</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-expectation-values-for-ridge-regression">Exercise 2: Expectation values for Ridge regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-deriving-the-expression-for-the-bias-variance-trade-off">Exercise 3: Deriving the expression for the Bias-Variance Trade-off</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-computing-the-bias-and-variance">Exercise 4: Computing the Bias and Variance</a></li>
|
||||
</ul>
|
||||
</nav></div>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
</div>
|
||||
<footer class="bd-footer-content">
|
||||
|
||||
<div class="bd-footer-content__inner container">
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
<p class="component-author">
|
||||
By Morten Hjorth-Jensen
|
||||
</p>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
|
||||
<p class="copyright">
|
||||
|
||||
© Copyright 2023.
|
||||
<br/>
|
||||
|
||||
</p>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
</div>
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||||
|
||||
<footer class="bd-footer">
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -227,6 +227,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -231,6 +231,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
Binary file not shown.
@@ -62,7 +62,7 @@
|
||||
<script>DOCUMENTATION_OPTIONS.pagename = 'project1';</script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
<link rel="prev" title="Exercises week 37" href="exercisesweek37.html" />
|
||||
<link rel="prev" title="Exercises week 37" href="exercisesweek38.html" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
</head>
|
||||
@@ -229,6 +229,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="current nav bd-sidenav">
|
||||
@@ -750,7 +752,7 @@ of code developers and contributors keeps increasing.</p>
|
||||
|
||||
<div class="prev-next-area">
|
||||
<a class="left-prev"
|
||||
href="exercisesweek37.html"
|
||||
href="exercisesweek38.html"
|
||||
title="previous page">
|
||||
<i class="fa-solid fa-angle-left"></i>
|
||||
<div class="prev-next-info">
|
||||
|
||||
@@ -228,6 +228,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -229,6 +229,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -228,6 +228,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -228,6 +228,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
@@ -230,6 +230,8 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Week 36: Linear Regression and Gradient descent</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b941c35",
|
||||
"id": "8e6632a0",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -14,7 +14,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dc05b096",
|
||||
"id": "82705c4f",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -27,7 +27,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2cf07405",
|
||||
"id": "921bf331",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -46,7 +46,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3c139edb",
|
||||
"id": "adff65d5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -58,7 +58,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aad4cfac",
|
||||
"id": "70418b3d",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6682282f",
|
||||
"id": "11a3cf73",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -83,7 +83,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89e2f4c4",
|
||||
"id": "04a06b51",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -99,7 +99,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b06d4e53",
|
||||
"id": "408db3d9",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -120,7 +120,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "63796480",
|
||||
"id": "37fb732c",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -140,12 +140,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "80748600",
|
||||
"id": "d861e1e3",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"Fill in the necessary details.\n",
|
||||
"Fill in the necessary details. Do we need to center the $y$-values? \n",
|
||||
"\n",
|
||||
"After this preprocessing, each column of $\\boldsymbol{X}_{\\mathrm{norm}}$ has mean zero and standard deviation $1$\n",
|
||||
"and $\\boldsymbol{y}_{\\mathrm{centered}}$ has mean 0. This makes the optimization landscape\n",
|
||||
@@ -156,7 +156,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "92751e5f",
|
||||
"id": "b3e774d0",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -168,7 +168,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "aedfbd7a",
|
||||
"id": "d5dc7708",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -179,7 +179,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "5d1288fa",
|
||||
"id": "4c9c86ac",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -187,7 +187,9 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set regularization parameter, either a single value or a vector of values\n",
|
||||
"lambda = ?\n",
|
||||
"# Note that lambda is a python keyword. The lambda keyword is used to create small, single-expression functions without a formal name. These are often called \"anonymous functions\" or \"lambda functions.\"\n",
|
||||
"lam = ?\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Analytical form for OLS and Ridge solution: theta_Ridge = (X^T X + lambda * I)^{-1} X^T y and theta_OLS = (X^T X)^{-1} X^T y\n",
|
||||
"I = np.eye(n_features)\n",
|
||||
@@ -200,7 +202,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "628f5e89",
|
||||
"id": "eeae00fd",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -214,7 +216,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f115ba4e",
|
||||
"id": "e1c215d5",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -226,7 +228,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a9b5189c",
|
||||
"id": "587dd3dc",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -238,7 +240,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a3969ff6",
|
||||
"id": "bfa34697",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -258,7 +260,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "34d87303",
|
||||
"id": "49245f55",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -273,19 +275,8 @@
|
||||
"# Initialize weights for gradient descent\n",
|
||||
"theta = np.zeros(n_features)\n",
|
||||
"\n",
|
||||
"# Arrays to store history for plotting\n",
|
||||
"cost_history = np.zeros(num_iters)\n",
|
||||
"\n",
|
||||
"# Gradient descent loop\n",
|
||||
"m = n_samples # number of data points\n",
|
||||
"for t in range(num_iters):\n",
|
||||
" # Compute prediction error\n",
|
||||
" error = X_norm.dot(theta) - y_centered \n",
|
||||
" # Compute cost for OLS and Ridge (MSE + regularization for Ridge) for monitoring\n",
|
||||
" cost_OLS = ?\n",
|
||||
" cost_Ridge = ?\n",
|
||||
" # You could add a history for both methods (optional)\n",
|
||||
" cost_history[t] = ?\n",
|
||||
" # Compute gradients for OSL and Ridge\n",
|
||||
" grad_OLS = ?\n",
|
||||
" grad_Ridge = ?\n",
|
||||
@@ -302,31 +293,33 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "989f70bb",
|
||||
"id": "f3f43f2c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"### 4a)\n",
|
||||
"\n",
|
||||
"Discuss the results as function of the learning rate parameters and the number of iterations."
|
||||
"Write first a gradient descent code for OLS only using the above template.\n",
|
||||
"Discuss the results as function of the learning rate parameters and the number of iterations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "370b2dad",
|
||||
"id": "9ba303be",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
"source": [
|
||||
"### 4b)\n",
|
||||
"\n",
|
||||
"Write then a similar code for Ridge regression using the above template.\n",
|
||||
"Try to add a stopping parameter as function of the number iterations and the difference between the new and old $\\theta$ values. How would you define a stopping criterion?"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ef197cd7",
|
||||
"id": "78362c6c",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -352,7 +345,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "4ccc2f65",
|
||||
"id": "8be1cebe",
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
@@ -381,7 +374,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "00e279ef",
|
||||
"id": "e2693666",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -395,7 +388,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c910b3f4",
|
||||
"id": "bc954d12",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
@@ -407,7 +400,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89e6e040",
|
||||
"id": "6534b610",
|
||||
"metadata": {
|
||||
"editable": true
|
||||
},
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1da77599",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercises week 37\n",
|
||||
"## September 15-19\n",
|
||||
"\n",
|
||||
"## Resampling and the Bias-Variance Trade-off"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e9f27b0e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Learning goals\n",
|
||||
"\n",
|
||||
"After completing these exercises, you will know how to\n",
|
||||
"- Derive expectation and variances values related to linear regression\n",
|
||||
"- Compute expectation and variances values related to linear regression\n",
|
||||
"- Compute and evaluate the trade-off between bias and variance of a model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "984af8e3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This week deals with various mean values and variances in linear regression methods (here it may be useful to look up chapter 3, equation (3.8) of [Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer](https://www.springer.com/gp/book/9780387848570)). The exercises are also a part of project 1 and can be reused in the theory part of the project.\n",
|
||||
"\n",
|
||||
"For more discussions on Ridge regression and calculation of expectation values, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n",
|
||||
"\n",
|
||||
"We assume that there exists a continuous function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim N(0, \\sigma^2)$ which describes our data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c16f7d0e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9fcf981a",
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"We further assume that this continous function can be modeled with a linear model $\\mathbf{\\tilde{y}}$ of some features $\\mathbf{X}$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4189366",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\boldsymbol{y} = \\boldsymbol{\\tilde{y}} + \\boldsymbol{\\varepsilon} = \\boldsymbol{X}\\boldsymbol{\\beta} +\\boldsymbol{\\varepsilon}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f4fca21b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We therefore get that our data $\\boldsymbol{y}$ has an expectation value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$, that is $\\boldsymbol{y}$ follows a normal distribution with mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5de0c7e6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 1: Expectation values for ordinary least squares expressions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d878c699",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "08b7007d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\boldsymbol{\\beta}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "46e93394",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Show that the variance of $\\boldsymbol{\\hat{\\beta}_{OLS}}$ is"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "be1b65be",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbf{Var}(\\boldsymbol{\\hat{\\beta}_{OLS}}) = \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d2143684",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can use the last expression when we define a [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval) for the parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$. \n",
|
||||
"A given parameter ${\\boldsymbol{\\hat{\\beta}_{OLS}}}_j$ is given by the diagonal matrix element of the above matrix."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f5c2dc22",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 2: Expectation values for Ridge regression"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3893e3e7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** With the expressions for the optimal parameters $\\boldsymbol{\\hat{\\beta}_{Ridge}}$ show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "79dc571f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "028209a1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We see that $\\mathbb{E} \\big[ \\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}} \\big] \\not= \\mathbb{E} \\big[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}\\big ]$ for any $\\lambda > 0$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "65f6f914",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Why do we say that Ridge regression gives a biased estimate? Is this a problem?"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b4e721fc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**c)** Show that the variance is"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "090eb1e1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbf{Var}[\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T}\\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6b8e8697",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We see that if the parameter $\\lambda$ goes to infinity then the variance of the Ridge parameters $\\boldsymbol{\\beta}$ goes to zero."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "74bc300b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 3: Deriving the expression for the Bias-Variance Trade-off"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eeb86010",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The aim of this exercise is to derive the equations for the bias-variance tradeoff to be used in project 1.\n",
|
||||
"\n",
|
||||
"The parameters $\\boldsymbol{\\hat{\\beta}_{OLS}}$ are found by optimizing the mean squared error via the so-called cost function"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "522a0d1d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "831db06c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**a)** Show that you can rewrite this in terms of a term which contains the variance of the model itself (the so-called variance term), a\n",
|
||||
"term which measures the deviation from the true data and the mean value of the model (the bias term) and finally the variance of the noise Note that in order to be able to evaluate the bias them, you will need to approximate the function $f$ with the model ${\\bf y}$.\n",
|
||||
"show that"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8cc52b3c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathrm{Bias}[\\tilde{y}]+\\mathrm{var}[\\tilde{y}]+\\sigma^2,\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8cb50416",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"with"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e49bdbb4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathrm{Bias}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right],\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eca5554a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"and"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b1054343",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathrm{var}[\\tilde{y}]=\\mathbb{E}\\left[\\left(\\tilde{\\boldsymbol{y}}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]\\right)^2\\right]=\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "70fbfcd7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Explain what the terms mean and discuss their interpretations."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b8f8b9d1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Exercise 4: Computing the Bias and Variance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9e012430",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Before you compute the bias and variance of a real model for different complexities, let's for now assume that you have sampled predictions and targets for a single model complexity using bootstrap resampling.\n",
|
||||
"\n",
|
||||
"**a)** Using the expression above, compute the mean squared error, bias and variance of the given data. Check that the sum of the bias and variance correctly gives (approximately) the mean squared error."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"id": "b5bf581c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"n = 100\n",
|
||||
"bootstraps = 1000\n",
|
||||
"\n",
|
||||
"predictions = np.random.rand(bootstraps, n) * 10 + 10\n",
|
||||
"targets = np.random.rand(bootstraps, n)\n",
|
||||
"\n",
|
||||
"mse = ...\n",
|
||||
"bias = ...\n",
|
||||
"variance = ..."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7b1dc621",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**b)** Change the prediction values in some way to increase the bias while decreasing the variance.\n",
|
||||
"\n",
|
||||
"**c)** Change the prediction values in some way to increase the variance while decreasing the bias."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8da63362",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**d)** Perform a bias-variance analysis of a polynomial OLS model fit to a one-dimensional function by computing and plotting the bias and variances values as a function of the polynomial degree of your model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "dd5855e4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures # use the fit_transform method of the created object!\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"from sklearn.metrics import mean_squared_error\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.utils import resample"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7e35fa37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"n = 100\n",
|
||||
"bootstraps = 1000\n",
|
||||
"\n",
|
||||
"x = np.linspace(-3, 3, n)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1)\n",
|
||||
"\n",
|
||||
"biases = []\n",
|
||||
"variances = []\n",
|
||||
"mses = []\n",
|
||||
"\n",
|
||||
"#for p in range(1, 5):\n",
|
||||
"# predictions = ...\n",
|
||||
"# targets = ...\n",
|
||||
"# for b in range(bootstraps):\n",
|
||||
"# x_sample, y_sample = ...\n",
|
||||
"# X = ...\n",
|
||||
"# X_train, X_test, y_train, y_test = ...\n",
|
||||
"#\n",
|
||||
"# predictions[b, :] = \n",
|
||||
"# targets[b, :] = \n",
|
||||
"# \n",
|
||||
"# biases.append(...)\n",
|
||||
"# variances.append(...)\n",
|
||||
"# mses.append(...)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "253b8461",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**e)** Discuss the bias-variance trade-off as function of your model complexity (the degree of the polynomial).\n",
|
||||
"\n",
|
||||
"**f)** Compute and discuss the bias and variance as function of the number of data points (choose a suitable polynomial degree to show something interesiting)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user