498 lines
27 KiB
HTML
498 lines
27 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 36: Statistical interpretation of Linear Regression and Resampling techniques">
|
|
<title>Week 36: Statistical interpretation of Linear Regression and Resampling techniques</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 36', 2, None, 'plans-for-week-36'),
|
|
('Thursday September 8', 2, None, 'thursday-september-8'),
|
|
('Summary from last Week and discussion of SVD, Ridge and Lasso '
|
|
'regression with examples',
|
|
2,
|
|
None,
|
|
'summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples'),
|
|
('Linear Regression and the SVD',
|
|
2,
|
|
None,
|
|
'linear-regression-and-the-svd'),
|
|
('What does it mean?', 2, None, 'what-does-it-mean'),
|
|
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
|
|
2,
|
|
None,
|
|
'and-finally-boldsymbol-x-boldsymbol-x-t'),
|
|
('Code for SVD and Inversion of Matrices',
|
|
2,
|
|
None,
|
|
'code-for-svd-and-inversion-of-matrices'),
|
|
('Inverse of Rectangular Matrix',
|
|
2,
|
|
None,
|
|
'inverse-of-rectangular-matrix'),
|
|
('Ridge and LASSO Regression',
|
|
2,
|
|
None,
|
|
'ridge-and-lasso-regression'),
|
|
('From OLS to Ridge and Lasso',
|
|
2,
|
|
None,
|
|
'from-ols-to-ridge-and-lasso'),
|
|
('Deriving the Ridge Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-ridge-regression-equations'),
|
|
('Note on Scikit-Learn', 2, None, 'note-on-scikit-learn'),
|
|
('Comparison with OLS', 2, None, 'comparison-with-ols'),
|
|
('SVD analysis', 2, None, 'svd-analysis'),
|
|
('Interpreting the Ridge results',
|
|
2,
|
|
None,
|
|
'interpreting-the-ridge-results'),
|
|
('More interpretations', 2, None, 'more-interpretations'),
|
|
('Deriving the Lasso Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-lasso-regression-equations'),
|
|
('Simple example to illustrate Ordinary Least Squares, Ridge and '
|
|
'Lasso Regression',
|
|
2,
|
|
None,
|
|
'simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression'),
|
|
('Ridge Regression', 2, None, 'ridge-regression'),
|
|
('Lasso Regression', 2, None, 'lasso-regression'),
|
|
('Yet another Example', 2, None, 'yet-another-example'),
|
|
('The OLS case', 2, None, 'the-ols-case'),
|
|
('The Ridge case', 2, None, 'the-ridge-case'),
|
|
('Writing the Cost Function',
|
|
2,
|
|
None,
|
|
'writing-the-cost-function'),
|
|
('Lasso case', 2, None, 'lasso-case'),
|
|
('The first Case', 2, None, 'the-first-case'),
|
|
('Simple code for solving the above problem',
|
|
2,
|
|
None,
|
|
'simple-code-for-solving-the-above-problem'),
|
|
('With Lasso Regression', 2, None, 'with-lasso-regression'),
|
|
('Another Example, now with a polynomial fit',
|
|
2,
|
|
None,
|
|
'another-example-now-with-a-polynomial-fit'),
|
|
('To think about, first part',
|
|
2,
|
|
None,
|
|
'to-think-about-first-part'),
|
|
('More thinking', 2, None, 'more-thinking'),
|
|
('Still thinking', 2, None, 'still-thinking'),
|
|
('What does centering (subtracting the mean values) mean '
|
|
'mathematically?',
|
|
2,
|
|
None,
|
|
'what-does-centering-subtracting-the-mean-values-mean-mathematically'),
|
|
('Further Manipulations', 2, None, 'further-manipulations'),
|
|
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
|
('Linear Regression code, Intercept handling first',
|
|
2,
|
|
None,
|
|
'linear-regression-code-intercept-handling-first'),
|
|
('Code Examples', 2, None, 'code-examples'),
|
|
('Taking out the mean', 2, None, 'taking-out-the-mean'),
|
|
('Friday September 9', 2, None, 'friday-september-9'),
|
|
('Linking the regression analysis with a statistical '
|
|
'interpretation',
|
|
2,
|
|
None,
|
|
'linking-the-regression-analysis-with-a-statistical-interpretation'),
|
|
('Assumptions made', 2, None, 'assumptions-made'),
|
|
('Expectation value and variance',
|
|
2,
|
|
None,
|
|
'expectation-value-and-variance'),
|
|
('Expectation value and variance for $\\boldsymbol{\\beta}$',
|
|
2,
|
|
None,
|
|
'expectation-value-and-variance-for-boldsymbol-beta'),
|
|
('Deriving OLS from a probability distribution',
|
|
2,
|
|
None,
|
|
'deriving-ols-from-a-probability-distribution'),
|
|
('Independent and Identically Distrubuted (iid)',
|
|
2,
|
|
None,
|
|
'independent-and-identically-distrubuted-iid'),
|
|
('Maximum Likelihood Estimation (MLE)',
|
|
2,
|
|
None,
|
|
'maximum-likelihood-estimation-mle'),
|
|
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
|
("More basic Statistics and Bayes' theorem",
|
|
2,
|
|
None,
|
|
'more-basic-statistics-and-bayes-theorem'),
|
|
('Marginal Probability', 2, None, 'marginal-probability'),
|
|
('Conditional Probability', 2, None, 'conditional-probability'),
|
|
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
|
("Interpretations of Bayes' Theorem",
|
|
2,
|
|
None,
|
|
'interpretations-of-bayes-theorem'),
|
|
("Example of Usage of Bayes' theorem",
|
|
2,
|
|
None,
|
|
'example-of-usage-of-bayes-theorem'),
|
|
('Doing it correctly', 2, None, 'doing-it-correctly'),
|
|
("Bayes' Theorem and Ridge and Lasso Regression",
|
|
2,
|
|
None,
|
|
'bayes-theorem-and-ridge-and-lasso-regression'),
|
|
('Test Function for what happens with OLS, Ridge and Lasso',
|
|
2,
|
|
None,
|
|
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
|
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
|
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
|
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
|
('Exercise 1: mean values and variances in linear regression',
|
|
2,
|
|
None,
|
|
'exercise-1-mean-values-and-variances-in-linear-regression'),
|
|
('Exercise 2: Adding Ridge and Lasso Regression',
|
|
2,
|
|
None,
|
|
'exercise-2-adding-ridge-and-lasso-regression')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week36-bs.html">Week 36: Statistical interpretation of Linear Regression and Resampling techniques</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week36-bs001.html#plans-for-week-36" style="font-size: 80%;">Plans for week 36</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs002.html#thursday-september-8" style="font-size: 80%;">Thursday September 8</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs003.html#summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples" style="font-size: 80%;">Summary from last Week and discussion of SVD, Ridge and Lasso regression with examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs004.html#linear-regression-and-the-svd" style="font-size: 80%;">Linear Regression and the SVD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs005.html#what-does-it-mean" style="font-size: 80%;">What does it mean?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs006.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;">And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs007.html#code-for-svd-and-inversion-of-matrices" style="font-size: 80%;">Code for SVD and Inversion of Matrices</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs008.html#inverse-of-rectangular-matrix" style="font-size: 80%;">Inverse of Rectangular Matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs009.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs010.html#from-ols-to-ridge-and-lasso" style="font-size: 80%;">From OLS to Ridge and Lasso</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs011.html#deriving-the-ridge-regression-equations" style="font-size: 80%;">Deriving the Ridge Regression Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs012.html#note-on-scikit-learn" style="font-size: 80%;">Note on Scikit-Learn</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs013.html#comparison-with-ols" style="font-size: 80%;">Comparison with OLS</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs014.html#svd-analysis" style="font-size: 80%;">SVD analysis</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs015.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs016.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs017.html#deriving-the-lasso-regression-equations" style="font-size: 80%;">Deriving the Lasso Regression Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs018.html#simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression" style="font-size: 80%;">Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs019.html#ridge-regression" style="font-size: 80%;">Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs020.html#lasso-regression" style="font-size: 80%;">Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs021.html#yet-another-example" style="font-size: 80%;">Yet another Example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs022.html#the-ols-case" style="font-size: 80%;">The OLS case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs023.html#the-ridge-case" style="font-size: 80%;">The Ridge case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs024.html#writing-the-cost-function" style="font-size: 80%;">Writing the Cost Function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs025.html#lasso-case" style="font-size: 80%;">Lasso case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs026.html#the-first-case" style="font-size: 80%;">The first Case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs027.html#simple-code-for-solving-the-above-problem" style="font-size: 80%;">Simple code for solving the above problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs028.html#with-lasso-regression" style="font-size: 80%;">With Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs029.html#another-example-now-with-a-polynomial-fit" style="font-size: 80%;">Another Example, now with a polynomial fit</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs030.html#to-think-about-first-part" style="font-size: 80%;">To think about, first part</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs031.html#more-thinking" style="font-size: 80%;">More thinking</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs032.html#still-thinking" style="font-size: 80%;">Still thinking</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs033.html#what-does-centering-subtracting-the-mean-values-mean-mathematically" style="font-size: 80%;">What does centering (subtracting the mean values) mean mathematically?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs034.html#further-manipulations" style="font-size: 80%;">Further Manipulations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs035.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs036.html#linear-regression-code-intercept-handling-first" style="font-size: 80%;">Linear Regression code, Intercept handling first</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs037.html#code-examples" style="font-size: 80%;">Code Examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs038.html#taking-out-the-mean" style="font-size: 80%;">Taking out the mean</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs039.html#friday-september-9" style="font-size: 80%;">Friday September 9</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs040.html#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs041.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs042.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs043.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs044.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs045.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs046.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs047.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs048.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs049.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs050.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs051.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs052.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs053.html#example-of-usage-of-bayes-theorem" style="font-size: 80%;">Example of Usage of Bayes' theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs054.html#doing-it-correctly" style="font-size: 80%;">Doing it correctly</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs055.html#bayes-theorem-and-ridge-and-lasso-regression" style="font-size: 80%;">Bayes' Theorem and Ridge and Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs056.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs057.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs058.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
|
<!-- navigation toc: --> <li><a href="#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
|
<!-- navigation toc: --> <li><a href="#exercise-1-mean-values-and-variances-in-linear-regression" style="font-size: 80%;">Exercise 1: mean values and variances in linear regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="#exercise-2-adding-ridge-and-lasso-regression" style="font-size: 80%;">Exercise 2: Adding Ridge and Lasso Regression</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0059"></a>
|
|
<!-- !split -->
|
|
<h2 id="lasso-and-bayes" class="anchor">Lasso and Bayes </h2>
|
|
|
|
<p>To derive the Lasso cost function, we simply replace the Gaussian prior with an exponential distribution (<a href="https://en.wikipedia.org/wiki/Laplace_distribution" target="_self">Laplace in this case</a>) with zero mean value, that is</p>
|
|
|
|
$$
|
|
p(\boldsymbol{\beta})=\prod_{j=0}^{p-1}\exp{\left(-\frac{\vert\beta_j\vert}{\tau}\right)}.
|
|
$$
|
|
|
|
<p>Our posterior probability becomes then (omitting the normalization factor which is just a constant)</p>
|
|
$$
|
|
p(\boldsymbol{\beta}\vert\boldsymbol{D})=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}\prod_{j=0}^{p-1}\exp{\left(-\frac{\vert\beta_j\vert}{\tau}\right)}.
|
|
$$
|
|
|
|
<p>Taking the negative
|
|
logarithm of the posterior probability and leaving out the
|
|
constants terms that do not depend on \( \beta \), we have
|
|
</p>
|
|
|
|
$$
|
|
C(\boldsymbol{\beta}=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\frac{1}{\tau}\vert\vert\boldsymbol{\beta}\vert\vert_1,
|
|
$$
|
|
|
|
<p>and replacing \( 1/\tau \) with \( \lambda \) we have</p>
|
|
|
|
$$
|
|
C(\boldsymbol{\beta}=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1,
|
|
$$
|
|
|
|
<p>which is our Lasso cost function! </p>
|
|
|
|
<!-- --- begin exercise --- -->
|
|
<h2 id="exercise-1-mean-values-and-variances-in-linear-regression" class="anchor">Exercise 1: mean values and variances in linear regression </h2>
|
|
|
|
<p>This exercise deals with various mean values ad variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of <a href="https://www.springer.com/gp/book/9780387848570" target="_self">Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer</a>).</p>
|
|
|
|
<p>The assumption we have made is
|
|
that there exists a function \( f(\boldsymbol{x}) \) and a normal distributed error \( \boldsymbol{\varepsilon}\sim \mathcal{N}(0, \sigma^2) \)
|
|
which describes our data
|
|
</p>
|
|
$$
|
|
\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon}
|
|
$$
|
|
|
|
<p>We then approximate this function with our model from the solution of the linear regression equations (ordinary least squares OLS), that is our
|
|
function \( f \) is approximated by \( \boldsymbol{\tilde{y}} \) where we minimized \( (\boldsymbol{y}-\boldsymbol{\tilde{y}})^2 \), with
|
|
</p>
|
|
$$
|
|
\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta}.
|
|
$$
|
|
|
|
<p>The matrix \( \boldsymbol{X} \) is the so-called design matrix.</p>
|
|
|
|
<!-- --- begin subexercise --- -->
|
|
<p>
|
|
<b>a)</b>
|
|
Show that the expectation value of \( \boldsymbol{y} \) for a given element \( i \)
|
|
</p>
|
|
$$
|
|
\begin{align*}
|
|
\mathbb{E}(y_i) & =\mathbf{X}_{i, \ast} \, \beta,
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>and that
|
|
its variance is
|
|
</p>
|
|
$$
|
|
\begin{align*} \mbox{Var}(y_i) & = \sigma^2.
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>Hence, \( y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta}, \sigma^2) \), that is \( \boldsymbol{y} \) follows a normal distribution with
|
|
mean value \( \boldsymbol{X}\boldsymbol{\beta} \) and variance \( \sigma^2 \).
|
|
</p>
|
|
|
|
<!-- --- end subexercise --- -->
|
|
|
|
<!-- --- begin subexercise --- -->
|
|
<p>
|
|
<b>b)</b>
|
|
With the OLS expressions for the parameters \( \boldsymbol{\beta} \) show that
|
|
</p>
|
|
$$
|
|
\mathbb{E}(\boldsymbol{\beta}) = \boldsymbol{\beta}.
|
|
$$
|
|
|
|
|
|
<!-- --- end subexercise --- -->
|
|
|
|
<!-- --- begin subexercise --- -->
|
|
<p>
|
|
<b>c)</b>
|
|
Show finally that the variance of \( \boldsymbol{\beta} \) is
|
|
</p>
|
|
$$
|
|
\begin{eqnarray*}
|
|
\mbox{Var}(\boldsymbol{\beta}) & = & \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}.
|
|
\end{eqnarray*}
|
|
$$
|
|
|
|
|
|
<!-- --- end subexercise --- -->
|
|
|
|
<!-- --- end exercise --- -->
|
|
|
|
<!-- --- begin exercise --- -->
|
|
<h2 id="exercise-2-adding-ridge-and-lasso-regression" class="anchor">Exercise 2: Adding Ridge and Lasso Regression </h2>
|
|
|
|
<p>This exercise is a continuation of the exercises from week 35.</p>
|
|
|
|
<p>We will
|
|
use the same function to generate our data set, still staying with a
|
|
simple function \( y(x) \) which we want to fit using linear regression,
|
|
but now extending the analysis to include the Ridge and the Lasso
|
|
regression methods.
|
|
</p>
|
|
|
|
<p>We will thus again generate our own dataset for a function \( y(x) \) where
|
|
\( x \in [0,1] \) and defined by random numbers computed with the uniform
|
|
distribution. The function \( y \) is a quadratic polynomial in \( x \) with
|
|
added stochastic noise according to the normal distribution \( \cal{N}(0,1) \).
|
|
</p>
|
|
|
|
<p>The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points).</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;">x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>)
|
|
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+5*</span>x<span style="color: #666666">*</span>x<span style="color: #666666">+0.1*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- --- begin subexercise --- -->
|
|
<p>
|
|
<b>a)</b>
|
|
Write your own code for the Ridge method (see chapter 3.4 of Hastie <em>et al.</em>, equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \).
|
|
</p>
|
|
|
|
<!-- --- end subexercise --- -->
|
|
|
|
<!-- --- begin subexercise --- -->
|
|
<p>
|
|
<b>b)</b>
|
|
Our next step is to study the variance of the parameters \( \beta_1 \) and \( \beta_2 \) (assuming that we are parameterizing our function with a second-order polynomial). We will use standard linear regression and the Ridge regression. You can now opt for either writing your own function or using <b>Scikit-Learn</b> to find the parameters \( \beta \). From your results calculate the variance of these parameters (recall that this is equal to the diagonal elements of the matrix \( (\hat{X}^T\hat{X})+\lambda\hat{I})^{-1} \)). Discuss the results of these variances as functions of \( \lambda \). In particular, try to link your discussion with the discussion in Hastie <em>et al.</em> and their figures 3.10 and 3.11. <b>Scikit-Learn</b> may not provide the variance of the parameters \( \beta \). This needs to be checked. With your own code you can however do so.
|
|
</p>
|
|
|
|
<!-- --- end subexercise --- -->
|
|
|
|
<!-- --- end exercise --- -->
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week36-bs058.html">«</a></li>
|
|
<li><a href="._week36-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week36-bs051.html">52</a></li>
|
|
<li><a href="._week36-bs052.html">53</a></li>
|
|
<li><a href="._week36-bs053.html">54</a></li>
|
|
<li><a href="._week36-bs054.html">55</a></li>
|
|
<li><a href="._week36-bs055.html">56</a></li>
|
|
<li><a href="._week36-bs056.html">57</a></li>
|
|
<li><a href="._week36-bs057.html">58</a></li>
|
|
<li><a href="._week36-bs058.html">59</a></li>
|
|
<li class="active"><a href="._week36-bs059.html">60</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|