Files
FYS-STK4155/doc/pub/week36/html/._week36-bs039.html
T
Morten Hjorth-Jensen ab45b7492c update
2023-05-29 22:09:43 +02:00

386 lines
23 KiB
HTML

<!--
HTML file automatically generated from DocOnce source
(https://github.com/doconce/doconce/)
doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 36: Statistical interpretation of Linear Regression and Resampling techniques">
<title>Week 36: Statistical interpretation of Linear Regression and Resampling techniques</title>
<!-- Bootstrap style: bootstrap -->
<!-- doconce format html week36.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week36-bs --no_mako -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plans for week 36', 2, None, 'plans-for-week-36'),
('Summary from last Week and discussion of SVD, Ridge and Lasso '
'regression with examples',
2,
None,
'summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples'),
('Linear Regression and the SVD',
2,
None,
'linear-regression-and-the-svd'),
('What does it mean?', 2, None, 'what-does-it-mean'),
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Code for SVD and Inversion of Matrices',
2,
None,
'code-for-svd-and-inversion-of-matrices'),
('Inverse of Rectangular Matrix',
2,
None,
'inverse-of-rectangular-matrix'),
('Ridge and LASSO Regression',
2,
None,
'ridge-and-lasso-regression'),
('From OLS to Ridge and Lasso',
2,
None,
'from-ols-to-ridge-and-lasso'),
('Deriving the Ridge Regression Equations',
2,
None,
'deriving-the-ridge-regression-equations'),
('Note on Scikit-Learn', 2, None, 'note-on-scikit-learn'),
('Comparison with OLS', 2, None, 'comparison-with-ols'),
('SVD analysis', 2, None, 'svd-analysis'),
('Interpreting the Ridge results',
2,
None,
'interpreting-the-ridge-results'),
('More interpretations', 2, None, 'more-interpretations'),
('Deriving the Lasso Regression Equations',
2,
None,
'deriving-the-lasso-regression-equations'),
('Simple example to illustrate Ordinary Least Squares, Ridge and '
'Lasso Regression',
2,
None,
'simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression'),
('Ridge Regression', 2, None, 'ridge-regression'),
('Lasso Regression', 2, None, 'lasso-regression'),
('Yet another Example', 2, None, 'yet-another-example'),
('The OLS case', 2, None, 'the-ols-case'),
('The Ridge case', 2, None, 'the-ridge-case'),
('Writing the Cost Function',
2,
None,
'writing-the-cost-function'),
('Lasso case', 2, None, 'lasso-case'),
('The first Case', 2, None, 'the-first-case'),
('Simple code for solving the above problem',
2,
None,
'simple-code-for-solving-the-above-problem'),
('With Lasso Regression', 2, None, 'with-lasso-regression'),
('Another Example, now with a polynomial fit',
2,
None,
'another-example-now-with-a-polynomial-fit'),
('To think about, first part',
2,
None,
'to-think-about-first-part'),
('More thinking', 2, None, 'more-thinking'),
('Still thinking', 2, None, 'still-thinking'),
('What does centering (subtracting the mean values) mean '
'mathematically?',
2,
None,
'what-does-centering-subtracting-the-mean-values-mean-mathematically'),
('Further Manipulations', 2, None, 'further-manipulations'),
('Wrapping it up', 2, None, 'wrapping-it-up'),
('Linear Regression code, Intercept handling first',
2,
None,
'linear-regression-code-intercept-handling-first'),
('Code Examples', 2, None, 'code-examples'),
('Taking out the mean', 2, None, 'taking-out-the-mean'),
('Friday September 9', 2, None, 'friday-september-9'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
'linking-the-regression-analysis-with-a-statistical-interpretation'),
('Assumptions made', 2, None, 'assumptions-made'),
('Expectation value and variance',
2,
None,
'expectation-value-and-variance'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
'expectation-value-and-variance-for-boldsymbol-beta'),
('Deriving OLS from a probability distribution',
2,
None,
'deriving-ols-from-a-probability-distribution'),
('Independent and Identically Distrubuted (iid)',
2,
None,
'independent-and-identically-distrubuted-iid'),
('Maximum Likelihood Estimation (MLE)',
2,
None,
'maximum-likelihood-estimation-mle'),
('A new Cost Function', 2, None, 'a-new-cost-function'),
("More basic Statistics and Bayes' theorem",
2,
None,
'more-basic-statistics-and-bayes-theorem'),
('Marginal Probability', 2, None, 'marginal-probability'),
('Conditional Probability', 2, None, 'conditional-probability'),
("Bayes' Theorem", 2, None, 'bayes-theorem'),
("Interpretations of Bayes' Theorem",
2,
None,
'interpretations-of-bayes-theorem'),
("Example of Usage of Bayes' theorem",
2,
None,
'example-of-usage-of-bayes-theorem'),
('Doing it correctly', 2, None, 'doing-it-correctly'),
("Bayes' Theorem and Ridge and Lasso Regression",
2,
None,
'bayes-theorem-and-ridge-and-lasso-regression'),
('Test Function for what happens with OLS, Ridge and Lasso',
2,
None,
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
('Exercise 1: mean values and variances in linear regression',
2,
None,
'exercise-1-mean-values-and-variances-in-linear-regression'),
('Exercise 2: Adding Ridge and Lasso Regression',
2,
None,
'exercise-2-adding-ridge-and-lasso-regression')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week36-bs.html">Week 36: Statistical interpretation of Linear Regression and Resampling techniques</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week36-bs001.html#plans-for-week-36" style="font-size: 80%;">Plans for week 36</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs002.html#summary-from-last-week-and-discussion-of-svd-ridge-and-lasso-regression-with-examples" style="font-size: 80%;">Summary from last Week and discussion of SVD, Ridge and Lasso regression with examples</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs003.html#linear-regression-and-the-svd" style="font-size: 80%;">Linear Regression and the SVD</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs004.html#what-does-it-mean" style="font-size: 80%;">What does it mean?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs005.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;">And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs006.html#code-for-svd-and-inversion-of-matrices" style="font-size: 80%;">Code for SVD and Inversion of Matrices</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs007.html#inverse-of-rectangular-matrix" style="font-size: 80%;">Inverse of Rectangular Matrix</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs008.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs009.html#from-ols-to-ridge-and-lasso" style="font-size: 80%;">From OLS to Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs010.html#deriving-the-ridge-regression-equations" style="font-size: 80%;">Deriving the Ridge Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs011.html#note-on-scikit-learn" style="font-size: 80%;">Note on Scikit-Learn</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs012.html#comparison-with-ols" style="font-size: 80%;">Comparison with OLS</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs013.html#svd-analysis" style="font-size: 80%;">SVD analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs014.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs015.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs016.html#deriving-the-lasso-regression-equations" style="font-size: 80%;">Deriving the Lasso Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs017.html#simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression" style="font-size: 80%;">Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs018.html#ridge-regression" style="font-size: 80%;">Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs019.html#lasso-regression" style="font-size: 80%;">Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs020.html#yet-another-example" style="font-size: 80%;">Yet another Example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs021.html#the-ols-case" style="font-size: 80%;">The OLS case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs022.html#the-ridge-case" style="font-size: 80%;">The Ridge case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs023.html#writing-the-cost-function" style="font-size: 80%;">Writing the Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs024.html#lasso-case" style="font-size: 80%;">Lasso case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs025.html#the-first-case" style="font-size: 80%;">The first Case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs026.html#simple-code-for-solving-the-above-problem" style="font-size: 80%;">Simple code for solving the above problem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs027.html#with-lasso-regression" style="font-size: 80%;">With Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs028.html#another-example-now-with-a-polynomial-fit" style="font-size: 80%;">Another Example, now with a polynomial fit</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs029.html#to-think-about-first-part" style="font-size: 80%;">To think about, first part</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs030.html#more-thinking" style="font-size: 80%;">More thinking</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs031.html#still-thinking" style="font-size: 80%;">Still thinking</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs032.html#what-does-centering-subtracting-the-mean-values-mean-mathematically" style="font-size: 80%;">What does centering (subtracting the mean values) mean mathematically?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs033.html#further-manipulations" style="font-size: 80%;">Further Manipulations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs034.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs035.html#linear-regression-code-intercept-handling-first" style="font-size: 80%;">Linear Regression code, Intercept handling first</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs036.html#code-examples" style="font-size: 80%;">Code Examples</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs037.html#taking-out-the-mean" style="font-size: 80%;">Taking out the mean</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs038.html#friday-september-9" style="font-size: 80%;">Friday September 9</a></li>
<!-- navigation toc: --> <li><a href="#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs040.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs041.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs042.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs043.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs044.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs045.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs046.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs047.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs048.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs049.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs050.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs051.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs052.html#example-of-usage-of-bayes-theorem" style="font-size: 80%;">Example of Usage of Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs053.html#doing-it-correctly" style="font-size: 80%;">Doing it correctly</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs054.html#bayes-theorem-and-ridge-and-lasso-regression" style="font-size: 80%;">Bayes' Theorem and Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs055.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs056.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs057.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#exercise-1-mean-values-and-variances-in-linear-regression" style="font-size: 80%;">Exercise 1: mean values and variances in linear regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#exercise-2-adding-ridge-and-lasso-regression" style="font-size: 80%;">Exercise 2: Adding Ridge and Lasso Regression</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0039"></a>
<!-- !split -->
<h2 id="linking-the-regression-analysis-with-a-statistical-interpretation" class="anchor">Linking the regression analysis with a statistical interpretation </h2>
<p>We will now couple the discussions of ordinary least squares, Ridge
and Lasso regression with a statistical interpretation, that is we
move from a linear algebra analysis to a statistical analysis. In
particular, we will focus on what the regularization terms can result
in. We will amongst other things show that the regularization
parameter can reduce considerably the variance of the parameters
\( \beta \).
</p>
<p>The
advantage of doing linear regression is that we actually end up with
analytical expressions for several statistical quantities.
Standard least squares and Ridge regression allow us to
derive quantities like the variance and other expectation values in a
rather straightforward way.
</p>
<p>It is assumed that \( \varepsilon_i
\sim \mathcal{N}(0, \sigma^2) \) and the \( \varepsilon_{i} \) are
independent, i.e.:
</p>
$$
\begin{align*}
\mbox{Cov}(\varepsilon_{i_1},
\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if}
& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right.
\end{align*}
$$
<p>The randomness of \( \varepsilon_i \) implies that
\( \mathbf{y}_i \) is also a random variable. In particular,
\( \mathbf{y}_i \) is normally distributed, because \( \varepsilon_i \sim
\mathcal{N}(0, \sigma^2) \) and \( \mathbf{X}_{i,\ast} \, \boldsymbol{\beta} \) is a
non-random scalar. To specify the parameters of the distribution of
\( \mathbf{y}_i \) we need to calculate its first two moments.
</p>
<p>Recall that \( \boldsymbol{X} \) is a matrix of dimensionality \( n\times p \). The
notation above \( \mathbf{X}_{i,\ast} \) means that we are looking at the
row number \( i \) and perform a sum over all values \( p \).
</p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week36-bs038.html">&laquo;</a></li>
<li><a href="._week36-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week36-bs031.html">32</a></li>
<li><a href="._week36-bs032.html">33</a></li>
<li><a href="._week36-bs033.html">34</a></li>
<li><a href="._week36-bs034.html">35</a></li>
<li><a href="._week36-bs035.html">36</a></li>
<li><a href="._week36-bs036.html">37</a></li>
<li><a href="._week36-bs037.html">38</a></li>
<li><a href="._week36-bs038.html">39</a></li>
<li class="active"><a href="._week36-bs039.html">40</a></li>
<li><a href="._week36-bs040.html">41</a></li>
<li><a href="._week36-bs041.html">42</a></li>
<li><a href="._week36-bs042.html">43</a></li>
<li><a href="._week36-bs043.html">44</a></li>
<li><a href="._week36-bs044.html">45</a></li>
<li><a href="._week36-bs045.html">46</a></li>
<li><a href="._week36-bs046.html">47</a></li>
<li><a href="._week36-bs047.html">48</a></li>
<li><a href="._week36-bs048.html">49</a></li>
<li><a href="">...</a></li>
<li><a href="._week36-bs058.html">59</a></li>
<li><a href="._week36-bs040.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>