Files
FYS-STK4155/doc/pub/week37/html/._week37-bs010.html
T
Morten Hjorth-Jensen c253fedc65 cleaning up week 37
2021-09-28 10:15:00 +02:00

399 lines
24 KiB
HTML

<!--
Automatically generated HTML file from DocOnce source
(https://github.com/doconce/doconce/)
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
<!-- Bootstrap style: bootstrap -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
('Thursday September 16, Summary of Ridge and Lasso Regression '
'and start Resampling methods',
2,
None,
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
('Deriving OLS from a probability distribution',
2,
None,
'deriving-ols-from-a-probability-distribution'),
('Independent and Identically Distrubuted (iid)',
2,
None,
'independent-and-identically-distrubuted-iid'),
('Maximum Likelihood Estimation (MLE)',
2,
None,
'maximum-likelihood-estimation-mle'),
('A new Cost Function', 2, None, 'a-new-cost-function'),
("Bayes' Theorem", 2, None, 'bayes-theorem'),
("Interpretations of Bayes' Theorem",
2,
None,
'interpretations-of-bayes-theorem'),
('Test Function for what happens with OLS, Ridge and Lasso',
2,
None,
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
('Why resampling methods', 2, None, 'why-resampling-methods'),
('Resampling methods', 2, None, 'resampling-methods'),
('Resampling approaches can be computationally expensive',
2,
None,
'resampling-approaches-can-be-computationally-expensive'),
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
('Statistical analysis', 2, None, 'statistical-analysis'),
('Resampling methods', 2, None, 'resampling-methods'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
'resampling-methods-jackknife-and-bootstrap'),
('Resampling methods: Jackknife',
2,
None,
'resampling-methods-jackknife'),
('Jackknife code example', 2, None, 'jackknife-code-example'),
('Resampling methods: Bootstrap',
2,
None,
'resampling-methods-bootstrap'),
('The Central Limit Theorem',
2,
None,
'the-central-limit-theorem'),
('Finding the Limit', 2, None, 'finding-the-limit'),
('Rewriting the $\\delta$-function',
2,
None,
'rewriting-the-delta-function'),
('Identifying Terms', 2, None, 'identifying-terms'),
('Wrapping it up', 2, None, 'wrapping-it-up'),
('Confidence Intervals', 2, None, 'confidence-intervals'),
('Standard Approach based on the Normal Distribution',
2,
None,
'standard-approach-based-on-the-normal-distribution'),
('Resampling methods: Bootstrap background',
2,
None,
'resampling-methods-bootstrap-background'),
('Resampling methods: More Bootstrap background',
2,
None,
'resampling-methods-more-bootstrap-background'),
('Resampling methods: Bootstrap approach',
2,
None,
'resampling-methods-bootstrap-approach'),
('Resampling methods: Bootstrap steps',
2,
None,
'resampling-methods-bootstrap-steps'),
('Code example for the Bootstrap method',
2,
None,
'code-example-for-the-bootstrap-method'),
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
('The bias-variance tradeoff',
2,
None,
'the-bias-variance-tradeoff'),
('A way to Read the Bias-Variance Tradeoff',
2,
None,
'a-way-to-read-the-bias-variance-tradeoff'),
('Example code for Bias-Variance tradeoff',
2,
None,
'example-code-for-bias-variance-tradeoff'),
('Understanding what happens',
2,
None,
'understanding-what-happens'),
('Summing up', 2, None, 'summing-up'),
("Another Example from Scikit-Learn's Repository",
2,
None,
'another-example-from-scikit-learn-s-repository'),
('Various steps in cross-validation',
2,
None,
'various-steps-in-cross-validation'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
('Cross-validation in brief',
2,
None,
'cross-validation-in-brief'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
'code-example-for-cross-validation-and-k-fold-cross-validation'),
('More examples on bootstrap and cross-validation and errors',
2,
None,
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
('The same example but now with cross-validation',
2,
None,
'the-same-example-but-now-with-cross-validation')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0010"></a>
<!-- !split -->
<h2 id="rerunning-the-above-code" class="anchor">Rerunning the above code </h2>
<p>
Let us write out the values of the coefficients \( \beta_i \) as functions
of the polynomial degree and noise. We will focus only on the Ridge
results and some few selected values of the hyperparameter \( \lambda \).
<p>
If we don't include any noise and run this code for different values
of the polynomial degree, we notice that the results for \( \beta_i \) do
not show great changes from one order to the next. This is an
indication that for higher polynomial orders, our parameters become
less important.
<p>
If we however add noise, what happens is that the polynomial fit is
trying to adjust the fit to traverse in the best possible way all data
points. This can lead to large fluctuations in the parameters
\( \beta_i \) as functions of polynomial order. It will also be reflected
in a larger value of the variance of each parameter \( \beta_i \). What
Ridge regression (and Lasso as well) are doing then is to try to
quench the fluctuations in the parameters of \( \beta_i \) which have a
large variance (normally for higher orders in the polynomial).
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> linear_model
<span style="color: #408080; font-style: italic"># Make data set.</span>
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n)
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n)
Maxpolydegree <span style="color: #666666">=</span> <span style="color: #666666">5</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(x),Maxpolydegree))
X[:,<span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
<span style="color: #008000; font-weight: bold">for</span> polydegree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>, Maxpolydegree):
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(polydegree):
X[:,degree] <span style="color: #666666">=</span> x<span style="color: #666666">**</span>(degree)
<span style="color: #408080; font-style: italic"># We split the data in test and training data</span>
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, y, test_size<span style="color: #666666">=0.2</span>)
<span style="color: #408080; font-style: italic"># Decide which values of lambda to use</span>
nlambdas <span style="color: #666666">=</span> <span style="color: #666666">5</span>
lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-3</span>, <span style="color: #666666">2</span>, nlambdas)
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(nlambdas):
lmb <span style="color: #666666">=</span> lambdas[i]
<span style="color: #408080; font-style: italic"># Make the fit using Ridge only</span>
RegRidge <span style="color: #666666">=</span> linear_model<span style="color: #666666">.</span>Ridge(lmb,fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
RegRidge<span style="color: #666666">.</span>fit(X_train,y_train)
<span style="color: #408080; font-style: italic"># and then make the prediction</span>
ypredictRidge <span style="color: #666666">=</span> RegRidge<span style="color: #666666">.</span>predict(X_test)
Coeffs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(RegRidge<span style="color: #666666">.</span>coef_)
BetaValues <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(Coeffs)
BetaValues<span style="color: #666666">.</span>columns <span style="color: #666666">=</span> [<span style="color: #BA2121">&#39;beta&#39;</span>]
display(BetaValues)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week37-bs009.html">&laquo;</a></li>
<li><a href="._week37-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week37-bs002.html">3</a></li>
<li><a href="._week37-bs003.html">4</a></li>
<li><a href="._week37-bs004.html">5</a></li>
<li><a href="._week37-bs005.html">6</a></li>
<li><a href="._week37-bs006.html">7</a></li>
<li><a href="._week37-bs007.html">8</a></li>
<li><a href="._week37-bs008.html">9</a></li>
<li><a href="._week37-bs009.html">10</a></li>
<li class="active"><a href="._week37-bs010.html">11</a></li>
<li><a href="._week37-bs011.html">12</a></li>
<li><a href="._week37-bs012.html">13</a></li>
<li><a href="._week37-bs013.html">14</a></li>
<li><a href="._week37-bs014.html">15</a></li>
<li><a href="._week37-bs015.html">16</a></li>
<li><a href="._week37-bs016.html">17</a></li>
<li><a href="._week37-bs017.html">18</a></li>
<li><a href="._week37-bs018.html">19</a></li>
<li><a href="._week37-bs019.html">20</a></li>
<li><a href="">...</a></li>
<li><a href="._week37-bs048.html">49</a></li>
<li><a href="._week37-bs011.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>