added dot files
This commit is contained in:
Morten Hjorth-Jensen
@@ -0,0 +1,386 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0040"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="understanding-what-happens" class="anchor">Understanding what happens </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> make_pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.utils</span> <span style="color: #008000; font-weight: bold">import</span> resample
|
||||
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2018</span>)
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">40</span>
|
||||
n_boostraps <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">14</span>
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(maxdegree):
|
||||
model <span style="color: #666666">=</span> make_pipeline(PolynomialFeatures(degree<span style="color: #666666">=</span>degree), LinearRegression(fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
y_pred <span style="color: #666666">=</span> np<span style="color: #666666">.</span>empty((y_test<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], n_boostraps))
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_boostraps):
|
||||
x_, y_ <span style="color: #666666">=</span> resample(x_train, y_train)
|
||||
y_pred[:, i] <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(x_, y_)<span style="color: #666666">.</span>predict(x_test)<span style="color: #666666">.</span>ravel()
|
||||
|
||||
polydegree[degree] <span style="color: #666666">=</span> degree
|
||||
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>) )
|
||||
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>))<span style="color: #666666">**2</span> )
|
||||
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>) )
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Polynomial degree:'</span>, degree)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs039.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs032.html">33</a></li>
|
||||
<li><a href="._week37-bs033.html">34</a></li>
|
||||
<li><a href="._week37-bs034.html">35</a></li>
|
||||
<li><a href="._week37-bs035.html">36</a></li>
|
||||
<li><a href="._week37-bs036.html">37</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li class="active"><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs041.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0041"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="summing-up" class="anchor">Summing up </h2>
|
||||
|
||||
<p>
|
||||
The bias-variance tradeoff summarizes the fundamental tension in
|
||||
machine learning, particularly supervised learning, between the
|
||||
complexity of a model and the amount of training data needed to train
|
||||
it. Since data is often limited, in practice it is often useful to
|
||||
use a less-complex model with higher bias, that is a model whose asymptotic
|
||||
performance is worse than another model because it is easier to
|
||||
train and less sensitive to sampling noise arising from having a
|
||||
finite-sized training dataset (smaller variance).
|
||||
|
||||
<p>
|
||||
The above equations tell us that in
|
||||
order to minimize the expected test error, we need to select a
|
||||
statistical learning method that simultaneously achieves low variance
|
||||
and low bias. Note that variance is inherently a nonnegative quantity,
|
||||
and squared bias is also nonnegative. Hence, we see that the expected
|
||||
test MSE can never lie below \( Var(\epsilon) \), the irreducible error.
|
||||
|
||||
<p>
|
||||
What do we mean by the variance and bias of a statistical learning
|
||||
method? The variance refers to the amount by which our model would change if we
|
||||
estimated it using a different training data set. Since the training
|
||||
data are used to fit the statistical learning method, different
|
||||
training data sets will result in a different estimate. But ideally the
|
||||
estimate for our model should not vary too much between training
|
||||
sets. However, if a method has high variance then small changes in
|
||||
the training data can result in large changes in the model. In general, more
|
||||
flexible statistical methods have higher variance.
|
||||
|
||||
<p>
|
||||
You may also find this recent <a href="https://www.pnas.org/content/116/32/15849" target="_self">article</a> of interest.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs040.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs033.html">34</a></li>
|
||||
<li><a href="._week37-bs034.html">35</a></li>
|
||||
<li><a href="._week37-bs035.html">36</a></li>
|
||||
<li><a href="._week37-bs036.html">37</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li class="active"><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs042.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0042"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="another-example-from-scikit-learn-s-repository" class="anchor">Another Example from Scikit-Learn's Repository </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic">============================</span>
|
||||
<span style="color: #BA2121; font-style: italic">Underfitting vs. Overfitting</span>
|
||||
<span style="color: #BA2121; font-style: italic">============================</span>
|
||||
|
||||
<span style="color: #BA2121; font-style: italic">This example demonstrates the problems of underfitting and overfitting and</span>
|
||||
<span style="color: #BA2121; font-style: italic">how we can use linear regression with polynomial features to approximate</span>
|
||||
<span style="color: #BA2121; font-style: italic">nonlinear functions. The plot shows the function that we want to approximate,</span>
|
||||
<span style="color: #BA2121; font-style: italic">which is a part of the cosine function. In addition, the samples from the</span>
|
||||
<span style="color: #BA2121; font-style: italic">real function and the approximations of different models are displayed. The</span>
|
||||
<span style="color: #BA2121; font-style: italic">models have polynomial features of different degrees. We can see that a</span>
|
||||
<span style="color: #BA2121; font-style: italic">linear function (polynomial with degree 1) is not sufficient to fit the</span>
|
||||
<span style="color: #BA2121; font-style: italic">training samples. This is called **underfitting**. A polynomial of degree 4</span>
|
||||
<span style="color: #BA2121; font-style: italic">approximates the true function almost perfectly. However, for higher degrees</span>
|
||||
<span style="color: #BA2121; font-style: italic">the model will **overfit** the training data, i.e. it learns the noise of the</span>
|
||||
<span style="color: #BA2121; font-style: italic">training data.</span>
|
||||
<span style="color: #BA2121; font-style: italic">We evaluate quantitatively **overfitting** / **underfitting** by using</span>
|
||||
<span style="color: #BA2121; font-style: italic">cross-validation. We calculate the mean squared error (MSE) on the validation</span>
|
||||
<span style="color: #BA2121; font-style: italic">set, the higher, the less likely the model generalizes correctly from the</span>
|
||||
<span style="color: #BA2121; font-style: italic">training data.</span>
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #19177C">__doc__</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_val_score
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">true_fun</span>(X):
|
||||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>cos(<span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">*</span> X)
|
||||
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
|
||||
n_samples <span style="color: #666666">=</span> <span style="color: #666666">30</span>
|
||||
degrees <span style="color: #666666">=</span> [<span style="color: #666666">1</span>, <span style="color: #666666">4</span>, <span style="color: #666666">15</span>]
|
||||
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sort(np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n_samples))
|
||||
y <span style="color: #666666">=</span> true_fun(X) <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_samples) <span style="color: #666666">*</span> <span style="color: #666666">0.1</span>
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">14</span>, <span style="color: #666666">5</span>))
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(degrees)):
|
||||
ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #008000">len</span>(degrees), i <span style="color: #666666">+</span> <span style="color: #666666">1</span>)
|
||||
plt<span style="color: #666666">.</span>setp(ax, xticks<span style="color: #666666">=</span>(), yticks<span style="color: #666666">=</span>())
|
||||
|
||||
polynomial_features <span style="color: #666666">=</span> PolynomialFeatures(degree<span style="color: #666666">=</span>degrees[i],
|
||||
include_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
|
||||
linear_regression <span style="color: #666666">=</span> LinearRegression()
|
||||
pipeline <span style="color: #666666">=</span> Pipeline([(<span style="color: #BA2121">"polynomial_features"</span>, polynomial_features),
|
||||
(<span style="color: #BA2121">"linear_regression"</span>, linear_regression)])
|
||||
pipeline<span style="color: #666666">.</span>fit(X[:, np<span style="color: #666666">.</span>newaxis], y)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Evaluate the models using crossvalidation</span>
|
||||
scores <span style="color: #666666">=</span> cross_val_score(pipeline, X[:, np<span style="color: #666666">.</span>newaxis], y,
|
||||
scoring<span style="color: #666666">=</span><span style="color: #BA2121">"neg_mean_squared_error"</span>, cv<span style="color: #666666">=10</span>)
|
||||
|
||||
X_test <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">100</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_test, pipeline<span style="color: #666666">.</span>predict(X_test[:, np<span style="color: #666666">.</span>newaxis]), label<span style="color: #666666">=</span><span style="color: #BA2121">"Model"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_test, true_fun(X_test), label<span style="color: #666666">=</span><span style="color: #BA2121">"True function"</span>)
|
||||
plt<span style="color: #666666">.</span>scatter(X, y, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'b'</span>, s<span style="color: #666666">=20</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Samples"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"x"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"y"</span>)
|
||||
plt<span style="color: #666666">.</span>xlim((<span style="color: #666666">0</span>, <span style="color: #666666">1</span>))
|
||||
plt<span style="color: #666666">.</span>ylim((<span style="color: #666666">-2</span>, <span style="color: #666666">2</span>))
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">"best"</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Degree </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">MSE = </span><span style="color: #BB6688; font-weight: bold">{:.2e}</span><span style="color: #BA2121">(+/- </span><span style="color: #BB6688; font-weight: bold">{:.2e}</span><span style="color: #BA2121">)"</span><span style="color: #666666">.</span>format(
|
||||
degrees[i], <span style="color: #666666">-</span>scores<span style="color: #666666">.</span>mean(), scores<span style="color: #666666">.</span>std()))
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs041.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs034.html">35</a></li>
|
||||
<li><a href="._week37-bs035.html">36</a></li>
|
||||
<li><a href="._week37-bs036.html">37</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li class="active"><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs043.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0043"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="various-steps-in-cross-validation" class="anchor">Various steps in cross-validation </h2>
|
||||
|
||||
<p>
|
||||
When the repetitive splitting of the data set is done randomly,
|
||||
samples may accidently end up in a fast majority of the splits in
|
||||
either training or test set. Such samples may have an unbalanced
|
||||
influence on either model building or prediction evaluation. To avoid
|
||||
this \( k \)-fold cross-validation structures the data splitting. The
|
||||
samples are divided into \( k \) more or less equally sized exhaustive and
|
||||
mutually exclusive subsets. In turn (at each split) one of these
|
||||
subsets plays the role of the test set while the union of the
|
||||
remaining subsets constitutes the training set. Such a splitting
|
||||
warrants a balanced representation of each sample in both training and
|
||||
test set over the splits. Still the division into the \( k \) subsets
|
||||
involves a degree of randomness. This may be fully excluded when
|
||||
choosing \( k=n \). This particular case is referred to as leave-one-out
|
||||
cross-validation (LOOCV).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs042.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs035.html">36</a></li>
|
||||
<li><a href="._week37-bs036.html">37</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li class="active"><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs044.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0044"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" class="anchor">How to set up the cross-validation for Ridge and/or Lasso </h2>
|
||||
|
||||
<ul>
|
||||
<li> Define a range of interest for the penalty parameter.</li>
|
||||
<li> Divide the data set into training and test set comprising samples \( \{1, \ldots, n\} \setminus i \) and \( \{ i \} \), respectively.</li>
|
||||
<li> Fit the linear regression model by means of ridge estimation for each \( \lambda \) in the grid using the training set, and the corresponding estimate of the error variance \( \boldsymbol{\sigma}_{-i}^2(\lambda) \), as</li>
|
||||
</ul>
|
||||
|
||||
$$
|
||||
\begin{align*}
|
||||
\boldsymbol{\beta}_{-i}(\lambda) & = ( \boldsymbol{X}_{-i, \ast}^{T}
|
||||
\boldsymbol{X}_{-i, \ast} + \lambda \boldsymbol{I}_{pp})^{-1}
|
||||
\boldsymbol{X}_{-i, \ast}^{T} \boldsymbol{y}_{-i}
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
|
||||
<ul>
|
||||
<li> Evaluate the prediction performance of these models on the test set by \( \log\{L[y_i, \boldsymbol{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\} \). Or, by the prediction error \( |y_i - \boldsymbol{X}_{i, \ast} \boldsymbol{\beta}_{-i}(\lambda)| \), the relative error, the error squared or the R2 score function.</li>
|
||||
<li> Repeat the first three steps such that each sample plays the role of the test set once.</li>
|
||||
<li> Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as</li>
|
||||
</ul>
|
||||
|
||||
$$
|
||||
\begin{align*}
|
||||
\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\}.
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs043.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs036.html">37</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li class="active"><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs045.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0045"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="cross-validation-in-brief" class="anchor">Cross-validation in brief </h2>
|
||||
|
||||
<p>
|
||||
For the various values of \( k \)
|
||||
|
||||
<ol>
|
||||
<li> shuffle the dataset randomly.</li>
|
||||
<li> Split the dataset into \( k \) groups.</li>
|
||||
<li> For each unique group:
|
||||
|
||||
<ol type="a"></li>
|
||||
<li> Decide which group to use as set for test data</li>
|
||||
<li> Take the remaining groups as a training data set</li>
|
||||
<li> Fit a model on the training set and evaluate it on the test set</li>
|
||||
<li> Retain the evaluation score and discard the model</li>
|
||||
</ol>
|
||||
|
||||
<li> Summarize the model using the sample of model evaluation scores</li>
|
||||
</ol>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs044.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs037.html">38</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li class="active"><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs046.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0046"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="code-example-for-cross-validation-and-k-fold-cross-validation" class="anchor">Code Example for Cross-validation and \( k \)-fold Cross-validation </h2>
|
||||
|
||||
<p>
|
||||
The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> KFold
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> Ridge
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_val_score
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
|
||||
<span style="color: #408080; font-style: italic"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
<span style="color: #408080; font-style: italic"># Useful for eventual debugging.</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">3155</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Generate the data.</span>
|
||||
nsamples <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(nsamples)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">3*</span>x<span style="color: #666666">**2</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(nsamples)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Cross-validation on Ridge regression using KFold only</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Decide degree on polynomial to fit</span>
|
||||
poly <span style="color: #666666">=</span> PolynomialFeatures(degree <span style="color: #666666">=</span> <span style="color: #666666">6</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Decide which values of lambda to use</span>
|
||||
nlambdas <span style="color: #666666">=</span> <span style="color: #666666">500</span>
|
||||
lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-3</span>, <span style="color: #666666">5</span>, nlambdas)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Initialize a KFold instance</span>
|
||||
k <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
kfold <span style="color: #666666">=</span> KFold(n_splits <span style="color: #666666">=</span> k)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Perform the cross-validation to estimate MSE</span>
|
||||
scores_KFold <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((nlambdas, k))
|
||||
|
||||
i <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> lmb <span style="color: #AA22FF; font-weight: bold">in</span> lambdas:
|
||||
ridge <span style="color: #666666">=</span> Ridge(alpha <span style="color: #666666">=</span> lmb)
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> train_inds, test_inds <span style="color: #AA22FF; font-weight: bold">in</span> kfold<span style="color: #666666">.</span>split(x):
|
||||
xtrain <span style="color: #666666">=</span> x[train_inds]
|
||||
ytrain <span style="color: #666666">=</span> y[train_inds]
|
||||
|
||||
xtest <span style="color: #666666">=</span> x[test_inds]
|
||||
ytest <span style="color: #666666">=</span> y[test_inds]
|
||||
|
||||
Xtrain <span style="color: #666666">=</span> poly<span style="color: #666666">.</span>fit_transform(xtrain[:, np<span style="color: #666666">.</span>newaxis])
|
||||
ridge<span style="color: #666666">.</span>fit(Xtrain, ytrain[:, np<span style="color: #666666">.</span>newaxis])
|
||||
|
||||
Xtest <span style="color: #666666">=</span> poly<span style="color: #666666">.</span>fit_transform(xtest[:, np<span style="color: #666666">.</span>newaxis])
|
||||
ypred <span style="color: #666666">=</span> ridge<span style="color: #666666">.</span>predict(Xtest)
|
||||
|
||||
scores_KFold[i,j] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum((ypred <span style="color: #666666">-</span> ytest[:, np<span style="color: #666666">.</span>newaxis])<span style="color: #666666">**2</span>)<span style="color: #666666">/</span>np<span style="color: #666666">.</span>size(ypred)
|
||||
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
i <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
|
||||
estimated_mse_KFold <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(scores_KFold, axis <span style="color: #666666">=</span> <span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Cross-validation using cross_val_score from sklearn along with KFold</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># kfold is an instance initialized above as:</span>
|
||||
<span style="color: #408080; font-style: italic"># kfold = KFold(n_splits = k)</span>
|
||||
|
||||
estimated_mse_sklearn <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(nlambdas)
|
||||
i <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> lmb <span style="color: #AA22FF; font-weight: bold">in</span> lambdas:
|
||||
ridge <span style="color: #666666">=</span> Ridge(alpha <span style="color: #666666">=</span> lmb)
|
||||
|
||||
X <span style="color: #666666">=</span> poly<span style="color: #666666">.</span>fit_transform(x[:, np<span style="color: #666666">.</span>newaxis])
|
||||
estimated_mse_folds <span style="color: #666666">=</span> cross_val_score(ridge, X, y[:, np<span style="color: #666666">.</span>newaxis], scoring<span style="color: #666666">=</span><span style="color: #BA2121">'neg_mean_squared_error'</span>, cv<span style="color: #666666">=</span>kfold)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># cross_val_score return an array containing the estimated negative mse for every fold.</span>
|
||||
<span style="color: #408080; font-style: italic"># we have to the the mean of every array in order to get an estimate of the mse of the model</span>
|
||||
estimated_mse_sklearn[i] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(<span style="color: #666666">-</span>estimated_mse_folds)
|
||||
|
||||
i <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Plot and compare the slightly different ways to perform cross-validation</span>
|
||||
|
||||
plt<span style="color: #666666">.</span>figure()
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(np<span style="color: #666666">.</span>log10(lambdas), estimated_mse_sklearn, label <span style="color: #666666">=</span> <span style="color: #BA2121">'cross_val_score'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(np<span style="color: #666666">.</span>log10(lambdas), estimated_mse_KFold, <span style="color: #BA2121">'r--'</span>, label <span style="color: #666666">=</span> <span style="color: #BA2121">'KFold'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'log10(lambda)'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'mse'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs045.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs038.html">39</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li class="active"><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs047.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0047"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="more-examples-on-bootstrap-and-cross-validation-and-errors" class="anchor">More examples on bootstrap and cross-validation and errors </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.utils</span> <span style="color: #008000; font-weight: bold">import</span> resample
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, <span style="color: #008000">format</span><span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
infile <span style="color: #666666">=</span> <span style="color: #008000">open</span>(data_path(<span style="color: #BA2121">"EoS.csv"</span>),<span style="color: #BA2121">'r'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read the EoS data as csv file and organize the data into two arrays with density and energies</span>
|
||||
EoS <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(infile, names<span style="color: #666666">=</span>(<span style="color: #BA2121">'Density'</span>, <span style="color: #BA2121">'Energy'</span>))
|
||||
EoS[<span style="color: #BA2121">'Energy'</span>] <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>to_numeric(EoS[<span style="color: #BA2121">'Energy'</span>], errors<span style="color: #666666">=</span><span style="color: #BA2121">'coerce'</span>)
|
||||
EoS <span style="color: #666666">=</span> EoS<span style="color: #666666">.</span>dropna()
|
||||
Energies <span style="color: #666666">=</span> EoS[<span style="color: #BA2121">'Energy'</span>]
|
||||
Density <span style="color: #666666">=</span> EoS[<span style="color: #BA2121">'Density'</span>]
|
||||
<span style="color: #408080; font-style: italic"># The design matrix now as function of various polytrops</span>
|
||||
|
||||
Maxpolydegree <span style="color: #666666">=</span> <span style="color: #666666">30</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(Density),Maxpolydegree))
|
||||
X[:,<span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
testerror <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(Maxpolydegree)
|
||||
trainingerror <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(Maxpolydegree)
|
||||
polynomial <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(Maxpolydegree)
|
||||
|
||||
trials <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> polydegree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>, Maxpolydegree):
|
||||
polynomial[polydegree] <span style="color: #666666">=</span> polydegree
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(polydegree):
|
||||
X[:,degree] <span style="color: #666666">=</span> Density<span style="color: #666666">**</span>(degree<span style="color: #666666">/3.0</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># loop over trials in order to estimate the expectation value of the MSE</span>
|
||||
testerror[polydegree] <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
trainingerror[polydegree] <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> samples <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(trials):
|
||||
x_train, x_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(X, Energies, test_size<span style="color: #666666">=0.2</span>)
|
||||
model <span style="color: #666666">=</span> LinearRegression(fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)<span style="color: #666666">.</span>fit(x_train, y_train)
|
||||
ypred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(x_train)
|
||||
ytilde <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(x_test)
|
||||
testerror[polydegree] <span style="color: #666666">+=</span> mean_squared_error(y_test, ytilde)
|
||||
trainingerror[polydegree] <span style="color: #666666">+=</span> mean_squared_error(y_train, ypred)
|
||||
|
||||
testerror[polydegree] <span style="color: #666666">/=</span> trials
|
||||
trainingerror[polydegree] <span style="color: #666666">/=</span> trials
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Degree of polynomial: </span><span style="color: #BB6688; font-weight: bold">%3d</span><span style="color: #BA2121">"</span><span style="color: #666666">%</span> polynomial[polydegree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Mean squared error on training data: </span><span style="color: #BB6688; font-weight: bold">%.8f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> trainingerror[polydegree])
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Mean squared error on test data: </span><span style="color: #BB6688; font-weight: bold">%.8f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> testerror[polydegree])
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(polynomial, np<span style="color: #666666">.</span>log10(trainingerror), label<span style="color: #666666">=</span><span style="color: #BA2121">'Training Error'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(polynomial, np<span style="color: #666666">.</span>log10(testerror), label<span style="color: #666666">=</span><span style="color: #BA2121">'Test Error'</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'Polynomial degree'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'log10[MSE]'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs046.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs039.html">40</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li class="active"><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs048.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs049.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0048"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="the-same-example-but-now-with-cross-validation" class="anchor">The same example but now with cross-validation </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># Common imports</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LinearRegression, Ridge, Lasso
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> KFold
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_val_score
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
||||
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
||||
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
||||
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
||||
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
||||
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
||||
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
||||
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, <span style="color: #008000">format</span><span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
||||
|
||||
infile <span style="color: #666666">=</span> <span style="color: #008000">open</span>(data_path(<span style="color: #BA2121">"EoS.csv"</span>),<span style="color: #BA2121">'r'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Read the EoS data as csv file and organize the data into two arrays with density and energies</span>
|
||||
EoS <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>read_csv(infile, names<span style="color: #666666">=</span>(<span style="color: #BA2121">'Density'</span>, <span style="color: #BA2121">'Energy'</span>))
|
||||
EoS[<span style="color: #BA2121">'Energy'</span>] <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>to_numeric(EoS[<span style="color: #BA2121">'Energy'</span>], errors<span style="color: #666666">=</span><span style="color: #BA2121">'coerce'</span>)
|
||||
EoS <span style="color: #666666">=</span> EoS<span style="color: #666666">.</span>dropna()
|
||||
Energies <span style="color: #666666">=</span> EoS[<span style="color: #BA2121">'Energy'</span>]
|
||||
Density <span style="color: #666666">=</span> EoS[<span style="color: #BA2121">'Density'</span>]
|
||||
<span style="color: #408080; font-style: italic"># The design matrix now as function of various polytrops</span>
|
||||
|
||||
Maxpolydegree <span style="color: #666666">=</span> <span style="color: #666666">30</span>
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(Density),Maxpolydegree))
|
||||
X[:,<span style="color: #666666">0</span>] <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
estimated_mse_sklearn <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(Maxpolydegree)
|
||||
polynomial <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(Maxpolydegree)
|
||||
k <span style="color: #666666">=5</span>
|
||||
kfold <span style="color: #666666">=</span> KFold(n_splits <span style="color: #666666">=</span> k)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> polydegree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>, Maxpolydegree):
|
||||
polynomial[polydegree] <span style="color: #666666">=</span> polydegree
|
||||
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(polydegree):
|
||||
X[:,degree] <span style="color: #666666">=</span> Density<span style="color: #666666">**</span>(degree<span style="color: #666666">/3.0</span>)
|
||||
OLS <span style="color: #666666">=</span> LinearRegression()
|
||||
<span style="color: #408080; font-style: italic"># loop over trials in order to estimate the expectation value of the MSE</span>
|
||||
estimated_mse_folds <span style="color: #666666">=</span> cross_val_score(OLS, X, Energies, scoring<span style="color: #666666">=</span><span style="color: #BA2121">'neg_mean_squared_error'</span>, cv<span style="color: #666666">=</span>kfold)
|
||||
<span style="color: #408080; font-style: italic">#[:, np.newaxis]</span>
|
||||
estimated_mse_sklearn[polydegree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(<span style="color: #666666">-</span>estimated_mse_folds)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(polynomial, np<span style="color: #666666">.</span>log10(estimated_mse_sklearn), label<span style="color: #666666">=</span><span style="color: #BA2121">'Test Error'</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'Polynomial degree'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'log10[MSE]'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs047.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs040.html">41</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li class="active"><a href="._week37-bs048.html">49</a></li>
|
||||
<li><a href="._week37-bs049.html">50</a></li>
|
||||
<li><a href="._week37-bs049.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 37: Summary of Ridge and Lasso Regression and Resampling Methods">
|
||||
|
||||
<title>Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 37', 2, None, 'plans-for-week-37'),
|
||||
('Thursday September 16, Summary of Ridge and Lasso Regression '
|
||||
'and start Resampling methods',
|
||||
2,
|
||||
None,
|
||||
'thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods'),
|
||||
('Deriving OLS from a probability distribution',
|
||||
2,
|
||||
None,
|
||||
'deriving-ols-from-a-probability-distribution'),
|
||||
('Independent and Identically Distrubuted (iid)',
|
||||
2,
|
||||
None,
|
||||
'independent-and-identically-distrubuted-iid'),
|
||||
('Maximum Likelihood Estimation (MLE)',
|
||||
2,
|
||||
None,
|
||||
'maximum-likelihood-estimation-mle'),
|
||||
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
||||
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
||||
("Interpretations of Bayes' Theorem",
|
||||
2,
|
||||
None,
|
||||
'interpretations-of-bayes-theorem'),
|
||||
('Test Function for what happens with OLS, Ridge and Lasso',
|
||||
2,
|
||||
None,
|
||||
'test-function-for-what-happens-with-ols-ridge-and-lasso'),
|
||||
('Rerunning the above code', 2, None, 'rerunning-the-above-code'),
|
||||
("Invoking Bayes' theorem", 2, None, 'invoking-bayes-theorem'),
|
||||
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
||||
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
||||
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling approaches can be computationally expensive',
|
||||
2,
|
||||
None,
|
||||
'resampling-approaches-can-be-computationally-expensive'),
|
||||
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
||||
('Statistical analysis', 2, None, 'statistical-analysis'),
|
||||
('Resampling methods', 2, None, 'resampling-methods'),
|
||||
('Resampling methods: Jackknife and Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife-and-bootstrap'),
|
||||
('Resampling methods: Jackknife',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-jackknife'),
|
||||
('Jackknife code example', 2, None, 'jackknife-code-example'),
|
||||
('Resampling methods: Bootstrap',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap'),
|
||||
('The Central Limit Theorem',
|
||||
2,
|
||||
None,
|
||||
'the-central-limit-theorem'),
|
||||
('Finding the Limit', 2, None, 'finding-the-limit'),
|
||||
('Rewriting the $\\delta$-function',
|
||||
2,
|
||||
None,
|
||||
'rewriting-the-delta-function'),
|
||||
('Identifying Terms', 2, None, 'identifying-terms'),
|
||||
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
||||
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
||||
('Standard Approach based on the Normal Distribution',
|
||||
2,
|
||||
None,
|
||||
'standard-approach-based-on-the-normal-distribution'),
|
||||
('Resampling methods: Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-background'),
|
||||
('Resampling methods: More Bootstrap background',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-more-bootstrap-background'),
|
||||
('Resampling methods: Bootstrap approach',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-approach'),
|
||||
('Resampling methods: Bootstrap steps',
|
||||
2,
|
||||
None,
|
||||
'resampling-methods-bootstrap-steps'),
|
||||
('Code example for the Bootstrap method',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-the-bootstrap-method'),
|
||||
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
||||
('The bias-variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'the-bias-variance-tradeoff'),
|
||||
('A way to Read the Bias-Variance Tradeoff',
|
||||
2,
|
||||
None,
|
||||
'a-way-to-read-the-bias-variance-tradeoff'),
|
||||
('Example code for Bias-Variance tradeoff',
|
||||
2,
|
||||
None,
|
||||
'example-code-for-bias-variance-tradeoff'),
|
||||
('Understanding what happens',
|
||||
2,
|
||||
None,
|
||||
'understanding-what-happens'),
|
||||
('Summing up', 2, None, 'summing-up'),
|
||||
("Another Example from Scikit-Learn's Repository",
|
||||
2,
|
||||
None,
|
||||
'another-example-from-scikit-learn-s-repository'),
|
||||
('Various steps in cross-validation',
|
||||
2,
|
||||
None,
|
||||
'various-steps-in-cross-validation'),
|
||||
('How to set up the cross-validation for Ridge and/or Lasso',
|
||||
2,
|
||||
None,
|
||||
'how-to-set-up-the-cross-validation-for-ridge-and-or-lasso'),
|
||||
('Cross-validation in brief',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-in-brief'),
|
||||
('Code Example for Cross-validation and $k$-fold '
|
||||
'Cross-validation',
|
||||
2,
|
||||
None,
|
||||
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
||||
('More examples on bootstrap and cross-validation and errors',
|
||||
2,
|
||||
None,
|
||||
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
||||
('The same example but now with cross-validation',
|
||||
2,
|
||||
None,
|
||||
'the-same-example-but-now-with-cross-validation'),
|
||||
('Cross-validation with Ridge',
|
||||
2,
|
||||
None,
|
||||
'cross-validation-with-ridge')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week37-bs.html">Week 37: Summary of Ridge and Lasso Regression and Resampling Methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37" style="font-size: 80%;">Plans for week 37</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs002.html#thursday-september-16-summary-of-ridge-and-lasso-regression-and-start-resampling-methods" style="font-size: 80%;">Thursday September 16, Summary of Ridge and Lasso Regression and start Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs003.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs004.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs005.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs006.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs007.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs008.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs009.html#test-function-for-what-happens-with-ols-ridge-and-lasso" style="font-size: 80%;">Test Function for what happens with OLS, Ridge and Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs010.html#rerunning-the-above-code" style="font-size: 80%;">Rerunning the above code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs011.html#invoking-bayes-theorem" style="font-size: 80%;">Invoking Bayes' theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs012.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs013.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs016.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs017.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs018.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs019.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs021.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs022.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs024.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs025.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs026.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs027.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs028.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs029.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs030.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs031.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs035.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs036.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs037.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs038.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs039.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs040.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs041.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs042.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs043.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs044.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0049"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="cross-validation-with-ridge" class="anchor">Cross-validation with Ridge </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> KFold
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> Ridge
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_val_score
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
|
||||
<span style="color: #408080; font-style: italic"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">3155</span>)
|
||||
<span style="color: #408080; font-style: italic"># Generate the data.</span>
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Decide degree on polynomial to fit</span>
|
||||
poly <span style="color: #666666">=</span> PolynomialFeatures(degree <span style="color: #666666">=</span> <span style="color: #666666">10</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Decide which values of lambda to use</span>
|
||||
nlambdas <span style="color: #666666">=</span> <span style="color: #666666">500</span>
|
||||
lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-3</span>, <span style="color: #666666">5</span>, nlambdas)
|
||||
<span style="color: #408080; font-style: italic"># Initialize a KFold instance</span>
|
||||
k <span style="color: #666666">=</span> <span style="color: #666666">5</span>
|
||||
kfold <span style="color: #666666">=</span> KFold(n_splits <span style="color: #666666">=</span> k)
|
||||
estimated_mse_sklearn <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(nlambdas)
|
||||
i <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> lmb <span style="color: #AA22FF; font-weight: bold">in</span> lambdas:
|
||||
ridge <span style="color: #666666">=</span> Ridge(alpha <span style="color: #666666">=</span> lmb)
|
||||
estimated_mse_folds <span style="color: #666666">=</span> cross_val_score(ridge, x, y, scoring<span style="color: #666666">=</span><span style="color: #BA2121">'neg_mean_squared_error'</span>, cv<span style="color: #666666">=</span>kfold)
|
||||
estimated_mse_sklearn[i] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(<span style="color: #666666">-</span>estimated_mse_folds)
|
||||
i <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
plt<span style="color: #666666">.</span>figure()
|
||||
plt<span style="color: #666666">.</span>plot(np<span style="color: #666666">.</span>log10(lambdas), estimated_mse_sklearn, label <span style="color: #666666">=</span> <span style="color: #BA2121">'cross_val_score'</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'log10(lambda)'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'MSE'</span>)
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week37-bs048.html">«</a></li>
|
||||
<li><a href="._week37-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week37-bs041.html">42</a></li>
|
||||
<li><a href="._week37-bs042.html">43</a></li>
|
||||
<li><a href="._week37-bs043.html">44</a></li>
|
||||
<li><a href="._week37-bs044.html">45</a></li>
|
||||
<li><a href="._week37-bs045.html">46</a></li>
|
||||
<li><a href="._week37-bs046.html">47</a></li>
|
||||
<li><a href="._week37-bs047.html">48</a></li>
|
||||
<li><a href="._week37-bs048.html">49</a></li>
|
||||
<li class="active"><a href="._week37-bs049.html">50</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user