mhjensen
274 lines
13 KiB
HTML
274 lines
13 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 36: Resampling techniques and Ordinary Least Square">
|
|
|
|
<title>Week 36: Resampling techniques and Ordinary Least Square</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 36', 2, None, '___sec0'),
|
|
('Thursday September 3', 2, None, '___sec1'),
|
|
('Why resampling methods', 2, None, '___sec2'),
|
|
('Resampling methods', 2, None, '___sec3'),
|
|
('Resampling approaches can be computationally expensive',
|
|
2,
|
|
None,
|
|
'___sec4'),
|
|
('Why resampling methods ?', 2, None, '___sec5'),
|
|
('Statistical analysis', 2, None, '___sec6'),
|
|
('Linking the regression analysis with a statistical '
|
|
'interpretation',
|
|
2,
|
|
None,
|
|
'___sec7'),
|
|
('Assumptions made', 2, None, '___sec8'),
|
|
('Expectation value and variance', 2, None, '___sec9'),
|
|
('Expectation value and variance for $\\boldsymbol{\\beta}$',
|
|
2,
|
|
None,
|
|
'___sec10'),
|
|
('Resampling methods', 2, None, '___sec11'),
|
|
('Resampling methods: Jackknife and Bootstrap',
|
|
2,
|
|
None,
|
|
'___sec12'),
|
|
('Resampling methods: Jackknife', 2, None, '___sec13'),
|
|
('Jackknife code example', 2, None, '___sec14'),
|
|
('Resampling methods: Bootstrap', 2, None, '___sec15'),
|
|
('Resampling methods: Bootstrap background', 2, None, '___sec16'),
|
|
('Resampling methods: More Bootstrap background',
|
|
2,
|
|
None,
|
|
'___sec17'),
|
|
('Resampling methods: Bootstrap approach', 2, None, '___sec18'),
|
|
('Resampling methods: Bootstrap steps', 2, None, '___sec19'),
|
|
('Code example for the Bootstrap method', 2, None, '___sec20'),
|
|
('Various steps in cross-validation', 2, None, '___sec21'),
|
|
('How to set up the cross-validation for Ridge and/or Lasso',
|
|
2,
|
|
None,
|
|
'___sec22'),
|
|
('Cross-validation in brief', 2, None, '___sec23'),
|
|
('Code Example for Cross-validation and $k$-fold '
|
|
'Cross-validation',
|
|
2,
|
|
None,
|
|
'___sec24'),
|
|
('The bias-variance tradeoff', 2, None, '___sec25'),
|
|
('Example code for Bias-Variance tradeoff', 2, None, '___sec26'),
|
|
('Understanding what happens', 2, None, '___sec27'),
|
|
('Summing up', 2, None, '___sec28'),
|
|
("Another Example from Scikit-Learn's Repository",
|
|
2,
|
|
None,
|
|
'___sec29'),
|
|
('More examples on bootstrap and cross-validation and errors',
|
|
2,
|
|
None,
|
|
'___sec30'),
|
|
('The same example but now with cross-validation',
|
|
2,
|
|
None,
|
|
'___sec31'),
|
|
('Cross-validation with Ridge', 2, None, '___sec32'),
|
|
('Friday September 4', 2, None, '___sec33')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week36-bs.html">Week 36: Resampling techniques and Ordinary Least Square</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week36-bs001.html#___sec0" style="font-size: 80%;">Plans for week 36</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs002.html#___sec1" style="font-size: 80%;">Thursday September 3</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs003.html#___sec2" style="font-size: 80%;">Why resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs004.html#___sec3" style="font-size: 80%;">Resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs005.html#___sec4" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs006.html#___sec5" style="font-size: 80%;">Why resampling methods ?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs007.html#___sec6" style="font-size: 80%;">Statistical analysis</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs008.html#___sec7" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs009.html#___sec8" style="font-size: 80%;">Assumptions made</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs010.html#___sec9" style="font-size: 80%;">Expectation value and variance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs011.html#___sec10" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">Resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs013.html#___sec12" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs014.html#___sec13" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs015.html#___sec14" style="font-size: 80%;">Jackknife code example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs016.html#___sec15" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs017.html#___sec16" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs018.html#___sec17" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs019.html#___sec18" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs020.html#___sec19" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs021.html#___sec20" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs022.html#___sec21" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs023.html#___sec22" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs024.html#___sec23" style="font-size: 80%;">Cross-validation in brief</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs025.html#___sec24" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs026.html#___sec25" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs027.html#___sec26" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs028.html#___sec27" style="font-size: 80%;">Understanding what happens</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs029.html#___sec28" style="font-size: 80%;">Summing up</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs030.html#___sec29" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs031.html#___sec30" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs032.html#___sec31" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs033.html#___sec32" style="font-size: 80%;">Cross-validation with Ridge</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week36-bs034.html#___sec33" style="font-size: 80%;">Friday September 4</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0012"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec11" class="anchor">Resampling methods </h2>
|
|
|
|
<p>
|
|
With all these analytical equations for both the OLS and Ridge
|
|
regression, we will now outline how to assess a given model. This will
|
|
lead us to a discussion of the so-called bias-variance tradeoff (see
|
|
below) and so-called resampling methods.
|
|
|
|
<p>
|
|
One of the quantities we have discussed as a way to measure errors is
|
|
the mean-squared error (MSE), mainly used for fitting of continuous
|
|
functions. Another choice is the absolute error.
|
|
|
|
<p>
|
|
In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data,
|
|
we discuss the
|
|
|
|
<ol>
|
|
<li> prediction error or simply the <b>test error</b> \( \mathrm{Err_{Test}} \), where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the</li>
|
|
<li> training error \( \mathrm{Err_{Train}} \), which is the average loss over the training data.</li>
|
|
</ol>
|
|
|
|
As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error.
|
|
For a certain level of complexity the test error will reach minimum, before starting to increase again. The
|
|
training error reaches a saturation.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week36-bs011.html">«</a></li>
|
|
<li><a href="._week36-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week36-bs004.html">5</a></li>
|
|
<li><a href="._week36-bs005.html">6</a></li>
|
|
<li><a href="._week36-bs006.html">7</a></li>
|
|
<li><a href="._week36-bs007.html">8</a></li>
|
|
<li><a href="._week36-bs008.html">9</a></li>
|
|
<li><a href="._week36-bs009.html">10</a></li>
|
|
<li><a href="._week36-bs010.html">11</a></li>
|
|
<li><a href="._week36-bs011.html">12</a></li>
|
|
<li class="active"><a href="._week36-bs012.html">13</a></li>
|
|
<li><a href="._week36-bs013.html">14</a></li>
|
|
<li><a href="._week36-bs014.html">15</a></li>
|
|
<li><a href="._week36-bs015.html">16</a></li>
|
|
<li><a href="._week36-bs016.html">17</a></li>
|
|
<li><a href="._week36-bs017.html">18</a></li>
|
|
<li><a href="._week36-bs018.html">19</a></li>
|
|
<li><a href="._week36-bs019.html">20</a></li>
|
|
<li><a href="._week36-bs020.html">21</a></li>
|
|
<li><a href="._week36-bs021.html">22</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week36-bs034.html">35</a></li>
|
|
<li><a href="._week36-bs013.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|