Morten Hjorth-Jensen
361 lines
21 KiB
HTML
361 lines
21 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Statistical interpretations and Resampling Methods">
|
|
<title>Week 37: Statistical interpretations and Resampling Methods</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37, lecture Monday',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lecture-monday'),
|
|
('Plans for week 37, lab sessions',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lab-sessions'),
|
|
('Material for lecture Monday September 9',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-september-9'),
|
|
('Deriving OLS from a probability distribution',
|
|
2,
|
|
None,
|
|
'deriving-ols-from-a-probability-distribution'),
|
|
('Independent and Identically Distrubuted (iid)',
|
|
2,
|
|
None,
|
|
'independent-and-identically-distrubuted-iid'),
|
|
('Maximum Likelihood Estimation (MLE)',
|
|
2,
|
|
None,
|
|
'maximum-likelihood-estimation-mle'),
|
|
('A new Cost Function', 2, None, 'a-new-cost-function'),
|
|
("More basic Statistics and Bayes' theorem",
|
|
2,
|
|
None,
|
|
'more-basic-statistics-and-bayes-theorem'),
|
|
('Marginal Probability', 2, None, 'marginal-probability'),
|
|
('Conditional Probability', 2, None, 'conditional-probability'),
|
|
("Bayes' Theorem", 2, None, 'bayes-theorem'),
|
|
("Interpretations of Bayes' Theorem",
|
|
2,
|
|
None,
|
|
'interpretations-of-bayes-theorem'),
|
|
("Example of Usage of Bayes' theorem",
|
|
2,
|
|
None,
|
|
'example-of-usage-of-bayes-theorem'),
|
|
('Doing it correctly', 2, None, 'doing-it-correctly'),
|
|
("Bayes' Theorem and Ridge and Lasso Regression",
|
|
2,
|
|
None,
|
|
'bayes-theorem-and-ridge-and-lasso-regression'),
|
|
('Ridge and Bayes', 2, None, 'ridge-and-bayes'),
|
|
('Lasso and Bayes', 2, None, 'lasso-and-bayes'),
|
|
('Why resampling methods', 2, None, 'why-resampling-methods'),
|
|
('Resampling methods', 2, None, 'resampling-methods'),
|
|
('Resampling approaches can be computationally expensive',
|
|
2,
|
|
None,
|
|
'resampling-approaches-can-be-computationally-expensive'),
|
|
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
|
('Statistical analysis', 2, None, 'statistical-analysis'),
|
|
('Resampling methods', 2, None, 'resampling-methods'),
|
|
('Resampling methods: Bootstrap',
|
|
2,
|
|
None,
|
|
'resampling-methods-bootstrap'),
|
|
('The Central Limit Theorem',
|
|
2,
|
|
None,
|
|
'the-central-limit-theorem'),
|
|
('Finding the Limit', 2, None, 'finding-the-limit'),
|
|
('Rewriting the $\\delta$-function',
|
|
2,
|
|
None,
|
|
'rewriting-the-delta-function'),
|
|
('Identifying Terms', 2, None, 'identifying-terms'),
|
|
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
|
('Confidence Intervals', 2, None, 'confidence-intervals'),
|
|
('Standard Approach based on the Normal Distribution',
|
|
2,
|
|
None,
|
|
'standard-approach-based-on-the-normal-distribution'),
|
|
('Resampling methods: Bootstrap background',
|
|
2,
|
|
None,
|
|
'resampling-methods-bootstrap-background'),
|
|
('Resampling methods: More Bootstrap background',
|
|
2,
|
|
None,
|
|
'resampling-methods-more-bootstrap-background'),
|
|
('Resampling methods: Bootstrap approach',
|
|
2,
|
|
None,
|
|
'resampling-methods-bootstrap-approach'),
|
|
('Resampling methods: Bootstrap steps',
|
|
2,
|
|
None,
|
|
'resampling-methods-bootstrap-steps'),
|
|
('Code example for the Bootstrap method',
|
|
2,
|
|
None,
|
|
'code-example-for-the-bootstrap-method'),
|
|
('Plotting the Histogram', 2, None, 'plotting-the-histogram'),
|
|
('The bias-variance tradeoff',
|
|
2,
|
|
None,
|
|
'the-bias-variance-tradeoff'),
|
|
('A way to Read the Bias-Variance Tradeoff',
|
|
2,
|
|
None,
|
|
'a-way-to-read-the-bias-variance-tradeoff'),
|
|
('Example code for Bias-Variance tradeoff',
|
|
2,
|
|
None,
|
|
'example-code-for-bias-variance-tradeoff'),
|
|
('Understanding what happens',
|
|
2,
|
|
None,
|
|
'understanding-what-happens'),
|
|
('Summing up', 2, None, 'summing-up'),
|
|
("Another Example from Scikit-Learn's Repository",
|
|
2,
|
|
None,
|
|
'another-example-from-scikit-learn-s-repository'),
|
|
('Various steps in cross-validation',
|
|
2,
|
|
None,
|
|
'various-steps-in-cross-validation'),
|
|
('Cross-validation in brief',
|
|
2,
|
|
None,
|
|
'cross-validation-in-brief'),
|
|
('Code Example for Cross-validation and $k$-fold '
|
|
'Cross-validation',
|
|
2,
|
|
None,
|
|
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
|
('More examples on bootstrap and cross-validation and errors',
|
|
2,
|
|
None,
|
|
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
|
('The same example but now with cross-validation',
|
|
2,
|
|
None,
|
|
'the-same-example-but-now-with-cross-validation'),
|
|
('Material for the lab sessions',
|
|
2,
|
|
None,
|
|
'material-for-the-lab-sessions'),
|
|
('Linking the regression analysis with a statistical '
|
|
'interpretation',
|
|
2,
|
|
None,
|
|
'linking-the-regression-analysis-with-a-statistical-interpretation'),
|
|
('Assumptions made', 2, None, 'assumptions-made'),
|
|
('Expectation value and variance',
|
|
2,
|
|
None,
|
|
'expectation-value-and-variance'),
|
|
('Expectation value and variance for $\\boldsymbol{\\beta}$',
|
|
2,
|
|
None,
|
|
'expectation-value-and-variance-for-boldsymbol-beta')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week37-bs.html">Week 37: Statistical interpretations and Resampling Methods</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37-lecture-monday" style="font-size: 80%;">Plans for week 37, lecture Monday</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs002.html#plans-for-week-37-lab-sessions" style="font-size: 80%;">Plans for week 37, lab sessions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs003.html#material-for-lecture-monday-september-9" style="font-size: 80%;">Material for lecture Monday September 9</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs004.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs005.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs006.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs007.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs008.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs010.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs011.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs012.html#interpretations-of-bayes-theorem" style="font-size: 80%;">Interpretations of Bayes' Theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs013.html#example-of-usage-of-bayes-theorem" style="font-size: 80%;">Example of Usage of Bayes' theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs014.html#doing-it-correctly" style="font-size: 80%;">Doing it correctly</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs015.html#bayes-theorem-and-ridge-and-lasso-regression" style="font-size: 80%;">Bayes' Theorem and Ridge and Lasso Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs016.html#ridge-and-bayes" style="font-size: 80%;">Ridge and Bayes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs017.html#lasso-and-bayes" style="font-size: 80%;">Lasso and Bayes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs020.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs022.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs025.html#the-central-limit-theorem" style="font-size: 80%;">The Central Limit Theorem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs026.html#finding-the-limit" style="font-size: 80%;">Finding the Limit</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs027.html#rewriting-the-delta-function" style="font-size: 80%;">Rewriting the \( \delta \)-function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs028.html#identifying-terms" style="font-size: 80%;">Identifying Terms</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs029.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs030.html#confidence-intervals" style="font-size: 80%;">Confidence Intervals</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs031.html#standard-approach-based-on-the-normal-distribution" style="font-size: 80%;">Standard Approach based on the Normal Distribution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs032.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs033.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs034.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs036.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs037.html#plotting-the-histogram" style="font-size: 80%;">Plotting the Histogram</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs038.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;">A way to Read the Bias-Variance Tradeoff</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs040.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs041.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs042.html#summing-up" style="font-size: 80%;">Summing up</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs043.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs044.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs045.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs046.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs048.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs049.html#material-for-the-lab-sessions" style="font-size: 80%;">Material for the lab sessions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs050.html#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs051.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs052.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0024"></a>
|
|
<!-- !split -->
|
|
<h2 id="resampling-methods-bootstrap" class="anchor">Resampling methods: Bootstrap </h2>
|
|
<div class="panel panel-default">
|
|
<div class="panel-body">
|
|
<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
|
<p>Bootstrapping is a <a href="https://en.wikipedia.org/wiki/Nonparametric_statistics" target="_self">non-parametric approach</a> to statistical inference
|
|
that substitutes computation for more traditional distributional
|
|
assumptions and asymptotic results. Bootstrapping offers a number of
|
|
advantages:
|
|
</p>
|
|
<ol>
|
|
<li> The bootstrap is quite general, although there are some cases in which it fails.</li>
|
|
<li> Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small.</li>
|
|
<li> It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically.</li>
|
|
<li> It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples).</li>
|
|
</ol>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<p>The textbook by <a href="https://www.cambridge.org/core/books/bootstrap-methods-and-their-application/ED2FD043579F27952363566DC09CBD6A" target="_self">Davison on the Bootstrap Methods and their Applications</a> provides many more insights and proofs. In this course we will take a more practical approach and use the results and theorems provided in the literature. For those interested in reading more about the bootstrap methods, we recommend the above text and the one by <a href="https://www.routledge.com/An-Introduction-to-the-Bootstrap/Efron-Tibshirani/p/book/9780412042317" target="_self">Efron and Tibshirani</a>.</p>
|
|
|
|
<p>Before we proceed however, we need to remind ourselves about a central theorem in statistics, namely the so-called <b>central limit theorem</b>.</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week37-bs023.html">«</a></li>
|
|
<li><a href="._week37-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs016.html">17</a></li>
|
|
<li><a href="._week37-bs017.html">18</a></li>
|
|
<li><a href="._week37-bs018.html">19</a></li>
|
|
<li><a href="._week37-bs019.html">20</a></li>
|
|
<li><a href="._week37-bs020.html">21</a></li>
|
|
<li><a href="._week37-bs021.html">22</a></li>
|
|
<li><a href="._week37-bs022.html">23</a></li>
|
|
<li><a href="._week37-bs023.html">24</a></li>
|
|
<li class="active"><a href="._week37-bs024.html">25</a></li>
|
|
<li><a href="._week37-bs025.html">26</a></li>
|
|
<li><a href="._week37-bs026.html">27</a></li>
|
|
<li><a href="._week37-bs027.html">28</a></li>
|
|
<li><a href="._week37-bs028.html">29</a></li>
|
|
<li><a href="._week37-bs029.html">30</a></li>
|
|
<li><a href="._week37-bs030.html">31</a></li>
|
|
<li><a href="._week37-bs031.html">32</a></li>
|
|
<li><a href="._week37-bs032.html">33</a></li>
|
|
<li><a href="._week37-bs033.html">34</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs053.html">54</a></li>
|
|
<li><a href="._week37-bs025.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|