320 lines
19 KiB
HTML
320 lines
19 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 39: Resampling methods and logistic regression">
|
|
<title>Week 39: Resampling methods and logistic regression</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 39, September 22-26, 2025',
|
|
2,
|
|
None,
|
|
'plan-for-week-39-september-22-26-2025'),
|
|
('Readings and Videos, resampling methods',
|
|
2,
|
|
None,
|
|
'readings-and-videos-resampling-methods'),
|
|
('Readings and Videos, logistic regression',
|
|
2,
|
|
None,
|
|
'readings-and-videos-logistic-regression'),
|
|
('Lab sessions week 39', 2, None, 'lab-sessions-week-39'),
|
|
('Lecture material', 2, None, 'lecture-material'),
|
|
('Resampling methods', 2, None, 'resampling-methods'),
|
|
('Resampling approaches can be computationally expensive',
|
|
2,
|
|
None,
|
|
'resampling-approaches-can-be-computationally-expensive'),
|
|
('Why resampling methods ?', 2, None, 'why-resampling-methods'),
|
|
('Statistical analysis', 2, None, 'statistical-analysis'),
|
|
('Resampling methods', 2, None, 'resampling-methods'),
|
|
('Resampling methods: Bootstrap',
|
|
2,
|
|
None,
|
|
'resampling-methods-bootstrap'),
|
|
('The bias-variance tradeoff',
|
|
2,
|
|
None,
|
|
'the-bias-variance-tradeoff'),
|
|
('A way to Read the Bias-Variance Tradeoff',
|
|
2,
|
|
None,
|
|
'a-way-to-read-the-bias-variance-tradeoff'),
|
|
('Understanding what happens',
|
|
2,
|
|
None,
|
|
'understanding-what-happens'),
|
|
('Summing up', 2, None, 'summing-up'),
|
|
("Another Example from Scikit-Learn's Repository",
|
|
2,
|
|
None,
|
|
'another-example-from-scikit-learn-s-repository'),
|
|
('Various steps in cross-validation',
|
|
2,
|
|
None,
|
|
'various-steps-in-cross-validation'),
|
|
('Cross-validation in brief',
|
|
2,
|
|
None,
|
|
'cross-validation-in-brief'),
|
|
('Code Example for Cross-validation and $k$-fold '
|
|
'Cross-validation',
|
|
2,
|
|
None,
|
|
'code-example-for-cross-validation-and-k-fold-cross-validation'),
|
|
('More examples on bootstrap and cross-validation and errors',
|
|
2,
|
|
None,
|
|
'more-examples-on-bootstrap-and-cross-validation-and-errors'),
|
|
('The same example but now with cross-validation',
|
|
2,
|
|
None,
|
|
'the-same-example-but-now-with-cross-validation'),
|
|
('Logistic Regression', 2, None, 'logistic-regression'),
|
|
('Classification problems', 2, None, 'classification-problems'),
|
|
('Optimization and Deep learning',
|
|
2,
|
|
None,
|
|
'optimization-and-deep-learning'),
|
|
('Basics', 2, None, 'basics'),
|
|
('Linear classifier', 2, None, 'linear-classifier'),
|
|
('Some selected properties', 2, None, 'some-selected-properties'),
|
|
('Simple example', 2, None, 'simple-example'),
|
|
('Plotting the mean value for each group',
|
|
2,
|
|
None,
|
|
'plotting-the-mean-value-for-each-group'),
|
|
('The logistic function', 2, None, 'the-logistic-function'),
|
|
('Examples of likelihood functions used in logistic regression '
|
|
'and nueral networks',
|
|
2,
|
|
None,
|
|
'examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks'),
|
|
('Two parameters', 2, None, 'two-parameters'),
|
|
('Maximum likelihood', 2, None, 'maximum-likelihood'),
|
|
('The cost function rewritten',
|
|
2,
|
|
None,
|
|
'the-cost-function-rewritten'),
|
|
('Minimizing the cross entropy',
|
|
2,
|
|
None,
|
|
'minimizing-the-cross-entropy'),
|
|
('A more compact expression',
|
|
2,
|
|
None,
|
|
'a-more-compact-expression'),
|
|
('Extending to more predictors',
|
|
2,
|
|
None,
|
|
'extending-to-more-predictors'),
|
|
('Including more classes', 2, None, 'including-more-classes'),
|
|
('More classes', 2, None, 'more-classes'),
|
|
('Optimization, the central part of any Machine Learning '
|
|
'algortithm',
|
|
2,
|
|
None,
|
|
'optimization-the-central-part-of-any-machine-learning-algortithm'),
|
|
('Revisiting our Logistic Regression case',
|
|
2,
|
|
None,
|
|
'revisiting-our-logistic-regression-case'),
|
|
('The equations to solve', 2, None, 'the-equations-to-solve'),
|
|
("Solving using Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'solving-using-newton-raphson-s-method'),
|
|
('Example code for Logistic Regression',
|
|
2,
|
|
None,
|
|
'example-code-for-logistic-regression'),
|
|
('Synthetic data generation',
|
|
3,
|
|
None,
|
|
'synthetic-data-generation')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week39-bs.html">Week 39: Resampling methods and logistic regression</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week39-bs001.html#plan-for-week-39-september-22-26-2025" style="font-size: 80%;"><b>Plan for week 39, September 22-26, 2025</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs002.html#readings-and-videos-resampling-methods" style="font-size: 80%;"><b>Readings and Videos, resampling methods</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs003.html#readings-and-videos-logistic-regression" style="font-size: 80%;"><b>Readings and Videos, logistic regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs004.html#lab-sessions-week-39" style="font-size: 80%;"><b>Lab sessions week 39</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs005.html#lecture-material" style="font-size: 80%;"><b>Lecture material</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs010.html#resampling-methods" style="font-size: 80%;"><b>Resampling methods</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs007.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;"><b>Resampling approaches can be computationally expensive</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs008.html#why-resampling-methods" style="font-size: 80%;"><b>Why resampling methods ?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs009.html#statistical-analysis" style="font-size: 80%;"><b>Statistical analysis</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs010.html#resampling-methods" style="font-size: 80%;"><b>Resampling methods</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#resampling-methods-bootstrap" style="font-size: 80%;"><b>Resampling methods: Bootstrap</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs012.html#the-bias-variance-tradeoff" style="font-size: 80%;"><b>The bias-variance tradeoff</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs013.html#a-way-to-read-the-bias-variance-tradeoff" style="font-size: 80%;"><b>A way to Read the Bias-Variance Tradeoff</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs014.html#understanding-what-happens" style="font-size: 80%;"><b>Understanding what happens</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs015.html#summing-up" style="font-size: 80%;"><b>Summing up</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs016.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;"><b>Another Example from Scikit-Learn's Repository</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs017.html#various-steps-in-cross-validation" style="font-size: 80%;"><b>Various steps in cross-validation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs018.html#cross-validation-in-brief" style="font-size: 80%;"><b>Cross-validation in brief</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs019.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;"><b>Code Example for Cross-validation and \( k \)-fold Cross-validation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs020.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;"><b>More examples on bootstrap and cross-validation and errors</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs021.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;"><b>The same example but now with cross-validation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs022.html#logistic-regression" style="font-size: 80%;"><b>Logistic Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs023.html#classification-problems" style="font-size: 80%;"><b>Classification problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs024.html#optimization-and-deep-learning" style="font-size: 80%;"><b>Optimization and Deep learning</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs025.html#basics" style="font-size: 80%;"><b>Basics</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs026.html#linear-classifier" style="font-size: 80%;"><b>Linear classifier</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs027.html#some-selected-properties" style="font-size: 80%;"><b>Some selected properties</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs028.html#simple-example" style="font-size: 80%;"><b>Simple example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs029.html#plotting-the-mean-value-for-each-group" style="font-size: 80%;"><b>Plotting the mean value for each group</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs030.html#the-logistic-function" style="font-size: 80%;"><b>The logistic function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs031.html#examples-of-likelihood-functions-used-in-logistic-regression-and-nueral-networks" style="font-size: 80%;"><b>Examples of likelihood functions used in logistic regression and nueral networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs032.html#two-parameters" style="font-size: 80%;"><b>Two parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs033.html#maximum-likelihood" style="font-size: 80%;"><b>Maximum likelihood</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#the-cost-function-rewritten" style="font-size: 80%;"><b>The cost function rewritten</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs035.html#minimizing-the-cross-entropy" style="font-size: 80%;"><b>Minimizing the cross entropy</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs036.html#a-more-compact-expression" style="font-size: 80%;"><b>A more compact expression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs037.html#extending-to-more-predictors" style="font-size: 80%;"><b>Extending to more predictors</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs038.html#including-more-classes" style="font-size: 80%;"><b>Including more classes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs039.html#more-classes" style="font-size: 80%;"><b>More classes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs040.html#optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;"><b>Optimization, the central part of any Machine Learning algortithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs041.html#revisiting-our-logistic-regression-case" style="font-size: 80%;"><b>Revisiting our Logistic Regression case</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs042.html#the-equations-to-solve" style="font-size: 80%;"><b>The equations to solve</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs043.html#solving-using-newton-raphson-s-method" style="font-size: 80%;"><b>Solving using Newton-Raphson's method</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs044.html#example-code-for-logistic-regression" style="font-size: 80%;"><b>Example code for Logistic Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs044.html#synthetic-data-generation" style="font-size: 80%;"> Synthetic data generation</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0011"></a>
|
|
<!-- !split -->
|
|
<h2 id="resampling-methods-bootstrap" class="anchor">Resampling methods: Bootstrap </h2>
|
|
<div class="panel panel-default">
|
|
<div class="panel-body">
|
|
<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
|
<p>Bootstrapping is a <a href="https://en.wikipedia.org/wiki/Nonparametric_statistics" target="_self">non-parametric approach</a> to statistical inference
|
|
that substitutes computation for more traditional distributional
|
|
assumptions and asymptotic results. Bootstrapping offers a number of
|
|
advantages:
|
|
</p>
|
|
<ol>
|
|
<li> The bootstrap is quite general, although there are some cases in which it fails.</li>
|
|
<li> Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small.</li>
|
|
<li> It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically.</li>
|
|
<li> It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples).</li>
|
|
</ol>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<p>The textbook by <a href="https://www.cambridge.org/core/books/bootstrap-methods-and-their-application/ED2FD043579F27952363566DC09CBD6A" target="_self">Davison on the Bootstrap Methods and their Applications</a> provides many more insights and proofs. In this course we will take a more practical approach and use the results and theorems provided in the literature. For those interested in reading more about the bootstrap methods, we recommend the above text and the one by <a href="https://www.routledge.com/An-Introduction-to-the-Bootstrap/Efron-Tibshirani/p/book/9780412042317" target="_self">Efron and Tibshirani</a>.</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week39-bs010.html">«</a></li>
|
|
<li><a href="._week39-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week39-bs003.html">4</a></li>
|
|
<li><a href="._week39-bs004.html">5</a></li>
|
|
<li><a href="._week39-bs005.html">6</a></li>
|
|
<li><a href="._week39-bs006.html">7</a></li>
|
|
<li><a href="._week39-bs007.html">8</a></li>
|
|
<li><a href="._week39-bs008.html">9</a></li>
|
|
<li><a href="._week39-bs009.html">10</a></li>
|
|
<li><a href="._week39-bs010.html">11</a></li>
|
|
<li class="active"><a href="._week39-bs011.html">12</a></li>
|
|
<li><a href="._week39-bs012.html">13</a></li>
|
|
<li><a href="._week39-bs013.html">14</a></li>
|
|
<li><a href="._week39-bs014.html">15</a></li>
|
|
<li><a href="._week39-bs015.html">16</a></li>
|
|
<li><a href="._week39-bs016.html">17</a></li>
|
|
<li><a href="._week39-bs017.html">18</a></li>
|
|
<li><a href="._week39-bs018.html">19</a></li>
|
|
<li><a href="._week39-bs019.html">20</a></li>
|
|
<li><a href="._week39-bs020.html">21</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week39-bs044.html">45</a></li>
|
|
<li><a href="._week39-bs012.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|