This commit is contained in:
Morten Hjorth-Jensen
2022-10-04 17:50:07 +02:00
parent ca80391647
commit e48ba29bba
225 changed files with 6376 additions and 4975 deletions
+199 -117
View File
@@ -7,8 +7,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>4. Ridge and Lasso Regression &#8212; Applied Data Analysis and Machine Learning</title>
<link href="_static/css/theme.css" rel="stylesheet" />
<link href="_static/css/index.c5995385ac14fb8791e8eb36b4908be2.css" rel="stylesheet" />
<link href="_static/css/theme.css" rel="stylesheet">
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
<link rel="stylesheet"
@@ -31,31 +31,37 @@
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
<link rel="preload" as="script" href="_static/js/index.1c5a1a01449ed65a7b51.js">
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
<script src="_static/jquery.js"></script>
<script src="_static/underscore.js"></script>
<script src="_static/doctools.js"></script>
<script src="_static/togglebutton.js"></script>
<script src="_static/clipboard.min.js"></script>
<script src="_static/copybutton.js"></script>
<script>let toggleHintShow = 'Click to show';</script>
<script>let toggleHintHide = 'Click to hide';</script>
<script>let toggleOpenOnPrint = 'true';</script>
<script src="_static/togglebutton.js"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
<script src="_static/sphinx-book-theme.12a9622fbb08dcb3a2a40b2c02b83a57.js"></script>
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
const thebe_selector = ".thebe,.cell"
const thebe_selector_input = "pre"
const thebe_selector_output = ".output, .cell_output"
</script>
<script async="async" src="_static/sphinx-thebe.js"></script>
<script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="5. Resampling Methods" href="chapter3.html" />
<link rel="prev" title="3. Linear Regression" href="chapter1.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en" />
<meta name="docsearch:language" content="None">
<!-- Google Analytics -->
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
@@ -94,7 +100,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
About the course
</span>
@@ -116,7 +122,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Review of Statistics with Resampling Techniques and Linear Algebra
</span>
@@ -133,7 +139,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
From Regression to Support Vector Machines
</span>
@@ -170,7 +176,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Decision Trees, Ensemble Methods and Boosting
</span>
@@ -187,7 +193,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Dimensionality Reduction
</span>
@@ -204,7 +210,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Deep Learning Methods
</span>
@@ -281,7 +287,7 @@ const thebe_selector_output = ".output, .cell_output"
data-placement="left">.ipynb</button></a>
<!-- Download PDF via print -->
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
onClick="window.print()" data-toggle="tooltip" data-placement="left">.pdf</button>
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
</div>
</div>
@@ -299,7 +305,7 @@ const thebe_selector_output = ".output, .cell_output"
</div>
<!-- Table of contents -->
<div class="d-none d-md-block col-md-2 bd-toc show">
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
<div class="tocsection onthispage pt-5 pb-3">
<i class="fas fa-list"></i> Contents
@@ -384,7 +390,95 @@ const thebe_selector_output = ".output, .cell_output"
</div>
<div id="main-content" class="row">
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
<!-- Table of contents that is only displayed when printing the page -->
<div id="jb-print-docs-body" class="onlyprint">
<h1>Ridge and Lasso Regression</h1>
<!-- Table of contents -->
<div id="print-main-content">
<div id="jb-print-toc">
<div>
<h2> Contents </h2>
</div>
<nav aria-label="Page">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#mathematical-interpretation-of-ordinary-least-squares">
4.1. Mathematical Interpretation of Ordinary Least Squares
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#the-singular-value-decomposition">
4.2. The singular value decomposition
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#basic-math-of-the-svd">
4.3. Basic math of the SVD
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#codes-for-the-svd">
4.4. Codes for the SVD
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#code-for-svd-and-inversion-of-matrices">
4.5. Code for SVD and Inversion of Matrices
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#mathematics-of-the-svd-and-implications">
4.6. Mathematics of the SVD and implications
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#further-properties-important-for-our-analyses-later">
4.7. Further properties (important for our analyses later)
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#meet-the-covariance-matrix">
4.8. Meet the Covariance Matrix
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#linking-with-the-svd">
4.9. Linking with the SVD
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#id1">
4.10. Ridge and Lasso Regression
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#linking-the-regression-analysis-with-a-statistical-interpretation">
4.11. Linking the regression analysis with a statistical interpretation
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#deriving-ols-from-a-probability-distribution">
4.12. Deriving OLS from a probability distribution
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#bayes-theorem-and-ridge-and-lasso-regression">
4.13. Bayes Theorem and Ridge and Lasso Regression
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#linking-bayes-theorem-with-ridge-and-lasso-regression">
4.14. Linking Bayes Theorem with Ridge and Lasso Regression
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div>
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
@@ -736,13 +830,13 @@ The simple answer is to use the linear algebra function for the pseudoinverse, t
[2 4 5]
[3 5 6]]
test U
[[ 4.44089210e-16 -4.69484813e-16 -6.67314874e-16]
[-4.69484813e-16 -4.44089210e-16 -1.54041041e-16]
[-6.67314874e-16 -1.54041041e-16 1.11022302e-16]]
[[-2.22044605e-16 -7.49932427e-16 -8.23408962e-16]
[-7.49932427e-16 0.00000000e+00 4.77954956e-17]
[-8.23408962e-16 4.77954956e-17 2.22044605e-16]]
test VT
[[ 2.22044605e-16 3.78156479e-17 1.85278920e-16]
[ 3.78156479e-17 0.00000000e+00 -6.33166055e-17]
[ 1.85278920e-16 -6.33166055e-17 -1.11022302e-16]]
[[ 3.33066907e-16 -7.32066545e-17 3.32714903e-16]
[-7.32066545e-17 0.00000000e+00 -1.82997013e-16]
[ 3.32714903e-16 -1.82997013e-16 -3.33066907e-16]]
[[0. 0. 0.]
[0. 0. 0.]
[0. 0. 0.]]
@@ -1113,10 +1207,10 @@ covariance matrix through the <strong>np.linalg.eig()</strong> function.</p>
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>-0.02697521163514974
3.9835443722554817
[[ 0.98467494 3.11283168]
[ 3.11283168 10.68965135]]
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.04413933503955871
4.12330280229368
[[0.80162359 2.38222896]
[2.38222896 8.12167821]]
</pre></div>
</div>
</div>
@@ -1153,10 +1247,10 @@ a more brute force way. Here we scale the mean values for each column of the des
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.07898165660100093
1.6984511994530214
[[1. 0.63862189]
[0.63862189 1. ]]
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.06786925114666595
1.9635449873404844
[[1. 0.65522261]
[0.65522261 1. ]]
</pre></div>
</div>
</div>
@@ -1186,30 +1280,30 @@ this matrix we easily see that it is a positive definite matrix.</p>
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[ 1.49901244 6.93685631]
[-1.07535606 -4.3823886 ]
[-1.23168292 -4.3014751 ]
[ 1.37683438 3.52093124]
[ 1.31424359 3.63367582]
[-1.15269628 -1.86879198]
[-0.93188452 -4.26291585]
[-0.66295776 -1.21924917]
[ 0.29883607 -0.44270138]
[ 0.56565106 2.38605872]]
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[-0.27091656 -1.29083183]
[ 0.31980301 0.87495119]
[-0.10835935 1.61413333]
[ 0.5188328 2.80380438]
[-0.04996008 -1.95742107]
[ 1.19432526 2.68719389]
[ 0.19710439 1.35590603]
[-0.23857423 -2.50104946]
[-0.94054854 -2.09034902]
[-0.62170669 -1.49633743]]
0 1
0 1.499012 6.936856
1 -1.075356 -4.382389
2 -1.231683 -4.301475
3 1.376834 3.520931
4 1.314244 3.633676
5 -1.152696 -1.868792
6 -0.931885 -4.262916
7 -0.662958 -1.219249
8 0.298836 -0.442701
9 0.565651 2.386059
0 -0.270917 -1.290832
1 0.319803 0.874951
2 -0.108359 1.614133
3 0.518833 2.803804
4 -0.049960 -1.957421
5 1.194325 2.687194
6 0.197104 1.355906
7 -0.238574 -2.501049
8 -0.940549 -2.090349
9 -0.621707 -1.496337
0 1
0 1.000000 0.946393
1 0.946393 1.000000
0 1.000000 0.800615
1 0.800615 1.000000
</pre></div>
</div>
</div>
@@ -1266,37 +1360,37 @@ this matrix we easily see that it is a positive definite matrix.</p>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 0 1 2 3 4 5 6 7 \
0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1 0.0 0.096516 0.086951 0.097634 0.093624 0.089369 0.089758 0.086888
2 0.0 0.086951 0.079242 0.086994 0.083832 0.080479 0.079719 0.077419
3 0.0 0.097634 0.086994 0.105161 0.100303 0.095184 0.100358 0.096893
4 0.0 0.093624 0.083832 0.100303 0.095935 0.091315 0.095522 0.092409
5 0.0 0.089369 0.080479 0.095184 0.091315 0.087208 0.090451 0.087689
6 0.0 0.089758 0.079719 0.100358 0.095522 0.090451 0.098135 0.094657
7 0.0 0.086888 0.077419 0.096893 0.092409 0.087689 0.094657 0.091440
8 0.0 0.084096 0.075188 0.093497 0.089355 0.084979 0.091236 0.088271
9 0.0 0.081343 0.072994 0.090128 0.086322 0.082286 0.087833 0.085113
10 0.0 0.081707 0.072554 0.093593 0.089041 0.084278 0.093078 0.089775
11 0.0 0.079268 0.070562 0.090694 0.086420 0.081932 0.090174 0.087081
12 0.0 0.076942 0.068664 0.087910 0.083899 0.079676 0.087372 0.084477
13 0.0 0.074715 0.066850 0.085224 0.081468 0.077499 0.084657 0.081953
14 0.0 0.072575 0.065110 0.082620 0.079110 0.075389 0.082014 0.079493
1 0.0 0.084006 0.079882 0.084682 0.084092 0.083417 0.076315 0.076097
2 0.0 0.079882 0.077644 0.078542 0.078962 0.079424 0.069534 0.069977
3 0.0 0.084682 0.078542 0.090649 0.088758 0.086665 0.085105 0.084008
4 0.0 0.084092 0.078962 0.088758 0.087573 0.086250 0.082424 0.081832
5 0.0 0.083417 0.079424 0.086665 0.086250 0.085776 0.079486 0.079438
6 0.0 0.076315 0.069534 0.085105 0.082424 0.079486 0.082288 0.080588
7 0.0 0.076097 0.069977 0.084008 0.081832 0.079438 0.080588 0.079264
8 0.0 0.076022 0.070618 0.082990 0.081357 0.079553 0.078908 0.077986
9 0.0 0.076079 0.071460 0.082027 0.080984 0.079823 0.077219 0.076729
10 0.0 0.068075 0.061188 0.078143 0.075043 0.071666 0.077200 0.075149
11 0.0 0.067712 0.061308 0.077144 0.074420 0.071445 0.075770 0.074006
12 0.0 0.067499 0.061604 0.076264 0.073938 0.071388 0.074418 0.072955
13 0.0 0.067443 0.062089 0.075498 0.073597 0.071505 0.073134 0.071991
14 0.0 0.067547 0.062777 0.074845 0.073400 0.071804 0.071908 0.071106
8 9 10 11 12 13 14
0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1 0.084096 0.081343 0.081707 0.079268 0.076942 0.074715 0.072575
2 0.075188 0.072994 0.072554 0.070562 0.068664 0.066850 0.065110
3 0.093497 0.090128 0.093593 0.090694 0.087910 0.085224 0.082620
4 0.089355 0.086322 0.089041 0.086420 0.083899 0.081468 0.079110
5 0.084979 0.082286 0.084278 0.081932 0.079676 0.077499 0.075389
6 0.091236 0.087833 0.093078 0.090174 0.087372 0.084657 0.082014
7 0.088271 0.085113 0.089775 0.087081 0.084477 0.081953 0.079493
8 0.085345 0.082426 0.086518 0.084024 0.081613 0.079273 0.076990
9 0.082426 0.079741 0.083272 0.080973 0.078749 0.076588 0.074480
10 0.086518 0.083272 0.089374 0.086610 0.083935 0.081334 0.078795
11 0.084024 0.080973 0.086610 0.084017 0.081503 0.079056 0.076665
12 0.081613 0.078749 0.083935 0.081503 0.079142 0.076843 0.074592
13 0.079273 0.076588 0.081334 0.079056 0.076843 0.074684 0.072569
14 0.076990 0.074480 0.078795 0.076665 0.074592 0.072569 0.070585
1 0.076022 0.076079 0.068075 0.067712 0.067499 0.067443 0.067547
2 0.070618 0.071460 0.061188 0.061308 0.061604 0.062089 0.062777
3 0.082990 0.082027 0.078143 0.077144 0.076264 0.075498 0.074845
4 0.081357 0.080984 0.075043 0.074420 0.073938 0.073597 0.073400
5 0.079553 0.079823 0.071666 0.071445 0.071388 0.071505 0.071804
6 0.078908 0.077219 0.077200 0.075770 0.074418 0.073134 0.071908
7 0.077986 0.076729 0.075149 0.074006 0.072955 0.071991 0.071106
8 0.077140 0.076349 0.073080 0.072240 0.071510 0.070887 0.070370
9 0.076349 0.076066 0.070961 0.070443 0.070056 0.069801 0.069681
10 0.073080 0.070961 0.073601 0.071922 0.070288 0.068689 0.067110
11 0.072240 0.070443 0.071922 0.070466 0.069065 0.067709 0.066388
12 0.071510 0.070056 0.070288 0.069065 0.067907 0.066808 0.065761
13 0.070887 0.069801 0.068689 0.067709 0.066808 0.065983 0.065228
14 0.070370 0.069681 0.067110 0.066388 0.065761 0.065228 0.064787
</pre></div>
</div>
</div>
@@ -2584,7 +2678,7 @@ order to another one.</p>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[ 1.0169643 0.27924636 -1.4087793 1.03308408 0. ]
Test MSE OLS
0.958228616652075
0.9582286166520774
</pre></div>
</div>
<img alt="_images/chapter2_322_1.png" src="_images/chapter2_322_1.png" />
@@ -2993,54 +3087,42 @@ decreasing <span class="math notranslate nohighlight">\(\lambda\)</span> and shr
</div>
<div class='prev-next-bottom'>
<div id="prev">
<a class="left-prev" href="chapter1.html" title="previous page">
<i class="prevnext-label fas fa-angle-left"></i>
<div class="prevnext-info">
<p class="prevnext-label">previous</p>
<p class="prevnext-title"><span class="section-number">3. </span>Linear Regression</p>
</div>
</a>
<!-- Previous / next buttons -->
<div class='prev-next-area'>
<a class='left-prev' id="prev-link" href="chapter1.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title"><span class="section-number">3. </span>Linear Regression</p>
</div>
</a>
<a class='right-next' id="next-link" href="chapter3.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title"><span class="section-number">5. </span>Resampling Methods</p>
</div>
<div id="next">
<a class="right-next" href="chapter3.html" title="next page">
<div class="prevnext-info">
<p class="prevnext-label">next</p>
<p class="prevnext-title"><span class="section-number">5. </span>Resampling Methods</p>
</div>
<i class="prevnext-label fas fa-angle-right"></i>
</a>
</div>
<i class="fas fa-angle-right"></i>
</a>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p>
By Morten Hjorth-Jensen<br/>
&copy; Copyright 2021.<br/>
</p>
</div>
</footer>
<p>
By Morten Hjorth-Jensen<br/>
&copy; Copyright 2021.<br/>
</p>
</footer>
</main>
</div>
</div>
<script src="_static/js/index.1c5a1a01449ed65a7b51.js"></script>
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
</body>
</html>