lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added svd algo

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2021-09-09 09:42:41 +02:00
parent f4d8b8ceca
commit f6d429b2ce
7 changed files with 343 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week36/html/week36-bs.html
+65 -60
View File
@@ -56,6 +56,10 @@ Automatically generated HTML file from DocOnce source
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Code for SVD and Inversion of Matrices',
2,
None,
'code-for-svd-and-inversion-of-matrices'),
('Ridge and LASSO Regression',
2,
None,
@@ -269,65 +273,66 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week36-bs004.html#linear-regression-and-the-svd" style="font-size: 80%;">Linear Regression and the SVD</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs005.html#what-does-it-mean" style="font-size: 80%;">What does it mean?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs006.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;">And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs007.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs008.html#from-ols-to-ridge-and-lasso" style="font-size: 80%;">From OLS to Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs009.html#deriving-the-ridge-regression-equations" style="font-size: 80%;">Deriving the Ridge Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs010.html#svd-analysis" style="font-size: 80%;">SVD analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs011.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs012.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs013.html#deriving-the-lasso-regression-equations" style="font-size: 80%;">Deriving the Lasso Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs014.html#simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression" style="font-size: 80%;">Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs015.html#ridge-regression" style="font-size: 80%;">Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs016.html#lasso-regression" style="font-size: 80%;">Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs017.html#yet-another-example" style="font-size: 80%;">Yet another Example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs018.html#the-ols-case" style="font-size: 80%;">The OLS case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs019.html#the-ridge-case" style="font-size: 80%;">The Ridge case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs020.html#writing-the-cost-function" style="font-size: 80%;">Writing the Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs021.html#lasso-case" style="font-size: 80%;">Lasso case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs022.html#the-first-case" style="font-size: 80%;">The first Case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs023.html#simple-code-for-solving-the-above-problem" style="font-size: 80%;">Simple code for solving the above problem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs024.html#with-lasso-regression" style="font-size: 80%;">With Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs025.html#another-example-now-with-a-polynomial-fit" style="font-size: 80%;">Another Example, now with a polynomial fit</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs026.html#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs027.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs028.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs029.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs030.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs031.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs032.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs033.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs034.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs035.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs036.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs037.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs038.html#frliday-september-10" style="font-size: 80%;">Frliday September 10</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs042.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs044.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs041.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs042.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs043.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs044.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs045.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs046.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs047.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs048.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs049.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs050.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs051.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs052.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs053.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs054.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs055.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs056.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs057.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs059.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs060.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs061.html#summing-up" style="font-size: 80%;">Summing up</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs062.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs063.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs064.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs065.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs007.html#code-for-svd-and-inversion-of-matrices" style="font-size: 80%;">Code for SVD and Inversion of Matrices</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs008.html#ridge-and-lasso-regression" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs009.html#from-ols-to-ridge-and-lasso" style="font-size: 80%;">From OLS to Ridge and Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs010.html#deriving-the-ridge-regression-equations" style="font-size: 80%;">Deriving the Ridge Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs011.html#svd-analysis" style="font-size: 80%;">SVD analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs012.html#interpreting-the-ridge-results" style="font-size: 80%;">Interpreting the Ridge results</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs013.html#more-interpretations" style="font-size: 80%;">More interpretations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs014.html#deriving-the-lasso-regression-equations" style="font-size: 80%;">Deriving the Lasso Regression Equations</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs015.html#simple-example-to-illustrate-ordinary-least-squares-ridge-and-lasso-regression" style="font-size: 80%;">Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs016.html#ridge-regression" style="font-size: 80%;">Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs017.html#lasso-regression" style="font-size: 80%;">Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs018.html#yet-another-example" style="font-size: 80%;">Yet another Example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs019.html#the-ols-case" style="font-size: 80%;">The OLS case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs020.html#the-ridge-case" style="font-size: 80%;">The Ridge case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs021.html#writing-the-cost-function" style="font-size: 80%;">Writing the Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs022.html#lasso-case" style="font-size: 80%;">Lasso case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs023.html#the-first-case" style="font-size: 80%;">The first Case</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs024.html#simple-code-for-solving-the-above-problem" style="font-size: 80%;">Simple code for solving the above problem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs025.html#with-lasso-regression" style="font-size: 80%;">With Lasso Regression</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs026.html#another-example-now-with-a-polynomial-fit" style="font-size: 80%;">Another Example, now with a polynomial fit</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs027.html#linking-the-regression-analysis-with-a-statistical-interpretation" style="font-size: 80%;">Linking the regression analysis with a statistical interpretation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs028.html#assumptions-made" style="font-size: 80%;">Assumptions made</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs029.html#expectation-value-and-variance" style="font-size: 80%;">Expectation value and variance</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs030.html#expectation-value-and-variance-for-boldsymbol-beta" style="font-size: 80%;">Expectation value and variance for \( \boldsymbol{\beta} \)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs031.html#deriving-ols-from-a-probability-distribution" style="font-size: 80%;">Deriving OLS from a probability distribution</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs032.html#independent-and-identically-distrubuted-iid" style="font-size: 80%;">Independent and Identically Distrubuted (iid)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs033.html#maximum-likelihood-estimation-mle" style="font-size: 80%;">Maximum Likelihood Estimation (MLE)</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs034.html#a-new-cost-function" style="font-size: 80%;">A new Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs035.html#more-basic-statistics-and-bayes-theorem" style="font-size: 80%;">More basic Statistics and Bayes' theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs036.html#marginal-probability" style="font-size: 80%;">Marginal Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs037.html#conditional-probability" style="font-size: 80%;">Conditional Probability</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs038.html#bayes-theorem" style="font-size: 80%;">Bayes' Theorem</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs039.html#frliday-september-10" style="font-size: 80%;">Frliday September 10</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs043.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs045.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs042.html#resampling-approaches-can-be-computationally-expensive" style="font-size: 80%;">Resampling approaches can be computationally expensive</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs043.html#why-resampling-methods" style="font-size: 80%;">Why resampling methods ?</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs044.html#statistical-analysis" style="font-size: 80%;">Statistical analysis</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs045.html#resampling-methods" style="font-size: 80%;">Resampling methods</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs046.html#resampling-methods-jackknife-and-bootstrap" style="font-size: 80%;">Resampling methods: Jackknife and Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs047.html#resampling-methods-jackknife" style="font-size: 80%;">Resampling methods: Jackknife</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs048.html#jackknife-code-example" style="font-size: 80%;">Jackknife code example</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs049.html#resampling-methods-bootstrap" style="font-size: 80%;">Resampling methods: Bootstrap</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs050.html#resampling-methods-bootstrap-background" style="font-size: 80%;">Resampling methods: Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs051.html#resampling-methods-more-bootstrap-background" style="font-size: 80%;">Resampling methods: More Bootstrap background</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs052.html#resampling-methods-bootstrap-approach" style="font-size: 80%;">Resampling methods: Bootstrap approach</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs053.html#resampling-methods-bootstrap-steps" style="font-size: 80%;">Resampling methods: Bootstrap steps</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs054.html#code-example-for-the-bootstrap-method" style="font-size: 80%;">Code example for the Bootstrap method</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs055.html#various-steps-in-cross-validation" style="font-size: 80%;">Various steps in cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs056.html#how-to-set-up-the-cross-validation-for-ridge-and-or-lasso" style="font-size: 80%;">How to set up the cross-validation for Ridge and/or Lasso</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs057.html#cross-validation-in-brief" style="font-size: 80%;">Cross-validation in brief</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs058.html#code-example-for-cross-validation-and-k-fold-cross-validation" style="font-size: 80%;">Code Example for Cross-validation and \( k \)-fold Cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs059.html#the-bias-variance-tradeoff" style="font-size: 80%;">The bias-variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs060.html#example-code-for-bias-variance-tradeoff" style="font-size: 80%;">Example code for Bias-Variance tradeoff</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs061.html#understanding-what-happens" style="font-size: 80%;">Understanding what happens</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs062.html#summing-up" style="font-size: 80%;">Summing up</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs063.html#another-example-from-scikit-learn-s-repository" style="font-size: 80%;">Another Example from Scikit-Learn's Repository</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs064.html#more-examples-on-bootstrap-and-cross-validation-and-errors" style="font-size: 80%;">More examples on bootstrap and cross-validation and errors</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs065.html#the-same-example-but-now-with-cross-validation" style="font-size: 80%;">The same example but now with cross-validation</a></li>
<!-- navigation toc: --> <li><a href="._week36-bs066.html#cross-validation-with-ridge" style="font-size: 80%;">Cross-validation with Ridge</a></li>
</ul>
</li>
@@ -386,7 +391,7 @@ MathJax.Hub.Config({
<li><a href="._week36-bs008.html">9</a></li>
<li><a href="._week36-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._week36-bs065.html">66</a></li>
<li><a href="._week36-bs066.html">67</a></li>
<li><a href="._week36-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/week36/html/week36-reveal.html
+51 -6
View File
@@ -328,6 +328,55 @@ values and the column vectors of \( \boldsymbol{V} \).
</section>
<section>
<h2 id="code-for-svd-and-inversion-of-matrices">Code for SVD and Inversion of Matrices </h2>
<p>
How do we use the SVD to invert a matrix \( \boldsymbol{X}^\boldsymbol{X} \) which is singular or near singular?
The simple answer is to use the linear algebra function for pseudoinvers, that is
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>Ainv = np.linlag.pinv(A)
</pre></div>
<p>
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #228B22"># SVD inversion</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">SVDinv</span>(A):
<span style="color: #CD5555">&#39;&#39;&#39; Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
<span style="color: #CD5555"> SVD is numerically more stable than the inversion algorithms provided by</span>
<span style="color: #CD5555"> numpy and scipy.linalg at the cost of being slower.</span>
<span style="color: #CD5555"> &#39;&#39;&#39;</span>
U, s, VT = np.linalg.svd(A)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;test U&#39;</span>)
<span style="color: #658b00">print</span>( (np.transpose(U) @ U - U <span style="color: #707a7c">@np</span>.transpose(U)))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;test VT&#39;</span>)
<span style="color: #658b00">print</span>( (np.transpose(VT) @ VT - VT <span style="color: #707a7c">@np</span>.transpose(VT)))
D = np.zeros((<span style="color: #658b00">len</span>(U),<span style="color: #658b00">len</span>(VT)))
D = np.diag(s)
UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
<span style="color: #8B008B; font-weight: bold">return</span> np.matmul(V,np.matmul(invD,UT))
<span style="color: #228B22">#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])</span>
X = np.array( [ [<span style="color: #B452CD">1</span>,<span style="color: #B452CD">2</span>],[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">3</span>]])
<span style="color: #658b00">print</span>(X)
A = np.transpose(X) @ X
<span style="color: #228B22"># Brute force inversion</span>
B = np.linalg.inv(A)
C = SVDinv(A)
<span style="color: #658b00">print</span>(np.abs(B-C))
</pre></div>
</section>
<section>
<h2 id="ridge-and-lasso-regression">Ridge and LASSO Regression </h2>
@@ -977,14 +1026,14 @@ lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
lmb = lambdas[i]
Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y
<span style="color: #228B22"># print(Ridgebeta)</span>
<span style="color: #658b00">print</span>(Ridgebeta)
<span style="color: #228B22"># and then make the prediction</span>
ypredictRidge = X @ Ridgebeta
MSERidgePredict[i] = MSE(y,ypredictRidge)
<span style="color: #228B22"># print(MSEPredict[i])</span>
RegLasso = linear_model.Lasso(lmb)
RegLasso.fit(X,y)
ypredictLasso = RegLasso.predict(X)
<span style="color: #658b00">print</span>(RegLasso_coef_)
MSELassoPredict[i] = MSE(y,ypredictLasso)
<span style="color: #228B22"># Now plot the results</span>
plt.figure()
@@ -1040,13 +1089,9 @@ OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
<span style="color: #658b00">print</span>(OLSbeta)
<span style="color: #228B22"># and then make the prediction</span>
ytildeOLS = X_train @ OLSbeta
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Training R2 for OLS&quot;</span>)
<span style="color: #658b00">print</span>(R2(y_train,ytildeOLS))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Training MSE for OLS&quot;</span>)
<span style="color: #658b00">print</span>(MSE(y_train,ytildeOLS))
ypredictOLS = X_test @ OLSbeta
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test R2 for OLS&quot;</span>)
<span style="color: #658b00">print</span>(R2(y_test,ypredictOLS))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test MSE OLS&quot;</span>)
<span style="color: #658b00">print</span>(MSE(y_test,ypredictOLS))
doc/pub/week36/html/week36-solarized.html
+54 -6
View File
@@ -76,6 +76,10 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Code for SVD and Inversion of Matrices',
2,
None,
'code-for-svd-and-inversion-of-matrices'),
('Ridge and LASSO Regression',
2,
None,
@@ -438,6 +442,54 @@ values and the column vectors of \( \boldsymbol{V} \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="code-for-svd-and-inversion-of-matrices">Code for SVD and Inversion of Matrices </h2>
<p>
How do we use the SVD to invert a matrix \( \boldsymbol{X}^\boldsymbol{X} \) which is singular or near singular?
The simple answer is to use the linear algebra function for pseudoinvers, that is
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>Ainv = np.linlag.pinv(A)
</pre></div>
<p>
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #228B22"># SVD inversion</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">SVDinv</span>(A):
<span style="color: #CD5555">&#39;&#39;&#39; Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
<span style="color: #CD5555"> SVD is numerically more stable than the inversion algorithms provided by</span>
<span style="color: #CD5555"> numpy and scipy.linalg at the cost of being slower.</span>
<span style="color: #CD5555"> &#39;&#39;&#39;</span>
U, s, VT = np.linalg.svd(A)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;test U&#39;</span>)
<span style="color: #658b00">print</span>( (np.transpose(U) @ U - U <span style="color: #707a7c">@np</span>.transpose(U)))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;test VT&#39;</span>)
<span style="color: #658b00">print</span>( (np.transpose(VT) @ VT - VT <span style="color: #707a7c">@np</span>.transpose(VT)))
D = np.zeros((<span style="color: #658b00">len</span>(U),<span style="color: #658b00">len</span>(VT)))
D = np.diag(s)
UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
<span style="color: #8B008B; font-weight: bold">return</span> np.matmul(V,np.matmul(invD,UT))
<span style="color: #228B22">#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])</span>
X = np.array( [ [<span style="color: #B452CD">1</span>,<span style="color: #B452CD">2</span>],[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">3</span>]])
<span style="color: #658b00">print</span>(X)
A = np.transpose(X) @ X
<span style="color: #228B22"># Brute force inversion</span>
B = np.linalg.inv(A)
C = SVDinv(A)
<span style="color: #658b00">print</span>(np.abs(B-C))
</pre></div>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="ridge-and-lasso-regression">Ridge and LASSO Regression </h2>
<p>
@@ -995,14 +1047,14 @@ lambdas = np.logspace(-<span style="color: #B452CD">4</span>, <span style="color
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(nlambdas):
lmb = lambdas[i]
Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y
<span style="color: #228B22"># print(Ridgebeta)</span>
<span style="color: #658b00">print</span>(Ridgebeta)
<span style="color: #228B22"># and then make the prediction</span>
ypredictRidge = X @ Ridgebeta
MSERidgePredict[i] = MSE(y,ypredictRidge)
<span style="color: #228B22"># print(MSEPredict[i])</span>
RegLasso = linear_model.Lasso(lmb)
RegLasso.fit(X,y)
ypredictLasso = RegLasso.predict(X)
<span style="color: #658b00">print</span>(RegLasso_coef_)
MSELassoPredict[i] = MSE(y,ypredictLasso)
<span style="color: #228B22"># Now plot the results</span>
plt.figure()
@@ -1057,13 +1109,9 @@ OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
<span style="color: #658b00">print</span>(OLSbeta)
<span style="color: #228B22"># and then make the prediction</span>
ytildeOLS = X_train @ OLSbeta
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Training R2 for OLS&quot;</span>)
<span style="color: #658b00">print</span>(R2(y_train,ytildeOLS))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Training MSE for OLS&quot;</span>)
<span style="color: #658b00">print</span>(MSE(y_train,ytildeOLS))
ypredictOLS = X_test @ OLSbeta
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test R2 for OLS&quot;</span>)
<span style="color: #658b00">print</span>(R2(y_test,ypredictOLS))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test MSE OLS&quot;</span>)
<span style="color: #658b00">print</span>(MSE(y_test,ypredictOLS))
doc/pub/week36/html/week36.html
+54 -6
View File
@@ -81,6 +81,10 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Code for SVD and Inversion of Matrices',
2,
None,
'code-for-svd-and-inversion-of-matrices'),
('Ridge and LASSO Regression',
2,
None,
@@ -443,6 +447,54 @@ values and the column vectors of \( \boldsymbol{V} \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="code-for-svd-and-inversion-of-matrices">Code for SVD and Inversion of Matrices </h2>
<p>
How do we use the SVD to invert a matrix \( \boldsymbol{X}^\boldsymbol{X} \) which is singular or near singular?
The simple answer is to use the linear algebra function for pseudoinvers, that is
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>Ainv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linlag<span style="color: #666666">.</span>pinv(A)
</pre></div>
<p>
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #408080; font-style: italic"># SVD inversion</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">SVDinv</span>(A):
<span style="color: #BA2121; font-style: italic">&#39;&#39;&#39; Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
<span style="color: #BA2121; font-style: italic"> SVD is numerically more stable than the inversion algorithms provided by</span>
<span style="color: #BA2121; font-style: italic"> numpy and scipy.linalg at the cost of being slower.</span>
<span style="color: #BA2121; font-style: italic"> &#39;&#39;&#39;</span>
U, s, VT <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>svd(A)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;test U&#39;</span>)
<span style="color: #008000">print</span>( (np<span style="color: #666666">.</span>transpose(U) <span style="color: #666666">@</span> U <span style="color: #666666">-</span> U <span style="color: #AA22FF">@np</span><span style="color: #666666">.</span>transpose(U)))
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;test VT&#39;</span>)
<span style="color: #008000">print</span>( (np<span style="color: #666666">.</span>transpose(VT) <span style="color: #666666">@</span> VT <span style="color: #666666">-</span> VT <span style="color: #AA22FF">@np</span><span style="color: #666666">.</span>transpose(VT)))
D <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(U),<span style="color: #008000">len</span>(VT)))
D <span style="color: #666666">=</span> np<span style="color: #666666">.</span>diag(s)
UT <span style="color: #666666">=</span> np<span style="color: #666666">.</span>transpose(U); V <span style="color: #666666">=</span> np<span style="color: #666666">.</span>transpose(VT); invD <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(D)
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>matmul(V,np<span style="color: #666666">.</span>matmul(invD,UT))
<span style="color: #408080; font-style: italic">#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ [<span style="color: #666666">1</span>,<span style="color: #666666">2</span>],[<span style="color: #666666">2</span>,<span style="color: #666666">3</span>]])
<span style="color: #008000">print</span>(X)
A <span style="color: #666666">=</span> np<span style="color: #666666">.</span>transpose(X) <span style="color: #666666">@</span> X
<span style="color: #408080; font-style: italic"># Brute force inversion</span>
B <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(A)
C <span style="color: #666666">=</span> SVDinv(A)
<span style="color: #008000">print</span>(np<span style="color: #666666">.</span>abs(B<span style="color: #666666">-</span>C))
</pre></div>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="ridge-and-lasso-regression">Ridge and LASSO Regression </h2>
<p>
@@ -1000,14 +1052,14 @@ lambdas <span style="color: #666666">=</span> np<span style="color: #666666">.</
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(nlambdas):
lmb <span style="color: #666666">=</span> lambdas[i]
Ridgebeta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #666666">+</span>lmb<span style="color: #666666">*</span>I) <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
<span style="color: #408080; font-style: italic"># print(Ridgebeta)</span>
<span style="color: #008000">print</span>(Ridgebeta)
<span style="color: #408080; font-style: italic"># and then make the prediction</span>
ypredictRidge <span style="color: #666666">=</span> X <span style="color: #666666">@</span> Ridgebeta
MSERidgePredict[i] <span style="color: #666666">=</span> MSE(y,ypredictRidge)
<span style="color: #408080; font-style: italic"># print(MSEPredict[i])</span>
RegLasso <span style="color: #666666">=</span> linear_model<span style="color: #666666">.</span>Lasso(lmb)
RegLasso<span style="color: #666666">.</span>fit(X,y)
ypredictLasso <span style="color: #666666">=</span> RegLasso<span style="color: #666666">.</span>predict(X)
<span style="color: #008000">print</span>(RegLasso_coef_)
MSELassoPredict[i] <span style="color: #666666">=</span> MSE(y,ypredictLasso)
<span style="color: #408080; font-style: italic"># Now plot the results</span>
plt<span style="color: #666666">.</span>figure()
@@ -1062,13 +1114,9 @@ OLSbeta <span style="color: #666666">=</span> np<span style="color: #666666">.</
<span style="color: #008000">print</span>(OLSbeta)
<span style="color: #408080; font-style: italic"># and then make the prediction</span>
ytildeOLS <span style="color: #666666">=</span> X_train <span style="color: #666666">@</span> OLSbeta
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Training R2 for OLS&quot;</span>)
<span style="color: #008000">print</span>(R2(y_train,ytildeOLS))
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Training MSE for OLS&quot;</span>)
<span style="color: #008000">print</span>(MSE(y_train,ytildeOLS))
ypredictOLS <span style="color: #666666">=</span> X_test <span style="color: #666666">@</span> OLSbeta
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test R2 for OLS&quot;</span>)
<span style="color: #008000">print</span>(R2(y_test,ypredictOLS))
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test MSE OLS&quot;</span>)
<span style="color: #008000">print</span>(MSE(y_test,ypredictOLS))
doc/pub/week36/ipynb/ipynb-week36-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week36/ipynb/week36.ipynb
+71 -6
View File
@@ -254,6 +254,75 @@
"values and the column vectors of $\\boldsymbol{V}$.\n",
"\n",
"\n",
"\n",
"## Code for SVD and Inversion of Matrices\n",
"\n",
"How do we use the SVD to invert a matrix $\\boldsymbol{X}^\\boldsymbol{X}$ which is singular or near singular?\n",
"The simple answer is to use the linear algebra function for pseudoinvers, that is"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"Ainv = np.linlag.pinv(A)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"import numpy as np\n",
"# SVD inversion\n",
"def SVDinv(A):\n",
" ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n",
" SVD is numerically more stable than the inversion algorithms provided by\n",
" numpy and scipy.linalg at the cost of being slower.\n",
" '''\n",
" U, s, VT = np.linalg.svd(A)\n",
" print('test U')\n",
" print( (np.transpose(U) @ U - U @np.transpose(U)))\n",
" print('test VT')\n",
" print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n",
"\n",
"\n",
" D = np.zeros((len(U),len(VT)))\n",
" D = np.diag(s)\n",
" UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n",
" return np.matmul(V,np.matmul(invD,UT))\n",
"\n",
"\n",
"#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n",
"X = np.array( [ [1,2],[2,3]])\n",
"print(X)\n",
"A = np.transpose(X) @ X\n",
"# Brute force inversion\n",
"B = np.linalg.inv(A)\n",
"C = SVDinv(A)\n",
"print(np.abs(B-C))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Ridge and LASSO Regression\n",
"\n",
"Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n",
@@ -1222,14 +1291,14 @@
"for i in range(nlambdas):\n",
" lmb = lambdas[i]\n",
" Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y\n",
"# print(Ridgebeta)\n",
" print(Ridgebeta)\n",
" # and then make the prediction\n",
" ypredictRidge = X @ Ridgebeta\n",
" MSERidgePredict[i] = MSE(y,ypredictRidge)\n",
"# print(MSEPredict[i])\n",
" RegLasso = linear_model.Lasso(lmb)\n",
" RegLasso.fit(X,y)\n",
" ypredictLasso = RegLasso.predict(X)\n",
" print(RegLasso_coef_)\n",
" MSELassoPredict[i] = MSE(y,ypredictLasso)\n",
"# Now plot the results\n",
"plt.figure()\n",
@@ -1293,13 +1362,9 @@
"print(OLSbeta)\n",
"# and then make the prediction\n",
"ytildeOLS = X_train @ OLSbeta\n",
"print(\"Training R2 for OLS\")\n",
"print(R2(y_train,ytildeOLS))\n",
"print(\"Training MSE for OLS\")\n",
"print(MSE(y_train,ytildeOLS))\n",
"ypredictOLS = X_test @ OLSbeta\n",
"print(\"Test R2 for OLS\")\n",
"print(R2(y_test,ypredictOLS))\n",
"print(\"Test MSE OLS\")\n",
"print(MSE(y_test,ypredictOLS))\n",
"\n",
doc/src/week36/week36.do.txt
+48 -6
View File
@@ -144,6 +144,52 @@ of our data (the columns of $\bm{X}$, the quantity of interest for us are the no
values and the column vectors of $\bm{V}$.
!split
===== Code for SVD and Inversion of Matrices =====
How do we use the SVD to invert a matrix $\bm{X}^\bm{X}$ which is singular or near singular?
The simple answer is to use the linear algebra function for pseudoinvers, that is
!bc pycod
Ainv = np.linlag.pinv(A)
!ec
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
!bc pycod
import numpy as np
# SVD inversion
def SVDinv(A):
''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
SVD is numerically more stable than the inversion algorithms provided by
numpy and scipy.linalg at the cost of being slower.
'''
U, s, VT = np.linalg.svd(A)
print('test U')
print( (np.transpose(U) @ U - U @np.transpose(U)))
print('test VT')
print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
D = np.zeros((len(U),len(VT)))
D = np.diag(s)
UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
return np.matmul(V,np.matmul(invD,UT))
#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
X = np.array( [ [1,2],[2,3]])
print(X)
A = np.transpose(X) @ X
# Brute force inversion
B = np.linalg.inv(A)
C = SVDinv(A)
print(np.abs(B-C))
!ec
!split
===== Ridge and LASSO Regression =====
@@ -698,14 +744,14 @@ lambdas = np.logspace(-4, 4, nlambdas)
for i in range(nlambdas):
lmb = lambdas[i]
Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y
# print(Ridgebeta)
print(Ridgebeta)
# and then make the prediction
ypredictRidge = X @ Ridgebeta
MSERidgePredict[i] = MSE(y,ypredictRidge)
# print(MSEPredict[i])
RegLasso = linear_model.Lasso(lmb)
RegLasso.fit(X,y)
ypredictLasso = RegLasso.predict(X)
print(RegLasso_coef_)
MSELassoPredict[i] = MSE(y,ypredictLasso)
# Now plot the results
plt.figure()
@@ -758,13 +804,9 @@ OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
print(OLSbeta)
# and then make the prediction
ytildeOLS = X_train @ OLSbeta
print("Training R2 for OLS")
print(R2(y_train,ytildeOLS))
print("Training MSE for OLS")
print(MSE(y_train,ytildeOLS))
ypredictOLS = X_test @ OLSbeta
print("Test R2 for OLS")
print(R2(y_test,ypredictOLS))
print("Test MSE OLS")
print(MSE(y_test,ypredictOLS))