Morten Hjorth-Jensen
1102 lines
83 KiB
HTML
1102 lines
83 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>3. Ridge and Lasso Regression — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link rel="stylesheet" href="_static/css/index.f658d18f9b420779cfdf24aa0a7e2d77.css">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/open-sans_all/1.44.1/index.css">
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/lato_latin-ext/1.44.1/index.css">
|
||
|
||
|
||
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
|
||
<link rel="stylesheet" href="_static/sphinx-book-theme.e7340bb3dbd8dde6db86f25597f54a1b.css" type="text/css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.d3f166471bb80abb5163.js">
|
||
|
||
<script id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/language_data.js"></script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script >var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.7d483ff0a819d6edff12ce0b1ead3928.js"></script>
|
||
<script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
|
||
<script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]], "processRefs": false, "processEnvironments": false}})</script>
|
||
<script async="async" src="https://unpkg.com/thebelab@latest/lib/index.js"></script>
|
||
<script >
|
||
const thebe_selector = ".thebe"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="4. Logistic Regression" href="chapter4.html" />
|
||
<link rel="prev" title="2. Resampling Methods" href="chapter2.html" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="en" />
|
||
|
||
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form>
|
||
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
|
||
<ul class="nav sidenav_l1">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p class="caption collapsible-parent">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav sidenav_l1">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p class="caption collapsible-parent">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="current nav sidenav_l1">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
1. Linear Regression, basic Elements
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
2. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
3. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
4. Logistic Regression
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/chapter3.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onClick="window.print()" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i>
|
||
Contents
|
||
</div>
|
||
<nav id="bd-toc-nav">
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-singular-value-decomposition">
|
||
3.1. The singular value decomposition
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-svd-a-fantastic-algorithm">
|
||
3.2. The SVD, a Fantastic Algorithm
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#economy-size-svd">
|
||
3.3. Economy-size SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id1">
|
||
3.4. Ridge and LASSO Regression
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-better-understanding-of-regularization">
|
||
3.5. A better understanding of regularization
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#introducing-the-covariance-and-correlation-functions">
|
||
3.6. Introducing the Covariance and Correlation functions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linking-with-svd">
|
||
3.7. Linking with SVD
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
|
||
<div>
|
||
|
||
<div class="section" id="ridge-and-lasso-regression">
|
||
<h1><span class="section-number">3. </span>Ridge and Lasso Regression<a class="headerlink" href="#ridge-and-lasso-regression" title="Permalink to this headline">¶</a></h1>
|
||
<p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember10.mp4?vrtx=view-as-webpage">Video of Lecture</a></p>
|
||
<div class="section" id="the-singular-value-decomposition">
|
||
<h2><span class="section-number">3.1. </span>The singular value decomposition<a class="headerlink" href="#the-singular-value-decomposition" title="Permalink to this headline">¶</a></h2>
|
||
<p>The examples we have looked at so far are cases where we normally can
|
||
invert the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Using a polynomial expansion as we
|
||
did both for the masses and the fitting of the equation of state,
|
||
leads to row vectors of the design matrix which are essentially
|
||
orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.</p>
|
||
<p>This may
|
||
however not the be case in general and a standard matrix inversion
|
||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.</p>
|
||
<p>There is however a way to partially circumvent this problem and also gain some insights about the ordinary least squares approach, and later shrinkage methods like Ridge and Lasso regressions.</p>
|
||
<p>This is given by the <strong>Singular Value Decomposition</strong> algorithm, perhaps
|
||
the most powerful linear algebra algorithm. Let us look at a
|
||
different example where we may have problems with the standard matrix
|
||
inversion algorithm. Thereafter we dive into the math of the SVD.</p>
|
||
<p>One of the typical problems we encounter with linear regression, in particular
|
||
when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> (our so-called design matrix) is high-dimensional,
|
||
are problems with near singular or singular matrices. The column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>
|
||
may be linearly dependent, normally referred to as super-collinearity.<br />
|
||
This means that the matrix may be rank deficient and it is basically impossible to
|
||
to model the data using linear regression. As an example, consider the matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\mathbf{X} & = \left[
|
||
\begin{array}{rrr}
|
||
1 & -1 & 2
|
||
\\
|
||
1 & 0 & 1
|
||
\\
|
||
1 & 2 & -1
|
||
\\
|
||
1 & 1 & 0
|
||
\end{array} \right]
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are linearly dependent. We see this easily since the
|
||
the first column is the row-wise sum of the other two columns. The rank (more correct,
|
||
the column rank) of a matrix is the dimension of the space spanned by the
|
||
column vectors. Hence, the rank of <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is equal to the number
|
||
of linearly independent columns. In this particular case the matrix has rank 2.</p>
|
||
<p>Super-collinearity of an <span class="math notranslate nohighlight">\((n \times p)\)</span>-dimensional design matrix <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> implies
|
||
that the inverse of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\boldsymbol{X} & = \left[
|
||
\begin{array}{rr}
|
||
1 & -1
|
||
\\
|
||
1 & -1
|
||
\end{array} \right].
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>We see easily that <span class="math notranslate nohighlight">\(\mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0\)</span>. Hence, <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is singular and its inverse is undefined.
|
||
This is equivalent to saying that the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has at least an eigenvalue which is zero.</p>
|
||
<p>If our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> which enters the linear regression problem</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\boldsymbol{\beta} = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>has linearly dependent column vectors, we will not be able to compute the inverse
|
||
of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> and we cannot find the parameters (estimators) <span class="math notranslate nohighlight">\(\beta_i\)</span>.
|
||
The estimators are only well-defined if <span class="math notranslate nohighlight">\((\boldsymbol{X}^{T}\boldsymbol{X})^{-1}\)</span> exits.
|
||
This is more likely to happen when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is high-dimensional. In this case it is likely to encounter a situation where
|
||
the regression parameters <span class="math notranslate nohighlight">\(\beta_i\)</span> cannot be estimated.</p>
|
||
<p>A cheap <em>ad hoc</em> approach is simply to add a small diagonal component to the matrix to invert, that is we change</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> is the identity matrix. When we discuss <strong>Ridge</strong> regression this is actually what we end up evaluating. The parameter <span class="math notranslate nohighlight">\(\lambda\)</span> is called a hyperparameter. More about this later.</p>
|
||
<p>From standard linear algebra we know that a square matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be diagonalized if and only it is
|
||
a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Normal_matrix">normal matrix</a>, that is if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times n}\)</span>
|
||
we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> or if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{C}}^{n\times n}\)</span> we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X}\)</span>.
|
||
The matrix has then a set of eigenpairs</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n),
|
||
\]</div>
|
||
<p>and the eigenvalues are given by the diagonal matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n).
|
||
\]</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in terms of an orthogonal/unitary transformation <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I}\)</span>.</p>
|
||
<p>Not all square matrices are diagonalizable. A matrix like the one discussed above</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \begin{bmatrix}
|
||
1& -1 \\
|
||
1& -1\\
|
||
\end{bmatrix}
|
||
\end{split}\]</div>
|
||
<p>is not diagonalizable, it is a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Defective_matrix">defective matrix</a>. It is easy to see that the condition
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> is not fulfilled.</p>
|
||
</div>
|
||
<div class="section" id="the-svd-a-fantastic-algorithm">
|
||
<h2><span class="section-number">3.2. </span>The SVD, a Fantastic Algorithm<a class="headerlink" href="#the-svd-a-fantastic-algorithm" title="Permalink to this headline">¶</a></h2>
|
||
<p>However, and this is the strength of the SVD algorithm, any general
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be decomposed in terms of a diagonal matrix and
|
||
two orthogonal/unitary matrices. The <a class="reference external" href="https://en.wikipedia.org/wiki/Singular_value_decomposition">Singular Value Decompostion
|
||
(SVD) theorem</a>
|
||
states that a general <span class="math notranslate nohighlight">\(m\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in
|
||
terms of a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> of dimensionality <span class="math notranslate nohighlight">\(m\times n\)</span>
|
||
and two orthognal matrices <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>, where the first has
|
||
dimensionality <span class="math notranslate nohighlight">\(m \times m\)</span> and the last dimensionality <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
We have then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T
|
||
\]</div>
|
||
<p>As an example, the above defective matrix can be decomposed as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\end{split}\]</div>
|
||
<p>with eigenvalues <span class="math notranslate nohighlight">\(\sigma_1=2\)</span> and <span class="math notranslate nohighlight">\(\sigma_2=0\)</span>.
|
||
The SVD exits always!</p>
|
||
<p>The SVD
|
||
decomposition (singular values) gives eigenvalues
|
||
<span class="math notranslate nohighlight">\(\sigma_i\geq\sigma_{i+1}\)</span> for all <span class="math notranslate nohighlight">\(i\)</span> and for dimensions larger than <span class="math notranslate nohighlight">\(i=p\)</span>, the
|
||
eigenvalues (singular values) are zero.</p>
|
||
<p>In the general case, where our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has dimension
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span>, the matrix is thus decomposed into an <span class="math notranslate nohighlight">\(n\times n\)</span>
|
||
orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, a <span class="math notranslate nohighlight">\(p\times p\)</span> orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>
|
||
and a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> with <span class="math notranslate nohighlight">\(r=\mathrm{min}(n,p)\)</span>
|
||
singular values <span class="math notranslate nohighlight">\(\sigma_i\geq 0\)</span> on the main diagonal and zeros filling
|
||
the rest of the matrix. There are at most <span class="math notranslate nohighlight">\(p\)</span> singular values
|
||
assuming that <span class="math notranslate nohighlight">\(n > p\)</span>. In our regression examples for the nuclear
|
||
masses and the equation of state this is indeed the case, while for
|
||
the Ising model we have <span class="math notranslate nohighlight">\(p > n\)</span>. These are often cases that lead to
|
||
near singular or singular matrices.</p>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are called the left singular vectors while the columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the right singular vectors.</p>
|
||
</div>
|
||
<div class="section" id="economy-size-svd">
|
||
<h2><span class="section-number">3.3. </span>Economy-size SVD<a class="headerlink" href="#economy-size-svd" title="Permalink to this headline">¶</a></h2>
|
||
<p>If we assume that <span class="math notranslate nohighlight">\(n > p\)</span>, then our matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> has dimension <span class="math notranslate nohighlight">\(n
|
||
\times n\)</span>. The last <span class="math notranslate nohighlight">\(n-p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> become however
|
||
irrelevant in our calculations since they are multiplied with the
|
||
zeros in <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>.</p>
|
||
<p>The economy-size decomposition removes extra rows or columns of zeros
|
||
from the diagonal matrix of singular values, <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>, along with the columns
|
||
in either <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> that multiply those zeros in the expression.
|
||
Removing these zeros and columns can improve execution time
|
||
and reduce storage requirements without compromising the accuracy of
|
||
the decomposition.</p>
|
||
<p>If <span class="math notranslate nohighlight">\(n > p\)</span>, we keep only the first <span class="math notranslate nohighlight">\(p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(p\times p\)</span>.
|
||
If <span class="math notranslate nohighlight">\(p > n\)</span>, then only the first <span class="math notranslate nohighlight">\(n\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are computed and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
The <span class="math notranslate nohighlight">\(n=p\)</span> case is obvious, we retain the full SVD.
|
||
In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="c1"># SVD inversion</span>
|
||
<span class="k">def</span> <span class="nf">SVDinv</span><span class="p">(</span><span class="n">A</span><span class="p">):</span>
|
||
<span class="sd">''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
|
||
<span class="sd"> SVD is numerically more stable than the inversion algorithms provided by</span>
|
||
<span class="sd"> numpy and scipy.linalg at the cost of being slower.</span>
|
||
<span class="sd"> '''</span>
|
||
<span class="n">U</span><span class="p">,</span> <span class="n">s</span><span class="p">,</span> <span class="n">VT</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">svd</span><span class="p">(</span><span class="n">A</span><span class="p">)</span>
|
||
<span class="c1"># print('test U')</span>
|
||
<span class="c1"># print( (np.transpose(U) @ U - U @np.transpose(U)))</span>
|
||
<span class="c1"># print('test VT')</span>
|
||
<span class="c1"># print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">U</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">VT</span><span class="p">)</span>
|
||
|
||
<span class="n">D</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">U</span><span class="p">),</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)))</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)):</span>
|
||
<span class="n">D</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="n">i</span><span class="p">]</span><span class="o">=</span><span class="n">s</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="n">UT</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">U</span><span class="p">);</span> <span class="n">V</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">VT</span><span class="p">);</span> <span class="n">invD</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">D</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">V</span><span class="p">,</span><span class="n">np</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">invD</span><span class="p">,</span><span class="n">UT</span><span class="p">))</span>
|
||
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">]</span> <span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">A</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="n">X</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">A</span><span class="p">)</span>
|
||
<span class="c1"># Brute force inversion of super-collinear matrix</span>
|
||
<span class="c1">#B = np.linalg.inv(A)</span>
|
||
<span class="c1">#print(B)</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">SVDinv</span><span class="p">(</span><span class="n">A</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[ 1. -1. 2.]
|
||
[ 1. 0. 1.]
|
||
[ 1. 2. -1.]
|
||
[ 1. 1. 0.]]
|
||
[[ 4. 2. 2.]
|
||
[ 2. 6. -4.]
|
||
[ 2. -4. 6.]]
|
||
[[-1.18404906e-16 8.16496581e-01 -5.77350269e-01]
|
||
[-7.07106781e-01 4.08248290e-01 5.77350269e-01]
|
||
[ 7.07106781e-01 4.08248290e-01 5.77350269e-01]]
|
||
[1.00000000e+01 6.00000000e+00 9.10898112e-32]
|
||
[[ 3.33066907e-17 -7.07106781e-01 7.07106781e-01]
|
||
[ 8.16496581e-01 4.08248290e-01 4.08248290e-01]
|
||
[ 5.77350269e-01 -5.77350269e-01 -5.77350269e-01]]
|
||
[[-3.65939208e+30 3.65939208e+30 3.65939208e+30]
|
||
[ 3.65939208e+30 -3.65939208e+30 -3.65939208e+30]
|
||
[ 3.65939208e+30 -3.65939208e+30 -3.65939208e+30]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has columns that are linearly dependent. The first
|
||
column is the row-wise sum of the other two columns. The rank of a
|
||
matrix (the column rank) is the dimension of space spanned by the
|
||
column vectors. The rank of the matrix is the number of linearly
|
||
independent columns, in this case just <span class="math notranslate nohighlight">\(2\)</span>. We see this from the
|
||
singular values when running the above code. Running the standard
|
||
inversion algorithm for matrix inversion with <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> results
|
||
in the program terminating due to a singular matrix.</p>
|
||
<p>There are several interesting mathematical properties which will be
|
||
relevant when we are going to discuss the differences between say
|
||
ordinary least squares (OLS) and <strong>Ridge</strong> regression.</p>
|
||
<p>We have from OLS that the parameters of the linear approximation are given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The matrix to invert can be rewritten in terms of our SVD decomposition as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>Using the orthogonality properties of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T = \boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{D}\)</span> being a diagonal matrix with values along the diagonal given by the singular values squared.</p>
|
||
<p>This means that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\boldsymbol{X}^T\boldsymbol{X})\boldsymbol{V} = \boldsymbol{V}\boldsymbol{D},
|
||
\]</div>
|
||
<p>that is the eigenvectors of <span class="math notranslate nohighlight">\((\boldsymbol{X}^T\boldsymbol{X})\)</span> are given by the columns of the right singular matrix of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and the eigenvalues are the squared singular values. It is easy to show (show this) that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D},
|
||
\]</div>
|
||
<p>that is, the eigenvectors of <span class="math notranslate nohighlight">\((\boldsymbol{X}\boldsymbol{X})^T\)</span> are the columns of the left singular matrix and the eigenvalues are the same.</p>
|
||
<p>Going back to our OLS equation we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We will come back to this expression when we discuss Ridge regression.</p>
|
||
<p>$<span class="math notranslate nohighlight">\( \tilde{y}^{OLS}=\boldsymbol{X}\hat{\beta}^{OLS}=\sum_{j=1}^p \boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}\)</span>$ and for Ridge we have</p>
|
||
<p>$<span class="math notranslate nohighlight">\( \tilde{y}^{Ridge}=\boldsymbol{X}\hat{\beta}^{Ridge}=\sum_{j=1}^p \boldsymbol{u}_j\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{u}_j^T\boldsymbol{y}\)</span>$ .</p>
|
||
<p>It is indeed the economy-sized SVD, note the summation runs up tp $<span class="math notranslate nohighlight">\(p\)</span><span class="math notranslate nohighlight">\( only and not \)</span><span class="math notranslate nohighlight">\(n\)</span>$.</p>
|
||
<p>Here we have that $<span class="math notranslate nohighlight">\(\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\)</span><span class="math notranslate nohighlight">\(, with \)</span><span class="math notranslate nohighlight">\(\Sigma\)</span><span class="math notranslate nohighlight">\( being an \)</span><span class="math notranslate nohighlight">\( n\times p\)</span><span class="math notranslate nohighlight">\( matrix and \)</span><span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span><span class="math notranslate nohighlight">\( being a \)</span><span class="math notranslate nohighlight">\( p\times p\)</span><span class="math notranslate nohighlight">\( matrix. We also have assumed here that \)</span><span class="math notranslate nohighlight">\( n > p\)</span>$.</p>
|
||
</div>
|
||
<div class="section" id="id1">
|
||
<h2><span class="section-number">3.4. </span>Ridge and LASSO Regression<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h2>
|
||
<p><a class="reference external" href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage">Video of Lecture</a></p>
|
||
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
||
our optimization problem is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>or we can state it as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
||
\]</div>
|
||
<p>where we have used the definition of a norm-2 vector, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
||
\]</div>
|
||
<p>By minimizing the above equation with respect to the parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we could then obtain an analytical expression for the
|
||
parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. We can add a regularization parameter <span class="math notranslate nohighlight">\(\lambda\)</span> by
|
||
defining a new cost function to be optimized, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
||
\]</div>
|
||
<p>which leads to the Ridge regression minimization problem where we
|
||
require that <span class="math notranslate nohighlight">\(\vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t\)</span>, where <span class="math notranslate nohighlight">\(t\)</span> is
|
||
a finite number larger than zero. By defining</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
|
||
\]</div>
|
||
<p>we have a new optimization equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
|
||
\]</div>
|
||
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.</p>
|
||
<p>Here we have defined the norm-1 as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
||
\]</div>
|
||
<p>Using the matrix-vector expression for Ridge regression,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>by taking the derivatives with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we obtain then
|
||
a slightly modified matrix inversion problem which for finite values
|
||
of <span class="math notranslate nohighlight">\(\lambda\)</span> does not suffer from singularity problems. We obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> being a <span class="math notranslate nohighlight">\(p\times p\)</span> identity matrix with the constraint that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sum_{i=0}^{p-1} \beta_i^2 \leq t,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(t\)</span> a finite positive number.</p>
|
||
<p>We see that Ridge regression is nothing but the standard
|
||
OLS with a modified diagonal term added to <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. The
|
||
consequences, in particular for our discussion of the bias-variance tradeoff
|
||
are rather interesting.</p>
|
||
<p>Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}.
|
||
\]</div>
|
||
<p>We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}
|
||
\]</div>
|
||
<p>For Ridge regression this becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{\beta}^{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y},
|
||
\]</div>
|
||
<p>with the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_j\)</span> being the columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>.</p>
|
||
<p>Since <span class="math notranslate nohighlight">\(\lambda \geq 0\)</span>, it means that compared to OLS, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1.
|
||
\]</div>
|
||
<p>Ridge regression finds the coordinates of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> with respect to the
|
||
orthonormal basis <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, it then shrinks the coordinates by
|
||
<span class="math notranslate nohighlight">\(\frac{\sigma_j^2}{\sigma_j^2+\lambda}\)</span>. Recall that the SVD has
|
||
eigenvalues ordered in a descending way, that is <span class="math notranslate nohighlight">\(\sigma_i \geq
|
||
\sigma_{i+1}\)</span>.</p>
|
||
<p>For small eigenvalues <span class="math notranslate nohighlight">\(\sigma_i\)</span> it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom.
|
||
Actually, calculating the variance of <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{v}_j\)</span> shows that this quantity is equal to <span class="math notranslate nohighlight">\(\sigma_j^2/n\)</span>.
|
||
With a parameter <span class="math notranslate nohighlight">\(\lambda\)</span> we can thus shrink the role of specific parameters.</p>
|
||
<p>For the sake of simplicity, let us assume that the design matrix is orthonormal, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}.
|
||
\]</div>
|
||
<p>In this case the standard OLS results in</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}},
|
||
\]</div>
|
||
<p>that is the Ridge estimator scales the OLS estimator by the inverse of a factor <span class="math notranslate nohighlight">\(1+\lambda\)</span>, and
|
||
the Ridge estimator converges to zero when the hyperparameter goes to
|
||
infinity.</p>
|
||
<p>We will come back to more interpreations after we have gone through some of the statistical analysis part.</p>
|
||
<p>For more discussions of Ridge and Lasso regression, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.
|
||
Similarly, <a class="reference external" href="https://arxiv.org/abs/1803.08823">Mehta et al’s article</a> is also recommended.</p>
|
||
</div>
|
||
<div class="section" id="a-better-understanding-of-regularization">
|
||
<h2><span class="section-number">3.5. </span>A better understanding of regularization<a class="headerlink" href="#a-better-understanding-of-regularization" title="Permalink to this headline">¶</a></h2>
|
||
<p>The parameter <span class="math notranslate nohighlight">\(\lambda\)</span> that we have introduced in the Ridge (and
|
||
Lasso as well) regression is often called a regularization parameter
|
||
or shrinkage parameter. It is common to call it a hyperparameter. What does it mean mathemtically?</p>
|
||
<p>Here we will first look at how to analyze the difference between the
|
||
standard OLS equations and the Ridge expressions in terms of a linear
|
||
algebra analysis using the SVD algorithm. Thereafter, we will link
|
||
(see the material on the bias-variance tradeoff below) these
|
||
observation to the statisical analysis of the results. In particular
|
||
we consider how the variance of the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> is
|
||
affected by changing the parameter <span class="math notranslate nohighlight">\(\lambda\)</span>.</p>
|
||
<p>We have our design matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>. With the SVD we decompose it as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U\Sigma V^T},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{U}\in {\mathbb{R}}^{n\times n}\)</span>, <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\in {\mathbb{R}}^{n\times p}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\boldsymbol{V}\in {\mathbb{R}}^{p\times p}\)</span>.</p>
|
||
<p>The matrices <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are unitary/orthonormal matrices, that is in case the matrices are real we have <span class="math notranslate nohighlight">\(\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{I}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="introducing-the-covariance-and-correlation-functions">
|
||
<h2><span class="section-number">3.6. </span>Introducing the Covariance and Correlation functions<a class="headerlink" href="#introducing-the-covariance-and-correlation-functions" title="Permalink to this headline">¶</a></h2>
|
||
<p>Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about
|
||
the definition of the covariance and the correlation function. These are quantities</p>
|
||
<p>Suppose we have defined two vectors
|
||
<span class="math notranslate nohighlight">\(\hat{x}\)</span> and <span class="math notranslate nohighlight">\(\hat{y}\)</span> with <span class="math notranslate nohighlight">\(n\)</span> elements each. The covariance matrix <span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where for example</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}).
|
||
\]</div>
|
||
<p>With this definition and recalling that the variance is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2,
|
||
\]</div>
|
||
<p>we can rewrite the covariance matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>The covariance takes values between zero and infinity and may thus
|
||
lead to problems with loss of numerical precision for particularly
|
||
large values. It is common to scale the covariance matrix by
|
||
introducing instead the correlation matrix defined via the so-called
|
||
correlation function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}.
|
||
\]</div>
|
||
<p>The correlation function is then given by values <span class="math notranslate nohighlight">\(\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]
|
||
\in [-1,1]\)</span>. This avoids eventual problems with too large values. We
|
||
can then define the correlation matrix for the two vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>In the above example this is the function we constructed using <strong>pandas</strong>.</p>
|
||
<p>In our derivation of the various regression algorithms like <strong>Ordinary Least Squares</strong> or <strong>Ridge regression</strong>
|
||
we defined the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\
|
||
x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\
|
||
x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\
|
||
\dots & \dots & \dots & \dots \dots & \dots \\
|
||
x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\
|
||
x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>, with the predictors/features <span class="math notranslate nohighlight">\(p\)</span> refering to the column numbers and the
|
||
entries <span class="math notranslate nohighlight">\(n\)</span> being the row elements.
|
||
We can rewrite the design/feature matrix in terms of its column vectors as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix},
|
||
\]</div>
|
||
<p>with a given vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}.
|
||
\]</div>
|
||
<p>With these definitions, we can now rewrite our <span class="math notranslate nohighlight">\(2\times 2\)</span>
|
||
correaltion/covariance matrix in terms of a moe general design/feature
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>. This leads to a <span class="math notranslate nohighlight">\(p\times p\)</span>
|
||
covariance matrix for the vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}_i\)</span> with <span class="math notranslate nohighlight">\(i=0,1,\dots,p-1\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix}
|
||
\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>and the correlation matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix}
|
||
1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>The Numpy function <strong>np.cov</strong> calculates the covariance elements using
|
||
the factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span> instead of <span class="math notranslate nohighlight">\(1/n\)</span> since it assumes we do not have
|
||
the exact mean values. The following simple function uses the
|
||
<strong>np.vstack</strong> function which takes each vector of dimension <span class="math notranslate nohighlight">\(1\times n\)</span>
|
||
and produces a <span class="math notranslate nohighlight">\(2\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{W}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{W} = \begin{bmatrix} x_0 & y_0 \\
|
||
x_1 & y_1 \\
|
||
x_2 & y_2\\
|
||
\dots & \dots \\
|
||
x_{n-2} & y_{n-2}\\
|
||
x_{n-1} & y_{n-1} &
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which in turn is converted into into the <span class="math notranslate nohighlight">\(2\times 2\)</span> covariance matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> via the Numpy function <strong>np.cov()</strong>. We note that we can also calculate
|
||
the mean value of each set of samples <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> etc using the Numpy
|
||
function <strong>np.mean(x)</strong>. We can also extract the eigenvalues of the
|
||
covariance matrix through the <strong>np.linalg.eig()</strong> function.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Importing various packages</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">))</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">))</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">W</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>-0.03494744740413562
|
||
3.896222705525891
|
||
[[ 1.15493691 3.39431833]
|
||
[ 3.39431833 10.95648659]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The previous example can be converted into the correlation matrix by
|
||
simply scaling the matrix elements with the variances. We should also
|
||
subtract the mean values for each column. This leads to the following
|
||
code which sets up the correlations matrix for the previous example in
|
||
a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the <span class="math notranslate nohighlight">\(2\times 2\)</span> correlation matrix (since we have only two vectors).</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="c1"># define two vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="c1">#scaling the x and y vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="n">variance_x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">variance_y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_y</span><span class="p">)</span>
|
||
<span class="n">cov_xy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_xx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_yy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xx</span><span class="o">/</span><span class="n">variance_x</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_yy</span><span class="o">/</span><span class="n">variance_y</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xy</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">variance_y</span><span class="o">*</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.0684659365902349
|
||
1.3864659735909566
|
||
[[1. 0.56083552]
|
||
[0.56083552 1. ]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We see that the matrix elements along the diagonal are one as they
|
||
should be and that the matrix is symmetric. Furthermore, diagonalizing
|
||
this matrix we easily see that it is a positive definite matrix.</p>
|
||
<p>The above procedure with <strong>numpy</strong> can be made more compact if we use <strong>pandas</strong>.</p>
|
||
<p>We whow here how we can set up the correlation matrix using <strong>pandas</strong>, as done in this simple code</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)))</span><span class="o">.</span><span class="n">T</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">Xpd</span><span class="p">)</span>
|
||
<span class="n">correlation_matrix</span> <span class="o">=</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">correlation_matrix</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[-1.0123897 -2.8039812 ]
|
||
[-0.28718135 -0.92440839]
|
||
[-0.74988099 -2.62660334]
|
||
[-0.10484789 -0.22717222]
|
||
[-0.62761155 -1.59313927]
|
||
[ 0.99624087 3.35932784]
|
||
[-0.26202974 -1.03435975]
|
||
[-0.36630178 -0.10312915]
|
||
[-0.4877836 -1.72320998]
|
||
[ 2.90178573 7.67667546]]
|
||
0 1
|
||
0 -1.012390 -2.803981
|
||
1 -0.287181 -0.924408
|
||
2 -0.749881 -2.626603
|
||
3 -0.104848 -0.227172
|
||
4 -0.627612 -1.593139
|
||
5 0.996241 3.359328
|
||
6 -0.262030 -1.034360
|
||
7 -0.366302 -0.103129
|
||
8 -0.487784 -1.723210
|
||
9 2.901786 7.676675
|
||
0 1
|
||
0 1.000000 0.989696
|
||
1 0.989696 1.000000
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We expand this model to the Franke function discussed above.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Common imports</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">):</span>
|
||
<span class="n">term1</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mf">0.25</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term2</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mf">49.0</span> <span class="o">-</span> <span class="mf">0.1</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">term3</span> <span class="o">=</span> <span class="mf">0.5</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="o">/</span><span class="mf">4.0</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">3</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term4</span> <span class="o">=</span> <span class="o">-</span><span class="mf">0.2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">4</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span> <span class="o">-</span> <span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">term1</span> <span class="o">+</span> <span class="n">term2</span> <span class="o">+</span> <span class="n">term3</span> <span class="o">+</span> <span class="n">term4</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span> <span class="p">):</span>
|
||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="n">N</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">l</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">n</span><span class="o">+</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span> <span class="c1"># Number of elements in beta</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">N</span><span class="p">,</span><span class="n">l</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">q</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">i</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">q</span><span class="o">+</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="p">(</span><span class="n">i</span><span class="o">-</span><span class="n">k</span><span class="p">))</span><span class="o">*</span><span class="p">(</span><span class="n">y</span><span class="o">**</span><span class="n">k</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">X</span>
|
||
|
||
|
||
<span class="c1"># Making meshgrid of datapoints and compute Franke's function</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">4</span>
|
||
<span class="n">N</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="c1"># subtract the mean values and set up the covariance matrix</span>
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">Xpd</span> <span class="o">-</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||
<span class="n">covariance_matrix</span> <span class="o">=</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">cov</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">covariance_matrix</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 0 1 2 3 4 5 6 7 \
|
||
0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
|
||
1 0.0 0.075348 0.082602 0.073564 0.078039 0.082574 0.065469 0.068545
|
||
2 0.0 0.082602 0.093536 0.081952 0.088312 0.094670 0.072999 0.077152
|
||
3 0.0 0.073564 0.081952 0.077804 0.082702 0.087489 0.072629 0.075932
|
||
4 0.0 0.078039 0.088312 0.082702 0.088620 0.094412 0.076979 0.080887
|
||
5 0.0 0.082574 0.094670 0.087489 0.094412 0.101207 0.081135 0.085647
|
||
6 0.0 0.065469 0.072999 0.072629 0.076979 0.081135 0.069875 0.072824
|
||
7 0.0 0.068545 0.077152 0.075932 0.080887 0.085647 0.072824 0.076146
|
||
8 0.0 0.071758 0.081453 0.079293 0.084869 0.090255 0.075775 0.079482
|
||
9 0.0 0.075149 0.085972 0.082762 0.088986 0.095035 0.078769 0.082882
|
||
10 0.0 0.057678 0.063987 0.065995 0.069640 0.073058 0.064814 0.067321
|
||
11 0.0 0.059991 0.066970 0.068481 0.072514 0.076323 0.067071 0.069824
|
||
12 0.0 0.062435 0.070113 0.071062 0.075503 0.079726 0.069384 0.072399
|
||
13 0.0 0.065032 0.073451 0.073761 0.078639 0.083307 0.071774 0.075069
|
||
14 0.0 0.067805 0.077022 0.076602 0.081951 0.087104 0.074261 0.077858
|
||
|
||
8 9 10 11 12 13 14
|
||
0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
|
||
1 0.071758 0.075149 0.057678 0.059991 0.062435 0.065032 0.067805
|
||
2 0.081453 0.085972 0.063987 0.066970 0.070113 0.073451 0.077022
|
||
3 0.079293 0.082762 0.065995 0.068481 0.071062 0.073761 0.076602
|
||
4 0.084869 0.088986 0.069640 0.072514 0.075503 0.078639 0.081951
|
||
5 0.090255 0.095035 0.073058 0.076323 0.079726 0.083307 0.087104
|
||
6 0.075775 0.078769 0.064814 0.067071 0.069384 0.071774 0.074261
|
||
7 0.079482 0.082882 0.067321 0.069824 0.072399 0.075069 0.077858
|
||
8 0.083219 0.087047 0.069793 0.072554 0.075401 0.078366 0.081476
|
||
9 0.087047 0.091332 0.072268 0.075299 0.078437 0.081717 0.085171
|
||
10 0.069793 0.072268 0.061016 0.062978 0.064967 0.067002 0.069095
|
||
11 0.072554 0.075299 0.062978 0.065108 0.067277 0.069504 0.071804
|
||
12 0.075401 0.078437 0.064967 0.067277 0.069637 0.072069 0.074593
|
||
13 0.078366 0.081717 0.067002 0.069504 0.072069 0.074724 0.077490
|
||
14 0.081476 0.085171 0.069095 0.071804 0.074593 0.077490 0.080521
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We note here that the covariance is zero for the first rows and
|
||
columns since all matrix elements in the design matrix were set to one
|
||
(we are fitting the function in terms of a polynomial of degree <span class="math notranslate nohighlight">\(n\)</span>).</p>
|
||
<p>This means that the variance for these elements will be zero and will
|
||
cause problems when we set up the correlation matrix. We can simply
|
||
drop these elements and construct a correlation
|
||
matrix without these elements.</p>
|
||
<p>We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{C}[\boldsymbol{x}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}= \mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}].
|
||
\]</div>
|
||
<p>To see this let us simply look at a design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{2\times 2}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{00} & x_{01}\\
|
||
x_{10} & x_{11}\\
|
||
\end{bmatrix}=\begin{bmatrix}
|
||
\boldsymbol{x}_{0} & \boldsymbol{x}_{1}\\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>If we then compute the expectation value</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}=\begin{bmatrix}
|
||
x_{00}^2+x_{01}^2 & x_{00}x_{10}+x_{01}x_{11}\\
|
||
x_{10}x_{00}+x_{11}x_{01} & x_{10}^2+x_{11}^2\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is just</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]=\begin{bmatrix} \mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] \\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where we wrote $<span class="math notranslate nohighlight">\(\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]\)</span><span class="math notranslate nohighlight">\( to indicate that this the covariance of the vectors \)</span>\boldsymbol{x}<span class="math notranslate nohighlight">\( of the design/feature matrix \)</span>\boldsymbol{X}$.</p>
|
||
<p>It is easy to generalize this to a matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="linking-with-svd">
|
||
<h2><span class="section-number">3.7. </span>Linking with SVD<a class="headerlink" href="#linking-with-svd" title="Permalink to this headline">¶</a></h2>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<div class='prev-next-bottom'>
|
||
|
||
<a class='left-prev' id="prev-link" href="chapter2.html" title="previous page"><span class="section-number">2. </span>Resampling Methods</a>
|
||
<a class='right-next' id="next-link" href="chapter4.html" title="next page"><span class="section-number">4. </span>Logistic Regression</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer mt-5 mt-md-0">
|
||
<div class="container">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2020.<br/>
|
||
</p>
|
||
</div>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<script src="_static/js/index.d3f166471bb80abb5163.js"></script>
|
||
|
||
|
||
|
||
</body>
|
||
</html> |