Files
FYS-STK4155/doc/pub/week35/html/._week35-bs049.html
T
Morten Hjorth-Jensen 0a52d67d28 addition
2025-08-26 07:02:47 +02:00

403 lines
24 KiB
HTML

<!--
HTML file automatically generated from DocOnce source
(https://github.com/doconce/doconce/)
doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression">
<title>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</title>
<!-- Bootstrap style: bootstrap -->
<!-- doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'),
('Reading recommendations:', 3, None, 'reading-recommendations'),
('Reminder from last week', 2, None, 'reminder-from-last-week'),
('The equations for ordinary least squares',
2,
None,
'the-equations-for-ordinary-least-squares'),
('The cost/loss function', 2, None, 'the-cost-loss-function'),
('Interpretations and optimizing our parameters',
2,
None,
'interpretations-and-optimizing-our-parameters'),
('Interpretations and optimizing our parameters',
2,
None,
'interpretations-and-optimizing-our-parameters'),
('Some useful matrix and vector expressions',
2,
None,
'some-useful-matrix-and-vector-expressions'),
('The Jacobian', 2, None, 'the-jacobian'),
('Derivatives, example 1', 2, None, 'derivatives-example-1'),
('Example 2', 2, None, 'example-2'),
('Example 3', 2, None, 'example-3'),
('Example 4', 2, None, 'example-4'),
('The mean squared error and its derivative',
2,
None,
'the-mean-squared-error-and-its-derivative'),
('Meet the Hessian Matrix', 2, None, 'meet-the-hessian-matrix'),
('Interpretations and optimizing our parameters',
2,
None,
'interpretations-and-optimizing-our-parameters'),
('Example relevant for the exercises',
2,
None,
'example-relevant-for-the-exercises'),
('Own code for Ordinary Least Squares',
2,
None,
'own-code-for-ordinary-least-squares'),
('Adding error analysis and training set up',
2,
None,
'adding-error-analysis-and-training-set-up'),
('Splitting our Data in Training and Test data',
2,
None,
'splitting-our-data-in-training-and-test-data'),
('The complete code with a simple data set',
2,
None,
'the-complete-code-with-a-simple-data-set'),
('Making your own test-train splitting',
2,
None,
'making-your-own-test-train-splitting'),
('Reducing the number of degrees of freedom, overarching view',
2,
None,
'reducing-the-number-of-degrees-of-freedom-overarching-view'),
('Preprocessing our data', 2, None, 'preprocessing-our-data'),
('Functionality in Scikit-Learn',
2,
None,
'functionality-in-scikit-learn'),
('More preprocessing', 2, None, 'more-preprocessing'),
('Frequently used scaling functions',
2,
None,
'frequently-used-scaling-functions'),
('Example of own Standard scaling',
2,
None,
'example-of-own-standard-scaling'),
('Min-Max Scaling', 2, None, 'min-max-scaling'),
('Testing the Means Squared Error as function of Complexity',
2,
None,
'testing-the-means-squared-error-as-function-of-complexity'),
('Mathematical Interpretation of Ordinary Least Squares',
2,
None,
'mathematical-interpretation-of-ordinary-least-squares'),
('Residual Error', 2, None, 'residual-error'),
('Simple case', 2, None, 'simple-case'),
('The singular value decomposition',
2,
None,
'the-singular-value-decomposition'),
('Linear Regression Problems',
2,
None,
'linear-regression-problems'),
('Fixing the singularity', 2, None, 'fixing-the-singularity'),
('Ridge and LASSO Regression',
2,
None,
'ridge-and-lasso-regression'),
('Deriving the Ridge Regression Equations',
2,
None,
'deriving-the-ridge-regression-equations'),
('Basic math of the SVD', 2, None, 'basic-math-of-the-svd'),
('The SVD, a Fantastic Algorithm',
2,
None,
'the-svd-a-fantastic-algorithm'),
('Economy-size SVD', 2, None, 'economy-size-svd'),
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
('Note about SVD Calculations',
2,
None,
'note-about-svd-calculations'),
('Mathematics of the SVD and implications',
2,
None,
'mathematics-of-the-svd-and-implications'),
('Example Matrix', 2, None, 'example-matrix'),
('Setting up the Matrix to be inverted',
2,
None,
'setting-up-the-matrix-to-be-inverted'),
('Further properties (important for our analyses later)',
2,
None,
'further-properties-important-for-our-analyses-later'),
('Meet the Covariance Matrix',
2,
None,
'meet-the-covariance-matrix'),
('Introducing the Covariance and Correlation functions',
2,
None,
'introducing-the-covariance-and-correlation-functions'),
('Covariance and Correlation Matrix',
2,
None,
'covariance-and-correlation-matrix'),
('Correlation Function and Design/Feature Matrix',
2,
None,
'correlation-function-and-design-feature-matrix'),
('Covariance Matrix Examples',
2,
None,
'covariance-matrix-examples'),
('Correlation Matrix', 2, None, 'correlation-matrix'),
('Correlation Matrix with Pandas',
2,
None,
'correlation-matrix-with-pandas'),
('Rewriting the Covariance and/or Correlation Matrix',
2,
None,
'rewriting-the-covariance-and-or-correlation-matrix'),
('Linking with the SVD', 2, None, 'linking-with-the-svd'),
('What does it mean?', 2, None, 'what-does-it-mean'),
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
2,
None,
'and-finally-boldsymbol-x-boldsymbol-x-t'),
('Back to Ridge and LASSO Regression',
2,
None,
'back-to-ridge-and-lasso-regression'),
('Interpreting the Ridge results',
2,
None,
'interpreting-the-ridge-results'),
('More interpretations', 2, None, 'more-interpretations'),
('Deriving the Lasso Regression Equations',
2,
None,
'deriving-the-lasso-regression-equations'),
('Material for exercises week 35',
2,
None,
'material-for-exercises-week-35'),
('Important technicalities: More on Rescaling data',
2,
None,
'important-technicalities-more-on-rescaling-data')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week35-bs.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week35-bs001.html#plans-for-week-35" style="font-size: 80%;"><b>Plans for week 35</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs001.html#reading-recommendations" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Reading recommendations:</a></li>
<!-- navigation toc: --> <li><a href="._week35-bs002.html#reminder-from-last-week" style="font-size: 80%;"><b>Reminder from last week</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs003.html#the-equations-for-ordinary-least-squares" style="font-size: 80%;"><b>The equations for ordinary least squares</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs004.html#the-cost-loss-function" style="font-size: 80%;"><b>The cost/loss function</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs015.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs015.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs007.html#some-useful-matrix-and-vector-expressions" style="font-size: 80%;"><b>Some useful matrix and vector expressions</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs008.html#the-jacobian" style="font-size: 80%;"><b>The Jacobian</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs009.html#derivatives-example-1" style="font-size: 80%;"><b>Derivatives, example 1</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs010.html#example-2" style="font-size: 80%;"><b>Example 2</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs011.html#example-3" style="font-size: 80%;"><b>Example 3</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs012.html#example-4" style="font-size: 80%;"><b>Example 4</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs013.html#the-mean-squared-error-and-its-derivative" style="font-size: 80%;"><b>The mean squared error and its derivative</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs014.html#meet-the-hessian-matrix" style="font-size: 80%;"><b>Meet the Hessian Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs015.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs016.html#example-relevant-for-the-exercises" style="font-size: 80%;"><b>Example relevant for the exercises</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs017.html#own-code-for-ordinary-least-squares" style="font-size: 80%;"><b>Own code for Ordinary Least Squares</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs018.html#adding-error-analysis-and-training-set-up" style="font-size: 80%;"><b>Adding error analysis and training set up</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs019.html#splitting-our-data-in-training-and-test-data" style="font-size: 80%;"><b>Splitting our Data in Training and Test data</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs020.html#the-complete-code-with-a-simple-data-set" style="font-size: 80%;"><b>The complete code with a simple data set</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs021.html#making-your-own-test-train-splitting" style="font-size: 80%;"><b>Making your own test-train splitting</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs022.html#reducing-the-number-of-degrees-of-freedom-overarching-view" style="font-size: 80%;"><b>Reducing the number of degrees of freedom, overarching view</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs023.html#preprocessing-our-data" style="font-size: 80%;"><b>Preprocessing our data</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs024.html#functionality-in-scikit-learn" style="font-size: 80%;"><b>Functionality in Scikit-Learn</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs025.html#more-preprocessing" style="font-size: 80%;"><b>More preprocessing</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs026.html#frequently-used-scaling-functions" style="font-size: 80%;"><b>Frequently used scaling functions</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs027.html#example-of-own-standard-scaling" style="font-size: 80%;"><b>Example of own Standard scaling</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs028.html#min-max-scaling" style="font-size: 80%;"><b>Min-Max Scaling</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs029.html#testing-the-means-squared-error-as-function-of-complexity" style="font-size: 80%;"><b>Testing the Means Squared Error as function of Complexity</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs030.html#mathematical-interpretation-of-ordinary-least-squares" style="font-size: 80%;"><b>Mathematical Interpretation of Ordinary Least Squares</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs031.html#residual-error" style="font-size: 80%;"><b>Residual Error</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs032.html#simple-case" style="font-size: 80%;"><b>Simple case</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs033.html#the-singular-value-decomposition" style="font-size: 80%;"><b>The singular value decomposition</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs034.html#linear-regression-problems" style="font-size: 80%;"><b>Linear Regression Problems</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs035.html#fixing-the-singularity" style="font-size: 80%;"><b>Fixing the singularity</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs036.html#ridge-and-lasso-regression" style="font-size: 80%;"><b>Ridge and LASSO Regression</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs037.html#deriving-the-ridge-regression-equations" style="font-size: 80%;"><b>Deriving the Ridge Regression Equations</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs038.html#basic-math-of-the-svd" style="font-size: 80%;"><b>Basic math of the SVD</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs039.html#the-svd-a-fantastic-algorithm" style="font-size: 80%;"><b>The SVD, a Fantastic Algorithm</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs040.html#economy-size-svd" style="font-size: 80%;"><b>Economy-size SVD</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs041.html#codes-for-the-svd" style="font-size: 80%;"><b>Codes for the SVD</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs042.html#note-about-svd-calculations" style="font-size: 80%;"><b>Note about SVD Calculations</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs043.html#mathematics-of-the-svd-and-implications" style="font-size: 80%;"><b>Mathematics of the SVD and implications</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs044.html#example-matrix" style="font-size: 80%;"><b>Example Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs045.html#setting-up-the-matrix-to-be-inverted" style="font-size: 80%;"><b>Setting up the Matrix to be inverted</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs046.html#further-properties-important-for-our-analyses-later" style="font-size: 80%;"><b>Further properties (important for our analyses later)</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs047.html#meet-the-covariance-matrix" style="font-size: 80%;"><b>Meet the Covariance Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs048.html#introducing-the-covariance-and-correlation-functions" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
<!-- navigation toc: --> <li><a href="#covariance-and-correlation-matrix" style="font-size: 80%;"><b>Covariance and Correlation Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs050.html#correlation-function-and-design-feature-matrix" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs051.html#covariance-matrix-examples" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs052.html#correlation-matrix" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs053.html#correlation-matrix-with-pandas" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs054.html#rewriting-the-covariance-and-or-correlation-matrix" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs055.html#linking-with-the-svd" style="font-size: 80%;"><b>Linking with the SVD</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs056.html#what-does-it-mean" style="font-size: 80%;"><b>What does it mean?</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs057.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;"><b>And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs058.html#back-to-ridge-and-lasso-regression" style="font-size: 80%;"><b>Back to Ridge and LASSO Regression</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs059.html#interpreting-the-ridge-results" style="font-size: 80%;"><b>Interpreting the Ridge results</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs060.html#more-interpretations" style="font-size: 80%;"><b>More interpretations</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs061.html#deriving-the-lasso-regression-equations" style="font-size: 80%;"><b>Deriving the Lasso Regression Equations</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs062.html#material-for-exercises-week-35" style="font-size: 80%;"><b>Material for exercises week 35</b></a></li>
<!-- navigation toc: --> <li><a href="._week35-bs062.html#important-technicalities-more-on-rescaling-data" style="font-size: 80%;"><b>Important technicalities: More on Rescaling data</b></a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0049"></a>
<!-- !split -->
<h2 id="covariance-and-correlation-matrix" class="anchor">Covariance and Correlation Matrix </h2>
<p>The covariance takes values between zero and infinity and may thus
lead to problems with loss of numerical precision for particularly
large values. It is common to scale the covariance matrix by
introducing instead the correlation matrix defined via the so-called
correlation function
</p>
$$
\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}.
$$
<p>The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]
\in [-1,1] \). This avoids eventual problems with too large values. We
can then define the correlation matrix for the two vectors \( \boldsymbol{x} \)
and \( \boldsymbol{y} \) as
</p>
$$
\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\
\mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\
\end{bmatrix},
$$
<p>In the above example this is the function we constructed using <b>pandas</b>.</p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week35-bs048.html">&laquo;</a></li>
<li><a href="._week35-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week35-bs041.html">42</a></li>
<li><a href="._week35-bs042.html">43</a></li>
<li><a href="._week35-bs043.html">44</a></li>
<li><a href="._week35-bs044.html">45</a></li>
<li><a href="._week35-bs045.html">46</a></li>
<li><a href="._week35-bs046.html">47</a></li>
<li><a href="._week35-bs047.html">48</a></li>
<li><a href="._week35-bs048.html">49</a></li>
<li class="active"><a href="._week35-bs049.html">50</a></li>
<li><a href="._week35-bs050.html">51</a></li>
<li><a href="._week35-bs051.html">52</a></li>
<li><a href="._week35-bs052.html">53</a></li>
<li><a href="._week35-bs053.html">54</a></li>
<li><a href="._week35-bs054.html">55</a></li>
<li><a href="._week35-bs055.html">56</a></li>
<li><a href="._week35-bs056.html">57</a></li>
<li><a href="._week35-bs057.html">58</a></li>
<li><a href="._week35-bs058.html">59</a></li>
<li><a href="">...</a></li>
<li><a href="._week35-bs062.html">63</a></li>
<li><a href="._week35-bs050.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>