Morten Hjorth-Jensen
383 lines
15 KiB
HTML
383 lines
15 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title></title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'),
|
|
('Reading recommendations:', 3, None, 'reading-recommendations'),
|
|
('Thursday September 1', 2, None, 'thursday-september-1'),
|
|
('Why Linear Regression (aka Ordinary Least Squares and family), '
|
|
'repeat from last week',
|
|
2,
|
|
None,
|
|
'why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'),
|
|
('Regression analysis, overarching aims',
|
|
2,
|
|
None,
|
|
'regression-analysis-overarching-aims'),
|
|
('Regression analysis, overarching aims II',
|
|
2,
|
|
None,
|
|
'regression-analysis-overarching-aims-ii'),
|
|
('Examples', 2, None, 'examples'),
|
|
('General linear models', 2, None, 'general-linear-models'),
|
|
('Rewriting the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'rewriting-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Rewriting the fitting procedure as a linear algebra problem, '
|
|
'more details',
|
|
2,
|
|
None,
|
|
'rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'generalizing-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'generalizing-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Optimizing our parameters',
|
|
2,
|
|
None,
|
|
'optimizing-our-parameters'),
|
|
('Our model for the nuclear binding energies',
|
|
2,
|
|
None,
|
|
'our-model-for-the-nuclear-binding-energies'),
|
|
('Optimizing our parameters, more details',
|
|
2,
|
|
None,
|
|
'optimizing-our-parameters-more-details'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Some useful matrix and vector expressions',
|
|
2,
|
|
None,
|
|
'some-useful-matrix-and-vector-expressions'),
|
|
('Meet the Hessian Matrix', 2, None, 'meet-the-hessian-matrix'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Own code for Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'own-code-for-ordinary-least-squares'),
|
|
('Adding error analysis and training set up',
|
|
2,
|
|
None,
|
|
'adding-error-analysis-and-training-set-up'),
|
|
('Splitting our Data in Training and Test data',
|
|
2,
|
|
None,
|
|
'splitting-our-data-in-training-and-test-data'),
|
|
('Examples', 2, None, 'examples'),
|
|
('Making your own test-train splitting',
|
|
2,
|
|
None,
|
|
'making-your-own-test-train-splitting'),
|
|
('The Boston housing data example',
|
|
2,
|
|
None,
|
|
'the-boston-housing-data-example'),
|
|
('Housing data, the code', 2, None, 'housing-data-the-code'),
|
|
('Reducing the number of degrees of freedom, overarching view',
|
|
2,
|
|
None,
|
|
'reducing-the-number-of-degrees-of-freedom-overarching-view'),
|
|
('Preprocessing our data', 2, None, 'preprocessing-our-data'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions'),
|
|
('Example of own Standard scaling',
|
|
2,
|
|
None,
|
|
'example-of-own-standard-scaling'),
|
|
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
|
('Testing the Means Squared Error as function of Complexity',
|
|
2,
|
|
None,
|
|
'testing-the-means-squared-error-as-function-of-complexity'),
|
|
('More preprocessing examples, Franke function and regression',
|
|
2,
|
|
None,
|
|
'more-preprocessing-examples-franke-function-and-regression'),
|
|
('Mathematical Interpretation of Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'mathematical-interpretation-of-ordinary-least-squares'),
|
|
('Residual Error', 2, None, 'residual-error'),
|
|
('Simple case', 2, None, 'simple-case'),
|
|
('The singular value decomposition',
|
|
2,
|
|
None,
|
|
'the-singular-value-decomposition'),
|
|
('Linear Regression Problems',
|
|
2,
|
|
None,
|
|
'linear-regression-problems'),
|
|
('Fixing the singularity', 2, None, 'fixing-the-singularity'),
|
|
('Basic math of the SVD', 2, None, 'basic-math-of-the-svd'),
|
|
('The SVD, a Fantastic Algorithm',
|
|
2,
|
|
None,
|
|
'the-svd-a-fantastic-algorithm'),
|
|
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
|
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
|
('Note about SVD Calculations',
|
|
2,
|
|
None,
|
|
'note-about-svd-calculations'),
|
|
('Friday September 2', 2, None, 'friday-september-2'),
|
|
('Mathematics of the SVD and implications',
|
|
2,
|
|
None,
|
|
'mathematics-of-the-svd-and-implications'),
|
|
('Example Matrix', 2, None, 'example-matrix'),
|
|
('Setting up the Matrix to be inverted',
|
|
2,
|
|
None,
|
|
'setting-up-the-matrix-to-be-inverted'),
|
|
('Further properties (important for our analyses later)',
|
|
2,
|
|
None,
|
|
'further-properties-important-for-our-analyses-later'),
|
|
('Meet the Covariance Matrix',
|
|
2,
|
|
None,
|
|
'meet-the-covariance-matrix'),
|
|
('Introducing the Covariance and Correlation functions',
|
|
2,
|
|
None,
|
|
'introducing-the-covariance-and-correlation-functions'),
|
|
('Covariance and Correlation Matrix',
|
|
2,
|
|
None,
|
|
'covariance-and-correlation-matrix'),
|
|
('Correlation Function and Design/Feature Matrix',
|
|
2,
|
|
None,
|
|
'correlation-function-and-design-feature-matrix'),
|
|
('Covariance Matrix Examples',
|
|
2,
|
|
None,
|
|
'covariance-matrix-examples'),
|
|
('Correlation Matrix', 2, None, 'correlation-matrix'),
|
|
('Correlation Matrix with Pandas',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas'),
|
|
('Correlation Matrix with Pandas and the Franke function',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas-and-the-franke-function'),
|
|
('Rewriting the Covariance and/or Correlation Matrix',
|
|
2,
|
|
None,
|
|
'rewriting-the-covariance-and-or-correlation-matrix'),
|
|
('Linking with the SVD', 2, None, 'linking-with-the-svd'),
|
|
('What does it mean?', 2, None, 'what-does-it-mean'),
|
|
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
|
|
2,
|
|
None,
|
|
'and-finally-boldsymbol-x-boldsymbol-x-t'),
|
|
('Ridge and LASSO Regression',
|
|
2,
|
|
None,
|
|
'ridge-and-lasso-regression'),
|
|
('Deriving the Ridge Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-ridge-regression-equations'),
|
|
('Interpreting the Ridge results',
|
|
2,
|
|
None,
|
|
'interpreting-the-ridge-results'),
|
|
('More interpretations', 2, None, 'more-interpretations'),
|
|
('Deriving the Lasso Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-lasso-regression-equations'),
|
|
('Exercises for week 35', 2, None, 'exercises-for-week-35'),
|
|
('Exercise 1: Setting up various Python environments',
|
|
2,
|
|
None,
|
|
'exercise-1-setting-up-various-python-environments'),
|
|
('Exercise 2: making your own data and exploring scikit-learn',
|
|
2,
|
|
None,
|
|
'exercise-2-making-your-own-data-and-exploring-scikit-learn'),
|
|
('Exercise 3: Normalizing our data',
|
|
2,
|
|
None,
|
|
'exercise-3-normalizing-our-data'),
|
|
('Exercise 4: Adding Ridge Regression',
|
|
2,
|
|
None,
|
|
'exercise-4-adding-ridge-regression'),
|
|
('Exercise 5: Analytical exercises',
|
|
2,
|
|
None,
|
|
'exercise-5-analytical-exercises')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0064"></a>
|
|
<!-- !split -->
|
|
<h2 id="ridge-and-lasso-regression" class="anchor">Ridge and LASSO Regression </h2>
|
|
|
|
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
|
our optimization problem is
|
|
</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
|
$$
|
|
|
|
<p>or we can state it as</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
|
$$
|
|
|
|
<p>where we have used the definition of a norm-2 vector, that is</p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
|
$$
|
|
|
|
<p>By minimizing the above equation with respect to the parameters
|
|
\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the
|
|
parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by
|
|
defining a new cost function to be optimized, that is
|
|
</p>
|
|
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
|
$$
|
|
|
|
<p>which leads to the Ridge regression minimization problem where we
|
|
require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is
|
|
a finite number larger than zero. By defining
|
|
</p>
|
|
|
|
$$
|
|
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
|
|
$$
|
|
|
|
<p>we have a new optimization equation</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
|
|
$$
|
|
|
|
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. </p>
|
|
|
|
<p>Here we have defined the norm-1 as </p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
|
$$
|
|
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week35-bs063.html">«</a></li>
|
|
<li><a href="._week35-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week35-bs056.html">57</a></li>
|
|
<li><a href="._week35-bs057.html">58</a></li>
|
|
<li><a href="._week35-bs058.html">59</a></li>
|
|
<li><a href="._week35-bs059.html">60</a></li>
|
|
<li><a href="._week35-bs060.html">61</a></li>
|
|
<li><a href="._week35-bs061.html">62</a></li>
|
|
<li><a href="._week35-bs062.html">63</a></li>
|
|
<li><a href="._week35-bs063.html">64</a></li>
|
|
<li class="active"><a href="._week35-bs064.html">65</a></li>
|
|
<li><a href="._week35-bs065.html">66</a></li>
|
|
<li><a href="._week35-bs066.html">67</a></li>
|
|
<li><a href="._week35-bs067.html">68</a></li>
|
|
<li><a href="._week35-bs068.html">69</a></li>
|
|
<li><a href="._week35-bs069.html">70</a></li>
|
|
<li><a href="._week35-bs065.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|