Morten Hjorth-Jensen
454 lines
27 KiB
HTML
454 lines
27 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression">
|
|
<title>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 35', 2, None, 'plans-for-week-35'),
|
|
('Reading recommendations:', 3, None, 'reading-recommendations'),
|
|
('For exercise sessions: Why Linear Regression (aka Ordinary '
|
|
'Least Squares and family), repeat from last week',
|
|
2,
|
|
None,
|
|
'for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'),
|
|
('The equations for ordinary least squares',
|
|
2,
|
|
None,
|
|
'the-equations-for-ordinary-least-squares'),
|
|
('The cost/loss function', 2, None, 'the-cost-loss-function'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Some useful matrix and vector expressions',
|
|
2,
|
|
None,
|
|
'some-useful-matrix-and-vector-expressions'),
|
|
('The Jacobian', 2, None, 'the-jacobian'),
|
|
('Derivatives, example 1', 2, None, 'derivatives-example-1'),
|
|
('Example 2', 2, None, 'example-2'),
|
|
('Example 3', 2, None, 'example-3'),
|
|
('Example 4', 2, None, 'example-4'),
|
|
('The mean squared error and its derivative',
|
|
2,
|
|
None,
|
|
'the-mean-squared-error-and-its-derivative'),
|
|
('Other useful relations', 2, None, 'other-useful-relations'),
|
|
('Meet the Hessian Matrix', 2, None, 'meet-the-hessian-matrix'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Example relevant for the exercises',
|
|
2,
|
|
None,
|
|
'example-relevant-for-the-exercises'),
|
|
('Own code for Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'own-code-for-ordinary-least-squares'),
|
|
('Adding error analysis and training set up',
|
|
2,
|
|
None,
|
|
'adding-error-analysis-and-training-set-up'),
|
|
('Splitting our Data in Training and Test data',
|
|
2,
|
|
None,
|
|
'splitting-our-data-in-training-and-test-data'),
|
|
('The complete code with a simple data set',
|
|
2,
|
|
None,
|
|
'the-complete-code-with-a-simple-data-set'),
|
|
('Making your own test-train splitting',
|
|
2,
|
|
None,
|
|
'making-your-own-test-train-splitting'),
|
|
('Reducing the number of degrees of freedom, overarching view',
|
|
2,
|
|
None,
|
|
'reducing-the-number-of-degrees-of-freedom-overarching-view'),
|
|
('Preprocessing our data', 2, None, 'preprocessing-our-data'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions'),
|
|
('Example of own Standard scaling',
|
|
2,
|
|
None,
|
|
'example-of-own-standard-scaling'),
|
|
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
|
('Testing the Means Squared Error as function of Complexity',
|
|
2,
|
|
None,
|
|
'testing-the-means-squared-error-as-function-of-complexity'),
|
|
('More preprocessing examples, two-dimensional example, the '
|
|
'Franke function',
|
|
2,
|
|
None,
|
|
'more-preprocessing-examples-two-dimensional-example-the-franke-function'),
|
|
('To think about, first part',
|
|
2,
|
|
None,
|
|
'to-think-about-first-part'),
|
|
('More thinking', 2, None, 'more-thinking'),
|
|
('Still thinking', 2, None, 'still-thinking'),
|
|
('What does centering (subtracting the mean values) mean '
|
|
'mathematically?',
|
|
2,
|
|
None,
|
|
'what-does-centering-subtracting-the-mean-values-mean-mathematically'),
|
|
('Further Manipulations', 2, None, 'further-manipulations'),
|
|
('Wrapping it up', 2, None, 'wrapping-it-up'),
|
|
('Linear Regression code, Intercept handling first',
|
|
2,
|
|
None,
|
|
'linear-regression-code-intercept-handling-first'),
|
|
('Material for lecture Monday, August 26',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-august-26'),
|
|
('Mathematical Interpretation of Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'mathematical-interpretation-of-ordinary-least-squares'),
|
|
('Residual Error', 2, None, 'residual-error'),
|
|
('Simple case', 2, None, 'simple-case'),
|
|
('The singular value decomposition',
|
|
2,
|
|
None,
|
|
'the-singular-value-decomposition'),
|
|
('Linear Regression Problems',
|
|
2,
|
|
None,
|
|
'linear-regression-problems'),
|
|
('Fixing the singularity', 2, None, 'fixing-the-singularity'),
|
|
('Basic math of the SVD', 2, None, 'basic-math-of-the-svd'),
|
|
('The SVD, a Fantastic Algorithm',
|
|
2,
|
|
None,
|
|
'the-svd-a-fantastic-algorithm'),
|
|
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
|
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
|
('Note about SVD Calculations',
|
|
2,
|
|
None,
|
|
'note-about-svd-calculations'),
|
|
('Mathematics of the SVD and implications',
|
|
2,
|
|
None,
|
|
'mathematics-of-the-svd-and-implications'),
|
|
('Example Matrix', 2, None, 'example-matrix'),
|
|
('Setting up the Matrix to be inverted',
|
|
2,
|
|
None,
|
|
'setting-up-the-matrix-to-be-inverted'),
|
|
('Further properties (important for our analyses later)',
|
|
2,
|
|
None,
|
|
'further-properties-important-for-our-analyses-later'),
|
|
('Meet the Covariance Matrix',
|
|
2,
|
|
None,
|
|
'meet-the-covariance-matrix'),
|
|
('Introducing the Covariance and Correlation functions',
|
|
2,
|
|
None,
|
|
'introducing-the-covariance-and-correlation-functions'),
|
|
('Covariance and Correlation Matrix',
|
|
2,
|
|
None,
|
|
'covariance-and-correlation-matrix'),
|
|
('Correlation Function and Design/Feature Matrix',
|
|
2,
|
|
None,
|
|
'correlation-function-and-design-feature-matrix'),
|
|
('Covariance Matrix Examples',
|
|
2,
|
|
None,
|
|
'covariance-matrix-examples'),
|
|
('Correlation Matrix', 2, None, 'correlation-matrix'),
|
|
('Correlation Matrix with Pandas',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas'),
|
|
('Correlation Matrix with Pandas and the Franke function',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas-and-the-franke-function'),
|
|
('Rewriting the Covariance and/or Correlation Matrix',
|
|
2,
|
|
None,
|
|
'rewriting-the-covariance-and-or-correlation-matrix'),
|
|
('Linking with the SVD', 2, None, 'linking-with-the-svd'),
|
|
('What does it mean?', 2, None, 'what-does-it-mean'),
|
|
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
|
|
2,
|
|
None,
|
|
'and-finally-boldsymbol-x-boldsymbol-x-t'),
|
|
('Ridge and LASSO Regression',
|
|
2,
|
|
None,
|
|
'ridge-and-lasso-regression'),
|
|
('Deriving the Ridge Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-ridge-regression-equations'),
|
|
('Interpreting the Ridge results',
|
|
2,
|
|
None,
|
|
'interpreting-the-ridge-results'),
|
|
('More interpretations', 2, None, 'more-interpretations'),
|
|
('Deriving the Lasso Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-lasso-regression-equations')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week35-bs.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week35-bs001.html#plans-for-week-35" style="font-size: 80%;"><b>Plans for week 35</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs001.html#reading-recommendations" style="font-size: 80%;"> Reading recommendations:</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs002.html#for-exercise-sessions-why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week" style="font-size: 80%;"><b>For exercise sessions: Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs003.html#the-equations-for-ordinary-least-squares" style="font-size: 80%;"><b>The equations for ordinary least squares</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs004.html#the-cost-loss-function" style="font-size: 80%;"><b>The cost/loss function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs016.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs016.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs007.html#some-useful-matrix-and-vector-expressions" style="font-size: 80%;"><b>Some useful matrix and vector expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs008.html#the-jacobian" style="font-size: 80%;"><b>The Jacobian</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs009.html#derivatives-example-1" style="font-size: 80%;"><b>Derivatives, example 1</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs010.html#example-2" style="font-size: 80%;"><b>Example 2</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs011.html#example-3" style="font-size: 80%;"><b>Example 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs012.html#example-4" style="font-size: 80%;"><b>Example 4</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs013.html#the-mean-squared-error-and-its-derivative" style="font-size: 80%;"><b>The mean squared error and its derivative</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs014.html#other-useful-relations" style="font-size: 80%;"><b>Other useful relations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs015.html#meet-the-hessian-matrix" style="font-size: 80%;"><b>Meet the Hessian Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs016.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs017.html#example-relevant-for-the-exercises" style="font-size: 80%;"><b>Example relevant for the exercises</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs018.html#own-code-for-ordinary-least-squares" style="font-size: 80%;"><b>Own code for Ordinary Least Squares</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs019.html#adding-error-analysis-and-training-set-up" style="font-size: 80%;"><b>Adding error analysis and training set up</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs020.html#splitting-our-data-in-training-and-test-data" style="font-size: 80%;"><b>Splitting our Data in Training and Test data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs021.html#the-complete-code-with-a-simple-data-set" style="font-size: 80%;"><b>The complete code with a simple data set</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs022.html#making-your-own-test-train-splitting" style="font-size: 80%;"><b>Making your own test-train splitting</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs023.html#reducing-the-number-of-degrees-of-freedom-overarching-view" style="font-size: 80%;"><b>Reducing the number of degrees of freedom, overarching view</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs024.html#preprocessing-our-data" style="font-size: 80%;"><b>Preprocessing our data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs025.html#functionality-in-scikit-learn" style="font-size: 80%;"><b>Functionality in Scikit-Learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs026.html#more-preprocessing" style="font-size: 80%;"><b>More preprocessing</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs027.html#frequently-used-scaling-functions" style="font-size: 80%;"><b>Frequently used scaling functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs028.html#example-of-own-standard-scaling" style="font-size: 80%;"><b>Example of own Standard scaling</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs029.html#min-max-scaling" style="font-size: 80%;"><b>Min-Max Scaling</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs030.html#testing-the-means-squared-error-as-function-of-complexity" style="font-size: 80%;"><b>Testing the Means Squared Error as function of Complexity</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs031.html#more-preprocessing-examples-two-dimensional-example-the-franke-function" style="font-size: 80%;"><b>More preprocessing examples, two-dimensional example, the Franke function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs032.html#to-think-about-first-part" style="font-size: 80%;"><b>To think about, first part</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs033.html#more-thinking" style="font-size: 80%;"><b>More thinking</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs034.html#still-thinking" style="font-size: 80%;"><b>Still thinking</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs035.html#what-does-centering-subtracting-the-mean-values-mean-mathematically" style="font-size: 80%;"><b>What does centering (subtracting the mean values) mean mathematically?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs036.html#further-manipulations" style="font-size: 80%;"><b>Further Manipulations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs037.html#wrapping-it-up" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs038.html#linear-regression-code-intercept-handling-first" style="font-size: 80%;"><b>Linear Regression code, Intercept handling first</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs039.html#material-for-lecture-monday-august-26" style="font-size: 80%;"><b>Material for lecture Monday, August 26</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs040.html#mathematical-interpretation-of-ordinary-least-squares" style="font-size: 80%;"><b>Mathematical Interpretation of Ordinary Least Squares</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs041.html#residual-error" style="font-size: 80%;"><b>Residual Error</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs042.html#simple-case" style="font-size: 80%;"><b>Simple case</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs043.html#the-singular-value-decomposition" style="font-size: 80%;"><b>The singular value decomposition</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs044.html#linear-regression-problems" style="font-size: 80%;"><b>Linear Regression Problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs045.html#fixing-the-singularity" style="font-size: 80%;"><b>Fixing the singularity</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs046.html#basic-math-of-the-svd" style="font-size: 80%;"><b>Basic math of the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs047.html#the-svd-a-fantastic-algorithm" style="font-size: 80%;"><b>The SVD, a Fantastic Algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs048.html#economy-size-svd" style="font-size: 80%;"><b>Economy-size SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs049.html#codes-for-the-svd" style="font-size: 80%;"><b>Codes for the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs050.html#note-about-svd-calculations" style="font-size: 80%;"><b>Note about SVD Calculations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs051.html#mathematics-of-the-svd-and-implications" style="font-size: 80%;"><b>Mathematics of the SVD and implications</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs052.html#example-matrix" style="font-size: 80%;"><b>Example Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs053.html#setting-up-the-matrix-to-be-inverted" style="font-size: 80%;"><b>Setting up the Matrix to be inverted</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs054.html#further-properties-important-for-our-analyses-later" style="font-size: 80%;"><b>Further properties (important for our analyses later)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs055.html#meet-the-covariance-matrix" style="font-size: 80%;"><b>Meet the Covariance Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs056.html#introducing-the-covariance-and-correlation-functions" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs057.html#covariance-and-correlation-matrix" style="font-size: 80%;"><b>Covariance and Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs058.html#correlation-function-and-design-feature-matrix" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs059.html#covariance-matrix-examples" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs060.html#correlation-matrix" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs061.html#correlation-matrix-with-pandas" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs062.html#correlation-matrix-with-pandas-and-the-franke-function" style="font-size: 80%;"><b>Correlation Matrix with Pandas and the Franke function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs063.html#rewriting-the-covariance-and-or-correlation-matrix" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs064.html#linking-with-the-svd" style="font-size: 80%;"><b>Linking with the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs065.html#what-does-it-mean" style="font-size: 80%;"><b>What does it mean?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs066.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;"><b>And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#ridge-and-lasso-regression" style="font-size: 80%;"><b>Ridge and LASSO Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs068.html#deriving-the-ridge-regression-equations" style="font-size: 80%;"><b>Deriving the Ridge Regression Equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs069.html#interpreting-the-ridge-results" style="font-size: 80%;"><b>Interpreting the Ridge results</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs070.html#more-interpretations" style="font-size: 80%;"><b>More interpretations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs071.html#deriving-the-lasso-regression-equations" style="font-size: 80%;"><b>Deriving the Lasso Regression Equations</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0067"></a>
|
|
<!-- !split -->
|
|
<h2 id="ridge-and-lasso-regression" class="anchor">Ridge and LASSO Regression </h2>
|
|
|
|
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
|
our optimization problem is
|
|
</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
|
$$
|
|
|
|
<p>or we can state it as</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
|
$$
|
|
|
|
<p>where we have used the definition of a norm-2 vector, that is</p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
|
$$
|
|
|
|
<p>By minimizing the above equation with respect to the parameters
|
|
\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the
|
|
parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by
|
|
defining a new cost function to be optimized, that is
|
|
</p>
|
|
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
|
$$
|
|
|
|
<p>which leads to the Ridge regression minimization problem where we
|
|
require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is
|
|
a finite number larger than zero. By defining
|
|
</p>
|
|
|
|
$$
|
|
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
|
|
$$
|
|
|
|
<p>we have a new optimization equation</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
|
|
$$
|
|
|
|
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. </p>
|
|
|
|
<p>Here we have defined the norm-1 as </p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
|
$$
|
|
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week35-bs066.html">«</a></li>
|
|
<li><a href="._week35-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week35-bs059.html">60</a></li>
|
|
<li><a href="._week35-bs060.html">61</a></li>
|
|
<li><a href="._week35-bs061.html">62</a></li>
|
|
<li><a href="._week35-bs062.html">63</a></li>
|
|
<li><a href="._week35-bs063.html">64</a></li>
|
|
<li><a href="._week35-bs064.html">65</a></li>
|
|
<li><a href="._week35-bs065.html">66</a></li>
|
|
<li><a href="._week35-bs066.html">67</a></li>
|
|
<li class="active"><a href="._week35-bs067.html">68</a></li>
|
|
<li><a href="._week35-bs068.html">69</a></li>
|
|
<li><a href="._week35-bs069.html">70</a></li>
|
|
<li><a href="._week35-bs070.html">71</a></li>
|
|
<li><a href="._week35-bs071.html">72</a></li>
|
|
<li><a href="._week35-bs068.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|