Morten Hjorth-Jensen
470 lines
28 KiB
HTML
470 lines
28 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression">
|
|
<title>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week35.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week35-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 35, August 30 -September 3',
|
|
2,
|
|
None,
|
|
'plans-for-week-35-august-30-september-3'),
|
|
('Thursday September 2', 2, None, 'thursday-september-2'),
|
|
('Why Linear Regression (aka Ordinary Least Squares and family), '
|
|
'repeat from last week',
|
|
2,
|
|
None,
|
|
'why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week'),
|
|
('Regression analysis, overarching aims',
|
|
2,
|
|
None,
|
|
'regression-analysis-overarching-aims'),
|
|
('Regression analysis, overarching aims II',
|
|
2,
|
|
None,
|
|
'regression-analysis-overarching-aims-ii'),
|
|
('Examples', 2, None, 'examples'),
|
|
('General linear models', 2, None, 'general-linear-models'),
|
|
('Rewriting the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'rewriting-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Rewriting the fitting procedure as a linear algebra problem, '
|
|
'more details',
|
|
2,
|
|
None,
|
|
'rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'generalizing-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Generalizing the fitting procedure as a linear algebra problem',
|
|
2,
|
|
None,
|
|
'generalizing-the-fitting-procedure-as-a-linear-algebra-problem'),
|
|
('Optimizing our parameters',
|
|
2,
|
|
None,
|
|
'optimizing-our-parameters'),
|
|
('Our model for the nuclear binding energies',
|
|
2,
|
|
None,
|
|
'our-model-for-the-nuclear-binding-energies'),
|
|
('Optimizing our parameters, more details',
|
|
2,
|
|
None,
|
|
'optimizing-our-parameters-more-details'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Some useful matrix and vector expressions',
|
|
2,
|
|
None,
|
|
'some-useful-matrix-and-vector-expressions'),
|
|
('Meet the Hessian Matrix', 2, None, 'meet-the-hessian-matrix'),
|
|
('Interpretations and optimizing our parameters',
|
|
2,
|
|
None,
|
|
'interpretations-and-optimizing-our-parameters'),
|
|
('Own code for Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'own-code-for-ordinary-least-squares'),
|
|
('Adding error analysis and training set up',
|
|
2,
|
|
None,
|
|
'adding-error-analysis-and-training-set-up'),
|
|
('Splitting our Data in Training and Test data',
|
|
2,
|
|
None,
|
|
'splitting-our-data-in-training-and-test-data'),
|
|
('Examples', 2, None, 'examples'),
|
|
('Making your own test-train splitting',
|
|
2,
|
|
None,
|
|
'making-your-own-test-train-splitting'),
|
|
('The Boston housing data example',
|
|
2,
|
|
None,
|
|
'the-boston-housing-data-example'),
|
|
('Housing data, the code', 2, None, 'housing-data-the-code'),
|
|
('Reducing the number of degrees of freedom, overarching view',
|
|
2,
|
|
None,
|
|
'reducing-the-number-of-degrees-of-freedom-overarching-view'),
|
|
('Preprocessing our data', 2, None, 'preprocessing-our-data'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions'),
|
|
('Example of own Standard scaling',
|
|
2,
|
|
None,
|
|
'example-of-own-standard-scaling'),
|
|
('Min-Max Scaling', 2, None, 'min-max-scaling'),
|
|
('Testing the Means Squared Error as function of Complexity',
|
|
2,
|
|
None,
|
|
'testing-the-means-squared-error-as-function-of-complexity'),
|
|
('More preprocessing examples, Franke function and regression',
|
|
2,
|
|
None,
|
|
'more-preprocessing-examples-franke-function-and-regression'),
|
|
('Mathematical Interpretation of Ordinary Least Squares',
|
|
2,
|
|
None,
|
|
'mathematical-interpretation-of-ordinary-least-squares'),
|
|
('Residual Error', 2, None, 'residual-error'),
|
|
('Simple case', 2, None, 'simple-case'),
|
|
('The singular value decomposition',
|
|
2,
|
|
None,
|
|
'the-singular-value-decomposition'),
|
|
('Linear Regression Problems',
|
|
2,
|
|
None,
|
|
'linear-regression-problems'),
|
|
('Fixing the singularity', 2, None, 'fixing-the-singularity'),
|
|
('Basic math of the SVD', 2, None, 'basic-math-of-the-svd'),
|
|
('The SVD, a Fantastic Algorithm',
|
|
2,
|
|
None,
|
|
'the-svd-a-fantastic-algorithm'),
|
|
('Economy-size SVD', 2, None, 'economy-size-svd'),
|
|
('Codes for the SVD', 2, None, 'codes-for-the-svd'),
|
|
('Note about SVD Calculations',
|
|
2,
|
|
None,
|
|
'note-about-svd-calculations'),
|
|
('Friday September 3', 2, None, 'friday-september-3'),
|
|
('Mathematics of the SVD and implications',
|
|
2,
|
|
None,
|
|
'mathematics-of-the-svd-and-implications'),
|
|
('Example Matrix', 2, None, 'example-matrix'),
|
|
('Setting up the Matrix to be inverted',
|
|
2,
|
|
None,
|
|
'setting-up-the-matrix-to-be-inverted'),
|
|
('Further properties (important for our analyses later)',
|
|
2,
|
|
None,
|
|
'further-properties-important-for-our-analyses-later'),
|
|
('Meet the Covariance Matrix',
|
|
2,
|
|
None,
|
|
'meet-the-covariance-matrix'),
|
|
('Introducing the Covariance and Correlation functions',
|
|
2,
|
|
None,
|
|
'introducing-the-covariance-and-correlation-functions'),
|
|
('Covariance and Correlation Matrix',
|
|
2,
|
|
None,
|
|
'covariance-and-correlation-matrix'),
|
|
('Correlation Function and Design/Feature Matrix',
|
|
2,
|
|
None,
|
|
'correlation-function-and-design-feature-matrix'),
|
|
('Covariance Matrix Examples',
|
|
2,
|
|
None,
|
|
'covariance-matrix-examples'),
|
|
('Correlation Matrix', 2, None, 'correlation-matrix'),
|
|
('Correlation Matrix with Pandas',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas'),
|
|
('Correlation Matrix with Pandas and the Franke function',
|
|
2,
|
|
None,
|
|
'correlation-matrix-with-pandas-and-the-franke-function'),
|
|
('Rewriting the Covariance and/or Correlation Matrix',
|
|
2,
|
|
None,
|
|
'rewriting-the-covariance-and-or-correlation-matrix'),
|
|
('Linking with the SVD', 2, None, 'linking-with-the-svd'),
|
|
('What does it mean?', 2, None, 'what-does-it-mean'),
|
|
('And finally $\\boldsymbol{X}\\boldsymbol{X}^T$',
|
|
2,
|
|
None,
|
|
'and-finally-boldsymbol-x-boldsymbol-x-t'),
|
|
('Ridge and LASSO Regression',
|
|
2,
|
|
None,
|
|
'ridge-and-lasso-regression'),
|
|
('Deriving the Ridge Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-ridge-regression-equations'),
|
|
('Interpreting the Ridge results',
|
|
2,
|
|
None,
|
|
'interpreting-the-ridge-results'),
|
|
('More interpretations', 2, None, 'more-interpretations'),
|
|
('Deriving the Lasso Regression Equations',
|
|
2,
|
|
None,
|
|
'deriving-the-lasso-regression-equations'),
|
|
('Exercises for week 36, September 6-10',
|
|
2,
|
|
None,
|
|
'exercises-for-week-36-september-6-10'),
|
|
('Exercise 1: Adding Ridge and Lasso Regression',
|
|
2,
|
|
None,
|
|
'exercise-1-adding-ridge-and-lasso-regression'),
|
|
('Exercise: Linear Regression for a two-dimensional function',
|
|
3,
|
|
None,
|
|
'exercise-linear-regression-for-a-two-dimensional-function')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week35-bs.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week35-bs001.html#plans-for-week-35-august-30-september-3" style="font-size: 80%;"><b>Plans for week 35, August 30 -September 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs002.html#thursday-september-2" style="font-size: 80%;"><b>Thursday September 2</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs003.html#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week" style="font-size: 80%;"><b>Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs004.html#regression-analysis-overarching-aims" style="font-size: 80%;"><b>Regression analysis, overarching aims</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs005.html#regression-analysis-overarching-aims-ii" style="font-size: 80%;"><b>Regression analysis, overarching aims II</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs023.html#examples" style="font-size: 80%;"><b>Examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs007.html#general-linear-models" style="font-size: 80%;"><b>General linear models</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs008.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Rewriting the fitting procedure as a linear algebra problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs009.html#rewriting-the-fitting-procedure-as-a-linear-algebra-problem-more-details" style="font-size: 80%;"><b>Rewriting the fitting procedure as a linear algebra problem, more details</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Generalizing the fitting procedure as a linear algebra problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs011.html#generalizing-the-fitting-procedure-as-a-linear-algebra-problem" style="font-size: 80%;"><b>Generalizing the fitting procedure as a linear algebra problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs012.html#optimizing-our-parameters" style="font-size: 80%;"><b>Optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs013.html#our-model-for-the-nuclear-binding-energies" style="font-size: 80%;"><b>Our model for the nuclear binding energies</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs014.html#optimizing-our-parameters-more-details" style="font-size: 80%;"><b>Optimizing our parameters, more details</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs019.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs019.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs017.html#some-useful-matrix-and-vector-expressions" style="font-size: 80%;"><b>Some useful matrix and vector expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs018.html#meet-the-hessian-matrix" style="font-size: 80%;"><b>Meet the Hessian Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs019.html#interpretations-and-optimizing-our-parameters" style="font-size: 80%;"><b>Interpretations and optimizing our parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs020.html#own-code-for-ordinary-least-squares" style="font-size: 80%;"><b>Own code for Ordinary Least Squares</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs021.html#adding-error-analysis-and-training-set-up" style="font-size: 80%;"><b>Adding error analysis and training set up</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs022.html#splitting-our-data-in-training-and-test-data" style="font-size: 80%;"><b>Splitting our Data in Training and Test data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs023.html#examples" style="font-size: 80%;"><b>Examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs024.html#making-your-own-test-train-splitting" style="font-size: 80%;"><b>Making your own test-train splitting</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs025.html#the-boston-housing-data-example" style="font-size: 80%;"><b>The Boston housing data example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs026.html#housing-data-the-code" style="font-size: 80%;"><b>Housing data, the code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs027.html#reducing-the-number-of-degrees-of-freedom-overarching-view" style="font-size: 80%;"><b>Reducing the number of degrees of freedom, overarching view</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs028.html#preprocessing-our-data" style="font-size: 80%;"><b>Preprocessing our data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs029.html#functionality-in-scikit-learn" style="font-size: 80%;"><b>Functionality in Scikit-Learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs030.html#more-preprocessing" style="font-size: 80%;"><b>More preprocessing</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs031.html#frequently-used-scaling-functions" style="font-size: 80%;"><b>Frequently used scaling functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs032.html#example-of-own-standard-scaling" style="font-size: 80%;"><b>Example of own Standard scaling</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs033.html#min-max-scaling" style="font-size: 80%;"><b>Min-Max Scaling</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs034.html#testing-the-means-squared-error-as-function-of-complexity" style="font-size: 80%;"><b>Testing the Means Squared Error as function of Complexity</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs035.html#more-preprocessing-examples-franke-function-and-regression" style="font-size: 80%;"><b>More preprocessing examples, Franke function and regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs036.html#mathematical-interpretation-of-ordinary-least-squares" style="font-size: 80%;"><b>Mathematical Interpretation of Ordinary Least Squares</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs037.html#residual-error" style="font-size: 80%;"><b>Residual Error</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs038.html#simple-case" style="font-size: 80%;"><b>Simple case</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs039.html#the-singular-value-decomposition" style="font-size: 80%;"><b>The singular value decomposition</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs040.html#linear-regression-problems" style="font-size: 80%;"><b>Linear Regression Problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs041.html#fixing-the-singularity" style="font-size: 80%;"><b>Fixing the singularity</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs042.html#basic-math-of-the-svd" style="font-size: 80%;"><b>Basic math of the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs043.html#the-svd-a-fantastic-algorithm" style="font-size: 80%;"><b>The SVD, a Fantastic Algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs044.html#economy-size-svd" style="font-size: 80%;"><b>Economy-size SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs045.html#codes-for-the-svd" style="font-size: 80%;"><b>Codes for the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs046.html#note-about-svd-calculations" style="font-size: 80%;"><b>Note about SVD Calculations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs047.html#friday-september-3" style="font-size: 80%;"><b>Friday September 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs048.html#mathematics-of-the-svd-and-implications" style="font-size: 80%;"><b>Mathematics of the SVD and implications</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs049.html#example-matrix" style="font-size: 80%;"><b>Example Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs050.html#setting-up-the-matrix-to-be-inverted" style="font-size: 80%;"><b>Setting up the Matrix to be inverted</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs051.html#further-properties-important-for-our-analyses-later" style="font-size: 80%;"><b>Further properties (important for our analyses later)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs052.html#meet-the-covariance-matrix" style="font-size: 80%;"><b>Meet the Covariance Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs053.html#introducing-the-covariance-and-correlation-functions" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs054.html#covariance-and-correlation-matrix" style="font-size: 80%;"><b>Covariance and Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs055.html#correlation-function-and-design-feature-matrix" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs056.html#covariance-matrix-examples" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs057.html#correlation-matrix" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs058.html#correlation-matrix-with-pandas" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs059.html#correlation-matrix-with-pandas-and-the-franke-function" style="font-size: 80%;"><b>Correlation Matrix with Pandas and the Franke function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs060.html#rewriting-the-covariance-and-or-correlation-matrix" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs061.html#linking-with-the-svd" style="font-size: 80%;"><b>Linking with the SVD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs062.html#what-does-it-mean" style="font-size: 80%;"><b>What does it mean?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs063.html#and-finally-boldsymbol-x-boldsymbol-x-t" style="font-size: 80%;"><b>And finally \( \boldsymbol{X}\boldsymbol{X}^T \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#ridge-and-lasso-regression" style="font-size: 80%;"><b>Ridge and LASSO Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs065.html#deriving-the-ridge-regression-equations" style="font-size: 80%;"><b>Deriving the Ridge Regression Equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs066.html#interpreting-the-ridge-results" style="font-size: 80%;"><b>Interpreting the Ridge results</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs067.html#more-interpretations" style="font-size: 80%;"><b>More interpretations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs068.html#deriving-the-lasso-regression-equations" style="font-size: 80%;"><b>Deriving the Lasso Regression Equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs069.html#exercises-for-week-36-september-6-10" style="font-size: 80%;"><b>Exercises for week 36, September 6-10</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs069.html#exercise-1-adding-ridge-and-lasso-regression" style="font-size: 80%;"><b>Exercise 1: Adding Ridge and Lasso Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week35-bs069.html#exercise-linear-regression-for-a-two-dimensional-function" style="font-size: 80%;"> Exercise: Linear Regression for a two-dimensional function</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0064"></a>
|
|
<!-- !split -->
|
|
<h2 id="ridge-and-lasso-regression" class="anchor">Ridge and LASSO Regression </h2>
|
|
|
|
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
|
our optimization problem is
|
|
</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
|
$$
|
|
|
|
<p>or we can state it as</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
|
$$
|
|
|
|
<p>where we have used the definition of a norm-2 vector, that is</p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
|
$$
|
|
|
|
<p>By minimizing the above equation with respect to the parameters
|
|
\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the
|
|
parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by
|
|
defining a new cost function to be optimized, that is
|
|
</p>
|
|
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
|
$$
|
|
|
|
<p>which leads to the Ridge regression minimization problem where we
|
|
require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is
|
|
a finite number larger than zero. By defining
|
|
</p>
|
|
|
|
$$
|
|
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
|
|
$$
|
|
|
|
<p>we have a new optimization equation</p>
|
|
$$
|
|
{\displaystyle \min_{\boldsymbol{\beta}\in
|
|
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
|
|
$$
|
|
|
|
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. </p>
|
|
|
|
<p>Here we have defined the norm-1 as </p>
|
|
$$
|
|
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
|
$$
|
|
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week35-bs063.html">«</a></li>
|
|
<li><a href="._week35-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week35-bs056.html">57</a></li>
|
|
<li><a href="._week35-bs057.html">58</a></li>
|
|
<li><a href="._week35-bs058.html">59</a></li>
|
|
<li><a href="._week35-bs059.html">60</a></li>
|
|
<li><a href="._week35-bs060.html">61</a></li>
|
|
<li><a href="._week35-bs061.html">62</a></li>
|
|
<li><a href="._week35-bs062.html">63</a></li>
|
|
<li><a href="._week35-bs063.html">64</a></li>
|
|
<li class="active"><a href="._week35-bs064.html">65</a></li>
|
|
<li><a href="._week35-bs065.html">66</a></li>
|
|
<li><a href="._week35-bs066.html">67</a></li>
|
|
<li><a href="._week35-bs067.html">68</a></li>
|
|
<li><a href="._week35-bs068.html">69</a></li>
|
|
<li><a href="._week35-bs069.html">70</a></li>
|
|
<li><a href="._week35-bs065.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|