mhjensen
254 lines
14 KiB
HTML
254 lines
14 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Ridge and Lasso Regression">
|
|
|
|
<title>Week 37: Ridge and Lasso Regression</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37', 2, None, '___sec0'),
|
|
('Thursday September 10', 2, None, '___sec1'),
|
|
('A Bayesian approach to develop intuition about skrinkage '
|
|
'methods',
|
|
2,
|
|
None,
|
|
'___sec2'),
|
|
('The singular value decomposition', 2, None, '___sec3'),
|
|
('Linear Regression Problems', 2, None, '___sec4'),
|
|
('Fixing the singularity', 2, None, '___sec5'),
|
|
('Basic math of the SVD', 2, None, '___sec6'),
|
|
('The SVD, a Fantastic Algorithm', 2, None, '___sec7'),
|
|
('Economy-size SVD', 2, None, '___sec8'),
|
|
('Codes for the SVD', 2, None, '___sec9'),
|
|
('Mathematical Properties', 2, None, '___sec10'),
|
|
('Friday September 12', 2, None, '___sec11'),
|
|
('Ridge and LASSO Regression', 2, None, '___sec12'),
|
|
('More on Ridge Regression', 2, None, '___sec13'),
|
|
('Interpreting the Ridge results', 2, None, '___sec14'),
|
|
('More interpretations', 2, None, '___sec15'),
|
|
('A better understanding of regularization', 2, None, '___sec16'),
|
|
('Decomposing the OLS and Ridge expressions',
|
|
2,
|
|
None,
|
|
'___sec17'),
|
|
('Introducing the Covariance and Correlation functions',
|
|
2,
|
|
None,
|
|
'___sec18'),
|
|
('Correlation Function and Design/Feature Matrix',
|
|
2,
|
|
None,
|
|
'___sec19'),
|
|
('Covariance Matrix Examples', 2, None, '___sec20'),
|
|
('Correlation Matrix', 2, None, '___sec21'),
|
|
('Correlation Matrix with Pandas', 2, None, '___sec22'),
|
|
('Correlation Matrix with Pandas and the Franke function',
|
|
2,
|
|
None,
|
|
'___sec23'),
|
|
('Rewriting the Covariance and/or Correlation Matrix',
|
|
2,
|
|
None,
|
|
'___sec24'),
|
|
('Linking with SVD', 2, None, '___sec25')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week37-bs.html">Week 37: Ridge and Lasso Regression</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week37-bs001.html#___sec0" style="font-size: 80%;">Plans for week 37</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs002.html#___sec1" style="font-size: 80%;">Thursday September 10</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs003.html#___sec2" style="font-size: 80%;">A Bayesian approach to develop intuition about skrinkage methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs004.html#___sec3" style="font-size: 80%;">The singular value decomposition</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs005.html#___sec4" style="font-size: 80%;">Linear Regression Problems</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs006.html#___sec5" style="font-size: 80%;">Fixing the singularity</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs007.html#___sec6" style="font-size: 80%;">Basic math of the SVD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs008.html#___sec7" style="font-size: 80%;">The SVD, a Fantastic Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#___sec8" style="font-size: 80%;">Economy-size SVD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs010.html#___sec9" style="font-size: 80%;">Codes for the SVD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs011.html#___sec10" style="font-size: 80%;">Mathematical Properties</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs012.html#___sec11" style="font-size: 80%;">Friday September 12</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs013.html#___sec12" style="font-size: 80%;">Ridge and LASSO Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs014.html#___sec13" style="font-size: 80%;">More on Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs015.html#___sec14" style="font-size: 80%;">Interpreting the Ridge results</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs016.html#___sec15" style="font-size: 80%;">More interpretations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs017.html#___sec16" style="font-size: 80%;">A better understanding of regularization</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs018.html#___sec17" style="font-size: 80%;">Decomposing the OLS and Ridge expressions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs019.html#___sec18" style="font-size: 80%;">Introducing the Covariance and Correlation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs020.html#___sec19" style="font-size: 80%;">Correlation Function and Design/Feature Matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#___sec20" style="font-size: 80%;">Covariance Matrix Examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec21" style="font-size: 80%;">Correlation Matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#___sec22" style="font-size: 80%;">Correlation Matrix with Pandas</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs024.html#___sec23" style="font-size: 80%;">Correlation Matrix with Pandas and the Franke function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs025.html#___sec24" style="font-size: 80%;">Rewriting the Covariance and/or Correlation Matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs026.html#___sec25" style="font-size: 80%;">Linking with SVD</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0022"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec21" class="anchor">Correlation Matrix </h2>
|
|
|
|
<p>
|
|
The previous example can be converted into the correlation matrix by
|
|
simply scaling the matrix elements with the variances. We should also
|
|
subtract the mean values for each column. This leads to the following
|
|
code which sets up the correlations matrix for the previous example in
|
|
a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors).
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
|
<span style="color: #408080; font-style: italic"># define two vectors </span>
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>random(size<span style="color: #666666">=</span>n)
|
|
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(size<span style="color: #666666">=</span>n)
|
|
<span style="color: #408080; font-style: italic">#scaling the x and y vectors </span>
|
|
x <span style="color: #666666">=</span> x <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(x)
|
|
y <span style="color: #666666">=</span> y <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y)
|
|
variance_x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@x</span>)<span style="color: #666666">/</span>n
|
|
variance_y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(y<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
<span style="color: #008000; font-weight: bold">print</span>(variance_x)
|
|
<span style="color: #008000; font-weight: bold">print</span>(variance_y)
|
|
cov_xy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
cov_xx <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@x</span>)<span style="color: #666666">/</span>n
|
|
cov_yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(y<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
C <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #666666">2</span>,<span style="color: #666666">2</span>))
|
|
C[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>]<span style="color: #666666">=</span> cov_xx<span style="color: #666666">/</span>variance_x
|
|
C[<span style="color: #666666">1</span>,<span style="color: #666666">1</span>]<span style="color: #666666">=</span> cov_yy<span style="color: #666666">/</span>variance_y
|
|
C[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]<span style="color: #666666">=</span> cov_xy<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sqrt(variance_y<span style="color: #666666">*</span>variance_x)
|
|
C[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>]<span style="color: #666666">=</span> C[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]
|
|
<span style="color: #008000; font-weight: bold">print</span>(C)
|
|
</pre></div>
|
|
<p>
|
|
We see that the matrix elements along the diagonal are one as they
|
|
should be and that the matrix is symmetric. Furthermore, diagonalizing
|
|
this matrix we easily see that it is a positive definite matrix.
|
|
|
|
<p>
|
|
The above procedure with <b>numpy</b> can be made more compact if we use <b>pandas</b>.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week37-bs021.html">«</a></li>
|
|
<li><a href="._week37-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs014.html">15</a></li>
|
|
<li><a href="._week37-bs015.html">16</a></li>
|
|
<li><a href="._week37-bs016.html">17</a></li>
|
|
<li><a href="._week37-bs017.html">18</a></li>
|
|
<li><a href="._week37-bs018.html">19</a></li>
|
|
<li><a href="._week37-bs019.html">20</a></li>
|
|
<li><a href="._week37-bs020.html">21</a></li>
|
|
<li><a href="._week37-bs021.html">22</a></li>
|
|
<li class="active"><a href="._week37-bs022.html">23</a></li>
|
|
<li><a href="._week37-bs023.html">24</a></li>
|
|
<li><a href="._week37-bs024.html">25</a></li>
|
|
<li><a href="._week37-bs025.html">26</a></li>
|
|
<li><a href="._week37-bs026.html">27</a></li>
|
|
<li><a href="._week37-bs023.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|