mhjensen
302 lines
17 KiB
HTML
302 lines
17 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction">
|
|
|
|
<title>Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Reducing the number of degrees of freedom, overarching view',
|
|
2,
|
|
None,
|
|
'___sec0'),
|
|
('Preprocessing our data', 2, None, '___sec1'),
|
|
('More preprocessing', 2, None, '___sec2'),
|
|
('Simple preprocessing examples, Franke function and regression',
|
|
2,
|
|
None,
|
|
'___sec3'),
|
|
('Simple preprocessing examples, breast cancer data and '
|
|
'classification, Support Vector Machines',
|
|
2,
|
|
None,
|
|
'___sec4'),
|
|
('More on Cancer Data, now with Logistic Regression',
|
|
2,
|
|
None,
|
|
'___sec5'),
|
|
('Why should we think of reducing the dimensionality',
|
|
2,
|
|
None,
|
|
'___sec6'),
|
|
('Basic ideas of the Principal Component Analysis (PCA)',
|
|
2,
|
|
None,
|
|
'___sec7'),
|
|
('Introducing the Covariance and Correlation functions',
|
|
2,
|
|
None,
|
|
'___sec8'),
|
|
('Correlation Function and Design/Feature Matrix',
|
|
2,
|
|
None,
|
|
'___sec9'),
|
|
('Covariance Matrix Examples', 2, None, '___sec10'),
|
|
('Correlation Matrix', 2, None, '___sec11'),
|
|
('Correlation Matrix with Pandas', 2, None, '___sec12'),
|
|
('Correlation Matrix with Pandas and the Franke function',
|
|
2,
|
|
None,
|
|
'___sec13'),
|
|
('Rewriting the Covariance and/or Correlation Matrix',
|
|
2,
|
|
None,
|
|
'___sec14'),
|
|
('Towards the PCA theorem', 2, None, '___sec15'),
|
|
('The Algorithm before the Theorem', 2, None, '___sec16'),
|
|
('Writing our own PCA code', 2, None, '___sec17'),
|
|
('Compute the sample mean and center the data',
|
|
3,
|
|
None,
|
|
'___sec18'),
|
|
('Compute the sample covariance', 3, None, '___sec19'),
|
|
('Diagonalize the sample covariance matrix to obtain the '
|
|
'principal components',
|
|
3,
|
|
None,
|
|
'___sec20'),
|
|
('Classical PCA Theorem', 2, None, '___sec21'),
|
|
('Proof of the PCA Theorem', 2, None, '___sec22'),
|
|
('PCA Proof continued', 2, None, '___sec23'),
|
|
('The final step', 2, None, '___sec24'),
|
|
('Geometric Interpretation and link with Singular Value '
|
|
'Decomposition',
|
|
2,
|
|
None,
|
|
'___sec25'),
|
|
('Principal Component Analysis', 2, None, '___sec26'),
|
|
('PCA and scikit-learn', 2, None, '___sec27'),
|
|
('Back to the Cancer Data', 2, None, '___sec28'),
|
|
('More on the PCA', 2, None, '___sec29'),
|
|
('Incremental PCA', 2, None, '___sec30'),
|
|
('Randomized PCA', 2, None, '___sec31'),
|
|
('Kernel PCA', 2, None, '___sec32'),
|
|
('LLE', 2, None, '___sec33'),
|
|
('Other techniques', 2, None, '___sec34')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="DimRed-bs.html">Data Analysis and Machine Learning: Preprocessing and Dimensionality Reduction</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;"><b>Reducing the number of degrees of freedom, overarching view</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;"><b>Preprocessing our data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;"><b>More preprocessing</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;"><b>Simple preprocessing examples, Franke function and regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;"><b>Simple preprocessing examples, breast cancer data and classification, Support Vector Machines</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;"><b>More on Cancer Data, now with Logistic Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;"><b>Why should we think of reducing the dimensionality</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;"><b>Basic ideas of the Principal Component Analysis (PCA)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs010.html#___sec9" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs011.html#___sec10" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs013.html#___sec12" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs014.html#___sec13" style="font-size: 80%;"><b>Correlation Matrix with Pandas and the Franke function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs015.html#___sec14" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs016.html#___sec15" style="font-size: 80%;"><b>Towards the PCA theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs017.html#___sec16" style="font-size: 80%;"><b>The Algorithm before the Theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs018.html#___sec17" style="font-size: 80%;"><b>Writing our own PCA code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs018.html#___sec18" style="font-size: 80%;"> Compute the sample mean and center the data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs018.html#___sec19" style="font-size: 80%;"> Compute the sample covariance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs018.html#___sec20" style="font-size: 80%;"> Diagonalize the sample covariance matrix to obtain the principal components</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs019.html#___sec21" style="font-size: 80%;"><b>Classical PCA Theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs020.html#___sec22" style="font-size: 80%;"><b>Proof of the PCA Theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs021.html#___sec23" style="font-size: 80%;"><b>PCA Proof continued</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs022.html#___sec24" style="font-size: 80%;"><b>The final step</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs023.html#___sec25" style="font-size: 80%;"><b>Geometric Interpretation and link with Singular Value Decomposition</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs024.html#___sec26" style="font-size: 80%;"><b>Principal Component Analysis</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs025.html#___sec27" style="font-size: 80%;"><b>PCA and scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs026.html#___sec28" style="font-size: 80%;"><b>Back to the Cancer Data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs027.html#___sec29" style="font-size: 80%;"><b>More on the PCA</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs028.html#___sec30" style="font-size: 80%;"><b>Incremental PCA</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs029.html#___sec31" style="font-size: 80%;"><b>Randomized PCA</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs030.html#___sec32" style="font-size: 80%;"><b>Kernel PCA</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs031.html#___sec33" style="font-size: 80%;"><b>LLE</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._DimRed-bs032.html#___sec34" style="font-size: 80%;"><b>Other techniques</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0012"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec11" class="anchor">Correlation Matrix </h2>
|
|
|
|
<p>
|
|
The previous example can be converted into the correlation matrix by
|
|
simply scaling the matrix elements with the variances. We should also
|
|
subtract the mean values for each column. This leads to the following
|
|
code which sets up the correlations matrix for the previous example in
|
|
a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors).
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
|
<span style="color: #408080; font-style: italic"># define two vectors </span>
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>random(size<span style="color: #666666">=</span>n)
|
|
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(size<span style="color: #666666">=</span>n)
|
|
<span style="color: #408080; font-style: italic">#scaling the x and y vectors </span>
|
|
x <span style="color: #666666">=</span> x <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(x)
|
|
y <span style="color: #666666">=</span> y <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y)
|
|
variance_x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@x</span>)<span style="color: #666666">/</span>n
|
|
variance_y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(y<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
<span style="color: #008000; font-weight: bold">print</span>(variance_x)
|
|
<span style="color: #008000; font-weight: bold">print</span>(variance_y)
|
|
cov_xy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
cov_xx <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(x<span style="color: #AA22FF">@x</span>)<span style="color: #666666">/</span>n
|
|
cov_yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(y<span style="color: #AA22FF">@y</span>)<span style="color: #666666">/</span>n
|
|
C <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #666666">2</span>,<span style="color: #666666">2</span>))
|
|
C[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>]<span style="color: #666666">=</span> cov_xx<span style="color: #666666">/</span>variance_x
|
|
C[<span style="color: #666666">1</span>,<span style="color: #666666">1</span>]<span style="color: #666666">=</span> cov_yy<span style="color: #666666">/</span>variance_y
|
|
C[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]<span style="color: #666666">=</span> cov_xy<span style="color: #666666">/</span>np<span style="color: #666666">.</span>sqrt(variance_y<span style="color: #666666">*</span>variance_x)
|
|
C[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>]<span style="color: #666666">=</span> C[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]
|
|
<span style="color: #008000; font-weight: bold">print</span>(C)
|
|
</pre></div>
|
|
<p>
|
|
We see that the matrix elements along the diagonal are one as they
|
|
should be and that the matrix is symmetric. Furthermore, diagonalizing
|
|
this matrix we easily see that it is a positive definite matrix.
|
|
|
|
<p>
|
|
The above procedure with <b>numpy</b> can be made more compact if we use <b>pandas</b>.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._DimRed-bs011.html">«</a></li>
|
|
<li><a href="._DimRed-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._DimRed-bs004.html">5</a></li>
|
|
<li><a href="._DimRed-bs005.html">6</a></li>
|
|
<li><a href="._DimRed-bs006.html">7</a></li>
|
|
<li><a href="._DimRed-bs007.html">8</a></li>
|
|
<li><a href="._DimRed-bs008.html">9</a></li>
|
|
<li><a href="._DimRed-bs009.html">10</a></li>
|
|
<li><a href="._DimRed-bs010.html">11</a></li>
|
|
<li><a href="._DimRed-bs011.html">12</a></li>
|
|
<li class="active"><a href="._DimRed-bs012.html">13</a></li>
|
|
<li><a href="._DimRed-bs013.html">14</a></li>
|
|
<li><a href="._DimRed-bs014.html">15</a></li>
|
|
<li><a href="._DimRed-bs015.html">16</a></li>
|
|
<li><a href="._DimRed-bs016.html">17</a></li>
|
|
<li><a href="._DimRed-bs017.html">18</a></li>
|
|
<li><a href="._DimRed-bs018.html">19</a></li>
|
|
<li><a href="._DimRed-bs019.html">20</a></li>
|
|
<li><a href="._DimRed-bs020.html">21</a></li>
|
|
<li><a href="._DimRed-bs021.html">22</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._DimRed-bs032.html">33</a></li>
|
|
<li><a href="._DimRed-bs013.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|