mhjensen
502 lines
20 KiB
HTML
502 lines
20 KiB
HTML
\
|
|
<!DOCTYPE html>
|
|
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Dimensionality Reduction">
|
|
|
|
<title>Data Analysis and Machine Learning: Dimensionality Reduction</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
|
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
|
|
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<!--
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
|
|
-->
|
|
|
|
<!-- For syntax highlighting -->
|
|
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
|
|
|
|
<!-- Printing and PDF exports -->
|
|
<script>
|
|
var link = document.createElement( 'link' );
|
|
link.rel = 'stylesheet';
|
|
link.type = 'text/css';
|
|
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
|
|
document.getElementsByTagName( 'head' )[0].appendChild( link );
|
|
</script>
|
|
|
|
<style type="text/css">
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
.reveal .alert-text-small { font-size: 80%; }
|
|
.reveal .alert-text-large { font-size: 130%; }
|
|
.reveal .alert-text-normal { font-size: 90%; }
|
|
.reveal .alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:5px solid #bababa;
|
|
-webkit-border-radius: 14px; -moz-border-radius: 14px;
|
|
border-radius:14px;
|
|
background-position: 10px 10px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 30px; /* 55px; if icon */
|
|
}
|
|
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
|
|
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
/*.reveal .alert li {margin-top: 1em}*/
|
|
.reveal .alert-block p+p {margin-top:5px}
|
|
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
|
|
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
|
|
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
|
|
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
|
|
|
|
</style>
|
|
|
|
|
|
|
|
<!-- Styles for table layout of slides -->
|
|
<style type="text/css">
|
|
td.padding {
|
|
padding-top:20px;
|
|
padding-bottom:20px;
|
|
padding-right:50px;
|
|
padding-left:50px;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
<body>
|
|
<div class="reveal">
|
|
|
|
<!-- Any section element inside the <div class="slides"> container
|
|
is displayed as a slide -->
|
|
|
|
<div class="slides">
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Dimensionality Reduction</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p> <br>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p> <br>
|
|
<center><h4>Oct 26, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec0">Reducing the number of degrees of freedom, overarching view </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Many Machine Learning problems involve thousands or even millions of features for each training
|
|
instance. Not only does this make training extremely slow, it can also make it much harder to find a good
|
|
solution, as we will see. This problem is often referred to as the curse of dimensionality.
|
|
Fortunately, in real-world problems, it is often possible to reduce the number of features considerably,
|
|
turning an intractable problem into a tractable one.
|
|
|
|
<p>
|
|
Here we will discuss some of the most popular dimensionality
|
|
reduction techniques: the principal component analysis PCA, Kernel PCA, and Locally Linear Embedding (LLE).
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec1">Principal Component Analysis </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
Principal Component Analysis (PCA) is by far the most popular dimensionality reduction algorithm.
|
|
First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it.
|
|
|
|
<p>
|
|
The following Python code uses NumPy’s <b>svd()</b> function to obtain all the principal components of the
|
|
training set, then extracts the first two principal components
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>X_centered = X - X.mean(axis=<span style="color: #B452CD">0</span>)
|
|
U, s, V = np.linalg.svd(X_centered)
|
|
c1 = V.T[:, <span style="color: #B452CD">0</span>]
|
|
c2 = V.T[:, <span style="color: #B452CD">1</span>]
|
|
</pre></div>
|
|
<p>
|
|
PCA assumes that the dataset is centered around the origin. Scikit-Learn’s PCA classes take care of centering
|
|
the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don’t
|
|
forget to center the data first.
|
|
|
|
<p>
|
|
Once you have identified all the principal components, you can reduce the dimensionality of the dataset
|
|
down to \( d \) dimensions by projecting it onto the hyperplane defined by the first \( d \) principal components.
|
|
Selecting this hyperplane ensures that the projection will preserve as much variance as possible.
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>W2 = V.T[:, :<span style="color: #B452CD">2</span>]
|
|
X2D = X_centered.dot(W2)
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec2">PCA and scikit-learn </h2>
|
|
|
|
<p>
|
|
Scikit-Learn’s PCA class implements PCA using SVD decomposition just like we did before. The
|
|
following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note
|
|
that it automatically takes care of centering the data):
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.decomposition</span> <span style="color: #8B008B; font-weight: bold">import</span> PCA
|
|
pca = PCA(n_components = <span style="color: #B452CD">2</span>)
|
|
X2D = pca.fit_transform(X)
|
|
</pre></div>
|
|
<p>
|
|
After fitting the PCA transformer to the dataset, you can access the principal components using the
|
|
components variable (note that it contains the PCs as horizontal vectors, so, for example, the first
|
|
principal component is equal to
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>pca.components_.T[:, <span style="color: #B452CD">0</span>]).
|
|
</pre></div>
|
|
<p>
|
|
Another very useful piece of information is the explained variance ratio of each principal component,
|
|
available via the \( explained\_variance\_ratio \) variable. It indicates the proportion of the dataset’s
|
|
variance that lies along the axis of each principal component.
|
|
More material to come here.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec3">More on the PCA </h2>
|
|
Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to
|
|
choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%).
|
|
Unless, of course, you are reducing dimensionality for data visualization — in that case you will
|
|
generally want to reduce the dimensionality down to 2 or 3.
|
|
The following code computes PCA without reducing dimensionality, then computes the minimum number
|
|
of dimensions required to preserve 95% of the training set’s variance:
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>pca = PCA()
|
|
pca.fit(X)
|
|
cumsum = np.cumsum(pca.explained_variance_ratio_)
|
|
d = np.argmax(cumsum >= <span style="color: #B452CD">0.95</span>) + <span style="color: #B452CD">1</span>
|
|
</pre></div>
|
|
<p>
|
|
You could then set \( n\_components=d \) and run PCA again. However, there is a much better option: instead
|
|
of specifying the number of principal components you want to preserve, you can set \( n\_components \) to be
|
|
a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve:
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>pca = PCA(n_components=<span style="color: #B452CD">0.95</span>)
|
|
X_reduced = pca.fit_transform(X)
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec4">Incremental PCA </h2>
|
|
One problem with the preceding implementation of PCA is that it requires the whole training set to fit in
|
|
memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have
|
|
been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch
|
|
at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new
|
|
instances arrive).
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec5">Randomized PCA </h2>
|
|
|
|
<p>
|
|
Scikit-Learn offers yet another option to perform PCA, called Randomized PCA. This is a stochastic
|
|
algorithm that quickly finds an approximation of the first d principal components. Its computational
|
|
complexity is \( O(m \times d^2)+O(d^3) \), instead of \( O(m \times n^2) + O(n^3) \), so it is dramatically faster than the
|
|
previous algorithms when \( d \) is much smaller than \( n \).
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec6">Kernel PCA </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
The kernel trick is a mathematical technique that implicitly maps instances into a
|
|
very high-dimensional space (called the feature space), enabling nonlinear classification and regression
|
|
with Support Vector Machines. Recall that a linear decision boundary in the high-dimensional feature
|
|
space corresponds to a complex nonlinear decision boundary in the original space.
|
|
It turns out that the same trick can be applied to PCA, making it possible to perform complex nonlinear
|
|
projections for dimensionality reduction. This is called Kernel PCA (kPCA). It is often good at
|
|
preserving clusters of instances after projection, or sometimes even unrolling datasets that lie close to a
|
|
twisted manifold.
|
|
For example, the following code uses Scikit-Learn’s KernelPCA class to perform kPCA with an
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.decomposition</span> <span style="color: #8B008B; font-weight: bold">import</span> KernelPCA
|
|
rbf_pca = KernelPCA(n_components = <span style="color: #B452CD">2</span>, kernel=<span style="color: #CD5555">"rbf"</span>, gamma=<span style="color: #B452CD">0.04</span>)
|
|
X_reduced = rbf_pca.fit_transform(X)
|
|
</pre></div>
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec7">LLE </h2>
|
|
|
|
<p>
|
|
Locally Linear Embedding (LLE) is another very powerful nonlinear dimensionality reduction
|
|
(NLDR) technique. It is a Manifold Learning technique that does not rely on projections like the previous
|
|
algorithms. In a nutshell, LLE works by first measuring how each training instance linearly relates to its
|
|
closest neighbors (c.n.), and then looking for a low-dimensional representation of the training set where
|
|
these local relationships are best preserved (more details shortly).
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec8">Other techniques </h2>
|
|
|
|
<p>
|
|
There are many other dimensionality reduction techniques, several of which are available in Scikit-Learn.
|
|
|
|
<p>
|
|
Here are some of the most popular:
|
|
|
|
<ul>
|
|
<p><li> <b>Multidimensional Scaling (MDS)</b> reduces dimensionality while trying to preserve the distances between the instances.</li>
|
|
<p><li> <b>Isomap</b> creates a graph by connecting each instance to its nearest neighbors, then reduces dimensionality while trying to preserve the geodesic distances between the instances.</li>
|
|
<p><li> <b>t-Distributed Stochastic Neighbor Embedding</b> (t-SNE) reduces dimensionality while trying to keep similar instances close and dissimilar instances apart. It is mostly used for visualization, in particular to visualize clusters of instances in high-dimensional space (e.g., to visualize the MNIST images in 2D).</li>
|
|
<p><li> Linear Discriminant Analysis (LDA) is actually a classification algorithm, but during training it learns the most discriminative axes between the classes, and these axes can then be used to define a hyperplane onto which to project the data. The benefit is that the projection will keep classes as far apart as possible, so LDA is a good technique to reduce dimensionality before running another classification algorithm such as a Support Vector Machine (SVM) classifier discussed in the SVM lectures.</li>
|
|
</ul>
|
|
</section>
|
|
|
|
|
|
|
|
</div> <!-- class="slides" -->
|
|
</div> <!-- class="reveal" -->
|
|
|
|
<script src="reveal.js/lib/js/head.min.js"></script>
|
|
<script src="reveal.js/js/reveal.js"></script>
|
|
|
|
<script>
|
|
// Full list of configuration options available here:
|
|
// https://github.com/hakimel/reveal.js#configuration
|
|
Reveal.initialize({
|
|
|
|
// Display navigation controls in the bottom right corner
|
|
controls: true,
|
|
|
|
// Display progress bar (below the horiz. slider)
|
|
progress: true,
|
|
|
|
// Display the page number of the current slide
|
|
slideNumber: true,
|
|
|
|
// Push each slide change to the browser history
|
|
history: false,
|
|
|
|
// Enable keyboard shortcuts for navigation
|
|
keyboard: true,
|
|
|
|
// Enable the slide overview mode
|
|
overview: true,
|
|
|
|
// Vertical centering of slides
|
|
//center: true,
|
|
center: false,
|
|
|
|
// Enables touch navigation on devices with touch input
|
|
touch: true,
|
|
|
|
// Loop the presentation
|
|
loop: false,
|
|
|
|
// Change the presentation direction to be RTL
|
|
rtl: false,
|
|
|
|
// Turns fragments on and off globally
|
|
fragments: true,
|
|
|
|
// Flags if the presentation is running in an embedded mode,
|
|
// i.e. contained within a limited portion of the screen
|
|
embedded: false,
|
|
|
|
// Number of milliseconds between automatically proceeding to the
|
|
// next slide, disabled when set to 0, this value can be overwritten
|
|
// by using a data-autoslide attribute on your slides
|
|
autoSlide: 0,
|
|
|
|
// Stop auto-sliding after user input
|
|
autoSlideStoppable: true,
|
|
|
|
// Enable slide navigation via mouse wheel
|
|
mouseWheel: false,
|
|
|
|
// Hides the address bar on mobile devices
|
|
hideAddressBar: true,
|
|
|
|
// Opens links in an iframe preview overlay
|
|
previewLinks: false,
|
|
|
|
// Transition style
|
|
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
|
|
|
|
// Transition speed
|
|
transitionSpeed: 'default', // default/fast/slow
|
|
|
|
// Transition style for full page slide backgrounds
|
|
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
|
|
|
|
// Number of slides away from the current that are visible
|
|
viewDistance: 3,
|
|
|
|
// Parallax background image
|
|
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
|
|
|
|
// Parallax background size
|
|
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
|
|
|
|
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
|
|
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
|
|
|
|
});
|
|
|
|
Reveal.initialize({
|
|
dependencies: [
|
|
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
|
|
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
|
|
|
|
// Interpret Markdown in <section> elements
|
|
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
|
|
// Syntax highlight for <code> elements
|
|
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
|
|
|
|
// Zoom in and out with Alt+click
|
|
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Speaker notes
|
|
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Remote control your reveal.js presentation using a touch device
|
|
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// MathJax
|
|
//{ src: 'reveal.js/plugin/math/math.js', async: true }
|
|
]
|
|
});
|
|
|
|
Reveal.initialize({
|
|
|
|
// The "normal" size of the presentation, aspect ratio will be preserved
|
|
// when the presentation is scaled to fit different resolutions. Can be
|
|
// specified using percentage units.
|
|
width: 1170, // original: 960,
|
|
height: 700,
|
|
|
|
// Factor of the display size that should remain empty around the content
|
|
margin: 0.1,
|
|
|
|
// Bounds for smallest/largest possible scale to apply to content
|
|
minScale: 0.2,
|
|
maxScale: 1.0
|
|
|
|
});
|
|
</script>
|
|
|
|
<!-- begin footer logo
|
|
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
|
|
<img src="somelogo.png">
|
|
</div>
|
|
end footer logo -->
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|