lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update on dim red

Browse Source
This commit is contained in:
mhjensen
2018-10-25 04:41:10 +02:00
parent d2efb8dd0b
commit 133fe9d0ea
10 changed files with 373 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/DimRed/html/._DimRed-bs000.html
+35 -7
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -108,7 +132,7 @@ end of tocinfo -->
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Oct 24, 2018</h4></center> <!-- date -->
<center><h4>Oct 25, 2018</h4></center> <!-- date -->
<br>
<p>
@@ -127,6 +151,10 @@ end of tocinfo -->
<li><a href="._DimRed-bs003.html">4</a></li>
<li><a href="._DimRed-bs004.html">5</a></li>
<li><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DimRed/html/._DimRed-bs001.html
+36 -8
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -102,8 +126,8 @@ Fortunately, in real-world problems, it is often possible to reduce the number o
turning an intractable problem into a tractable one.
<p>
and we will go through three of the most popular dimensionality
reduction techniques: PCA, Kernel PCA, and LLE.
Here we will discuss some of the most popular dimensionality
reduction techniques: the principal component analysis PCA, Kernel PCA, and Locally Linear Embedding (LLE).
<p>
</div>
@@ -121,6 +145,10 @@ reduction techniques: PCA, Kernel PCA, and LLE.
<li><a href="._DimRed-bs003.html">4</a></li>
<li><a href="._DimRed-bs004.html">5</a></li>
<li><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs002.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DimRed/html/._DimRed-bs002.html
+51 -125
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -97,133 +121,31 @@ Principal Component Analysis (PCA) is by far the most popular dimensionality red
First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it.
<p>
The following Python code uses NumPy&#8217;s svd() function to obtain all the principal components of the
training set, then extracts the first two PCs:
X_centered = X - X.mean(axis=0)
U, s, V = np.linalg.svd(X_centered)
c1 = V.T[:, 0]
c2 = V.T[:, 1]
The following Python code uses NumPy&#8217;s <b>svd()</b> function to obtain all the principal components of the
training set, then extracts the first two principal components
<p>
PCA assumes that the dataset is centered around the origin. As we will see, Scikit-Learn&#8217;s PCA classes take care of centering
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>X_centered <span style="color: #666666">=</span> X <span style="color: #666666">-</span> X<span style="color: #666666">.</span>mean(axis<span style="color: #666666">=0</span>)
U, s, V <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>svd(X_centered)
c1 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, <span style="color: #666666">0</span>]
c2 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, <span style="color: #666666">1</span>]
</pre></div>
<p>
PCA assumes that the dataset is centered around the origin. Scikit-Learn&#8217;s PCA classes take care of centering
the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don&#8217;t
forget to center the data first.
<p>
Once you have identified all the principal components, you can reduce the dimensionality of the dataset
down to d dimensions by projecting it onto the hyperplane defined by the first d principal components.
Selecting this hyperplane ensures that the projection will preserve as much variance as possible. For
example, in Figure 8-2 the 3D dataset is projected down to the 2D plane defined by the first two principal
components, preserving a large part of the dataset&#8217;s variance. As a result, the 2D projection looks very
much like the original 3D dataset.
down to \( d \) dimensions by projecting it onto the hyperplane defined by the first \( d \) principal components.
Selecting this hyperplane ensures that the projection will preserve as much variance as possible.
<p>
W2 = V.T[:, :2]
X2D = X_centered.dot(W2)
<p>
Scikit-Learn&#8217;s PCA class implements PCA using SVD decomposition just like we did before. The
following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note
that it automatically takes care of centering the data):
from sklearn.decomposition import PCA
pca = PCA(n_components = 2)
X2D = pca.fit_transform(X)
After fitting the PCA transformer to the dataset, you can access the principal components using the
components_ variable (note that it contains the PCs as horizontal vectors, so, for example, the first
principal component is equal to pca.components_.T[:, 0]).
<p>
Another very useful piece of information is the explained variance ratio of each principal component,
available via the explained_variance_ratio_ variable. It indicates the proportion of the dataset&#8217;s
variance that lies along the axis of each principal component. For example, let&#8217;s look at the explained
variance ratios of the first two components of the 3D dataset represented in Figure 8-2:
>>> print(pca.explained_variance_ratio_)
array([ 0.84248607, 0.14631839])
This tells you that 84.2% of the dataset&#8217;s variance lies along the first axis, and 14.6% lies along the
second axis. This leaves less than 1.2% for the third axis, so it is reasonable to assume that it probably
carries little information.
<p>
Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to
choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%).
Unless, of course, you are reducing dimensionality for data visualization &#8212; in that case you will
generally want to reduce the dimensionality down to 2 or 3.
The following code computes PCA without reducing dimensionality, then computes the minimum number
of dimensions required to preserve 95% of the training set&#8217;s variance:
pca = PCA()
pca.fit(X)
cumsum = np.cumsum(pca.explained_variance_ratio_)
d = np.argmax(cumsum >= 0.95) + 1
You could then set n_components=d and run PCA again. However, there is a much better option: instead
of specifying the number of principal components you want to preserve, you can set n_components to be
a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve:
pca = PCA(n_components=0.95)
X_reduced = pca.fit_transform(X)
<p>
Obviously after dimensionality reduction, the training set takes up much less space. For example, try
applying PCA to the MNIST dataset while preserving 95% of its variance. You should find that each
instance will have just over 150 features, instead of the original 784 features. So while most of the
variance is preserved, the dataset is now less than 20% of its original size! This is a reasonable
compression ratio, and you can see how this can speed up a classification algorithm (such as an SVM
classifier) tremendously.
It is also possible to decompress the reduced dataset back to 784 dimensions by applying the inverse
transformation of the PCA projection. Of course this won&#8217;t give you back the original data, since the
projection lost a bit of information (within the 5% variance that was dropped), but it will likely be quite
close to the original data. The mean squared distance between the original data and the reconstructed data
(compressed and then decompressed) is called the reconstruction error. For example, the following code
compresses the MNIST dataset down to 154 dimensions, then uses the inverse_transform() method to
decompress it back to 784 dimensions. Figure 8-9 shows a few digits from the original training set (on the
left), and the corresponding digits after compression and decompression. You can see that there is a slight
image quality loss, but the digits are still mostly intact.
pca = PCA(n_components = 154)
X_mnist_reduced = pca.fit_transform(X_mnist)
X_mnist_recovered = pca.inverse_transform(X_mnist_reduced)
Figure
<p>
Incremental PCA
One problem with the preceding implementation of PCA is that it requires the whole training set to fit in
memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have
been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch
at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new
instances arrive).
The following code splits the MNIST dataset into 100 mini-batches (using NumPy&#8217;s array_split()
function) and feeds them to Scikit-Learn&#8217;s IncrementalPCA class5 to reduce the dimensionality of the
MNIST dataset down to 154 dimensions (just like before). Note that you must call the partial_fit()
method with each mini-batch rather than the fit() method with the whole training set:
from sklearn.decomposition import IncrementalPCA
n_batches = 100
inc_pca = IncrementalPCA(n_components=154)
for X_batch in np.array_split(X_mnist, n_batches):
inc_pca.partial_fit(X_batch)
X_mnist_reduced = inc_pca.transform(X_mnist)
<p>
Alternatively, you can use NumPy&#8217;s memmap class, which allows you to manipulate a large array stored in
a binary file on disk as if it were entirely in memory; the class loads only the data it needs in memory,
when it needs it. Since the IncrementalPCA class uses only a small part of the array at any given time,
the memory usage remains under control. This makes it possible to call the usual fit() method, as you
can see in the following code:
X_mm = np.memmap(filename, dtype="float32", mode="readonly", shape=(m, n))
batch_size = m // n_batches
inc_pca = IncrementalPCA(n_components=154, batch_size=batch_size)
inc_pca.fit(X_mm)
<p>
Randomized PCA
Scikit-Learn offers yet another option to perform PCA, called Randomized PCA. This is a stochastic
algorithm that quickly finds an approximation of the first d principal components. Its computational
complexity is O(m &#215; d2) + O(d3), instead of O(m &#215; n2) + O(n3), so it is dramatically faster than the
previous algorithms when d is much smaller than n.
rnd_pca = PCA(n_components=154, svd_solver="randomized")
X_reduced = rnd_pca.fit_transform(X_mnist)
<p>
</div>
</div>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>W2 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, :<span style="color: #666666">2</span>]
X2D <span style="color: #666666">=</span> X_centered<span style="color: #666666">.</span>dot(W2)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -235,6 +157,10 @@ X_reduced = rnd_pca.fit_transform(X_mnist)
<li><a href="._DimRed-bs003.html">4</a></li>
<li><a href="._DimRed-bs004.html">5</a></li>
<li><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs003.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DimRed/html/._DimRed-bs003.html
+58 -28
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -87,33 +111,35 @@ end of tocinfo -->
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0003"></a>
<!-- !split -->
<!-- !split -->
<h2 id="___sec2" class="anchor">Kernel PCA </h2>
<div class="panel panel-default">
<div class="panel-body">
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
<h2 id="___sec2" class="anchor">PCA and scikit-learn </h2>
<p>
Kernel PCA
The kernel trick is a mathematical technique that implicitly maps instances into a
very high-dimensional space (called the feature space), enabling nonlinear classification and regression
with Support Vector Machines. Recall that a linear decision boundary in the high-dimensional feature
space corresponds to a complex nonlinear decision boundary in the original space.
It turns out that the same trick can be applied to PCA, making it possible to perform complex nonlinear
projections for dimensionality reduction. This is called Kernel PCA (kPCA). It is often good at
preserving clusters of instances after projection, or sometimes even unrolling datasets that lie close to a
twisted manifold.
For example, the following code uses Scikit-Learn&#8217;s KernelPCA class to perform kPCA with an
from sklearn.decomposition import KernelPCA
rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.04)
X_reduced = rbf_pca.fit_transform(X)
Figure 8-
Scikit-Learn&#8217;s PCA class implements PCA using SVD decomposition just like we did before. The
following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note
that it automatically takes care of centering the data):
<p>
</div>
</div>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.decomposition</span> <span style="color: #008000; font-weight: bold">import</span> PCA
pca <span style="color: #666666">=</span> PCA(n_components <span style="color: #666666">=</span> <span style="color: #666666">2</span>)
X2D <span style="color: #666666">=</span> pca<span style="color: #666666">.</span>fit_transform(X)
</pre></div>
<p>
After fitting the PCA transformer to the dataset, you can access the principal components using the
components variable (note that it contains the PCs as horizontal vectors, so, for example, the first
principal component is equal to
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pca<span style="color: #666666">.</span>components_<span style="color: #666666">.</span>T[:, <span style="color: #666666">0</span>])<span style="color: #666666">.</span>
</pre></div>
<p>
Another very useful piece of information is the explained variance ratio of each principal component,
available via the \( explained\_variance\_ratio \) variable. It indicates the proportion of the dataset&#8217;s
variance that lies along the axis of each principal component.
More material to come here.
<p>
<p>
@@ -126,6 +152,10 @@ Figure 8-
<li class="active"><a href="._DimRed-bs003.html">4</a></li>
<li><a href="._DimRed-bs004.html">5</a></li>
<li><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs004.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DimRed/html/._DimRed-bs004.html
+57 -14
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs005.html#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -89,16 +113,31 @@ end of tocinfo -->
<a name="part0004"></a>
<!-- !split -->
<h2 id="___sec3" class="anchor">LLE </h2>
<h2 id="___sec3" class="anchor">More on the PCA </h2>
Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to
choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%).
Unless, of course, you are reducing dimensionality for data visualization &#8212; in that case you will
generally want to reduce the dimensionality down to 2 or 3.
The following code computes PCA without reducing dimensionality, then computes the minimum number
of dimensions required to preserve 95% of the training set&#8217;s variance:
<p>
Locally Linear Embedding (LLE)8 is another very powerful nonlinear dimensionality reduction
(NLDR) technique. It is a Manifold Learning technique that does not rely on projections like the previous
algorithms. In a nutshell, LLE works by first measuring how each training instance linearly relates to its
closest neighbors (c.n.), and then looking for a low-dimensional representation of the training set where
these local relationships are best preserved (more details shortly). This makes it particularly good at
unrolling twisted manifolds, especially when there is not too much noise.
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pca <span style="color: #666666">=</span> PCA()
pca<span style="color: #666666">.</span>fit(X)
cumsum <span style="color: #666666">=</span> np<span style="color: #666666">.</span>cumsum(pca<span style="color: #666666">.</span>explained_variance_ratio_)
d <span style="color: #666666">=</span> np<span style="color: #666666">.</span>argmax(cumsum <span style="color: #666666">&gt;=</span> <span style="color: #666666">0.95</span>) <span style="color: #666666">+</span> <span style="color: #666666">1</span>
</pre></div>
<p>
You could then set \( n\_components=d \) and run PCA again. However, there is a much better option: instead
of specifying the number of principal components you want to preserve, you can set \( n\_components \) to be
a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve:
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>pca <span style="color: #666666">=</span> PCA(n_components<span style="color: #666666">=0.95</span>)
X_reduced <span style="color: #666666">=</span> pca<span style="color: #666666">.</span>fit_transform(X)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -110,6 +149,10 @@ unrolling twisted manifolds, especially when there is not too much noise.
<li><a href="._DimRed-bs003.html">4</a></li>
<li class="active"><a href="._DimRed-bs004.html">5</a></li>
<li><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs005.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/DimRed/html/._DimRed-bs005.html
+41 -23
View File
@@ -45,13 +45,33 @@ Automatically generated HTML file from DocOnce source
None,
'___sec0'),
('Principal Component Analysis', 2, None, '___sec1'),
('Kernel PCA', 2, None, '___sec2'),
('LLE', 2, None, '___sec3'),
('Other techniques', 2, None, '___sec4')]}
('PCA and scikit-learn', 2, None, '___sec2'),
('More on the PCA', 2, None, '___sec3'),
('Incremental PCA', 2, None, '___sec4'),
('Randomized PCA', 2, None, '___sec5'),
('Kernel PCA', 2, None, '___sec6'),
('LLE', 2, None, '___sec7'),
('Other techniques', 2, None, '___sec8')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
@@ -71,9 +91,13 @@ end of tocinfo -->
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._DimRed-bs001.html#___sec0" style="font-size: 80%;">Reducing the number of degrees of freedom, overarching view</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs002.html#___sec1" style="font-size: 80%;">Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Other techniques</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs003.html#___sec2" style="font-size: 80%;">PCA and scikit-learn</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs004.html#___sec3" style="font-size: 80%;">More on the PCA</a></li>
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Incremental PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs006.html#___sec5" style="font-size: 80%;">Randomized PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs007.html#___sec6" style="font-size: 80%;">Kernel PCA</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs008.html#___sec7" style="font-size: 80%;">LLE</a></li>
<!-- navigation toc: --> <li><a href="._DimRed-bs009.html#___sec8" style="font-size: 80%;">Other techniques</a></li>
</ul>
</li>
@@ -89,25 +113,14 @@ end of tocinfo -->
<a name="part0005"></a>
<!-- !split -->
<h2 id="___sec4" class="anchor">Other techniques </h2>
<h2 id="___sec4" class="anchor">Incremental PCA </h2>
One problem with the preceding implementation of PCA is that it requires the whole training set to fit in
memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have
been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch
at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new
instances arrive).
<p>
There are many other dimensionality reduction techniques, several of which are available in Scikit-Learn.
Here are some of the most popular:
Multidimensional Scaling (MDS) reduces dimensionality while trying to preserve the distances
between the instances (see Figure 8-13).
Isomap creates a graph by connecting each instance to its nearest neighbors, then reduces
dimensionality while trying to preserve the geodesic distances9 between the instances.
t-Distributed Stochastic Neighbor Embedding (t-SNE) reduces dimensionality while trying to keep
similar instances close and dissimilar instances apart. It is mostly used for visualization, in
particular to visualize clusters of instances in high-dimensional space (e.g., to visualize the MNIST
images in 2D).
Linear Discriminant Analysis (LDA) is actually a classification algorithm, but during training it
learns the most discriminative axes between the classes, and these axes can then be used to define a
hyperplane onto which to project the data. The benefit is that the projection will keep classes as far
apart as possible, so LDA is a good technique to reduce dimensionality before running another
classification algorithm such as an SVM classifier
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
@@ -118,6 +131,11 @@ classification algorithm such as an SVM classifier
<li><a href="._DimRed-bs003.html">4</a></li>
<li><a href="._DimRed-bs004.html">5</a></li>
<li class="active"><a href="._DimRed-bs005.html">6</a></li>
<li><a href="._DimRed-bs006.html">7</a></li>
<li><a href="._DimRed-bs007.html">8</a></li>
<li><a href="._DimRed-bs008.html">9</a></li>
<li><a href="._DimRed-bs009.html">10</a></li>
<li><a href="._DimRed-bs006.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/Regression/ipynb/Regression.ipynb
+22 -22
View File
File diff suppressed because one or more lines are too long
doc/src/DimRed/DimRed.do.txt
+47 -96
View File
@@ -13,8 +13,8 @@ solution, as we will see. This problem is often referred to as the curse of dime
Fortunately, in real-world problems, it is often possible to reduce the number of features considerably,
turning an intractable problem into a tractable one.
and we will go through three of the most popular dimensionality
reduction techniques: PCA, Kernel PCA, and LLE.
Here we will discuss some of the most popular dimensionality
reduction techniques: the principal component analysis PCA, Kernel PCA, and Locally Linear Embedding (LLE).
!eblock
@@ -26,126 +26,86 @@ reduction techniques: PCA, Kernel PCA, and LLE.
Principal Component Analysis (PCA) is by far the most popular dimensionality reduction algorithm.
First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it.
The following Python code uses NumPy’s svd() function to obtain all the principal components of the
training set, then extracts the first two PCs:
The following Python code uses NumPy’s _svd()_ function to obtain all the principal components of the
training set, then extracts the first two principal components
!bc pycod
X_centered = X - X.mean(axis=0)
U, s, V = np.linalg.svd(X_centered)
c1 = V.T[:, 0]
c2 = V.T[:, 1]
!ec
PCA assumes that the dataset is centered around the origin. As we will see, Scikit-Learn’s PCA classes take care of centering
PCA assumes that the dataset is centered around the origin. Scikit-Learn’s PCA classes take care of centering
the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don’t
forget to center the data first.
Once you have identified all the principal components, you can reduce the dimensionality of the dataset
down to d dimensions by projecting it onto the hyperplane defined by the first d principal components.
Selecting this hyperplane ensures that the projection will preserve as much variance as possible. For
example, in Figure 8-2 the 3D dataset is projected down to the 2D plane defined by the first two principal
components, preserving a large part of the dataset’s variance. As a result, the 2D projection looks very
much like the original 3D dataset.
down to $d$ dimensions by projecting it onto the hyperplane defined by the first $d$ principal components.
Selecting this hyperplane ensures that the projection will preserve as much variance as possible.
!bc pycod
W2 = V.T[:, :2]
X2D = X_centered.dot(W2)
!ec
!split
===== PCA and scikit-learn =====
Scikit-Learn’s PCA class implements PCA using SVD decomposition just like we did before. The
following code applies PCA to reduce the dimensionality of the dataset down to two dimensions (note
that it automatically takes care of centering the data):
!bc pycod
from sklearn.decomposition import PCA
pca = PCA(n_components = 2)
X2D = pca.fit_transform(X)
!ec
After fitting the PCA transformer to the dataset, you can access the principal components using the
components_ variable (note that it contains the PCs as horizontal vectors, so, for example, the first
principal component is equal to pca.components_.T[:, 0]).
components variable (note that it contains the PCs as horizontal vectors, so, for example, the first
principal component is equal to
!bc pycod
pca.components_.T[:, 0]).
!ec
Another very useful piece of information is the explained variance ratio of each principal component,
available via the explained_variance_ratio_ variable. It indicates the proportion of the dataset’s
variance that lies along the axis of each principal component. For example, let’s look at the explained
variance ratios of the first two components of the 3D dataset represented in Figure 8-2:
>>> print(pca.explained_variance_ratio_)
array([ 0.84248607, 0.14631839])
This tells you that 84.2% of the dataset’s variance lies along the first axis, and 14.6% lies along the
second axis. This leaves less than 1.2% for the third axis, so it is reasonable to assume that it probably
carries little information.
available via the $explained\_variance\_ratio$ variable. It indicates the proportion of the dataset’s
variance that lies along the axis of each principal component.
More material to come here.
!split
===== More on the PCA =====
Instead of arbitrarily choosing the number of dimensions to reduce down to, it is generally preferable to
choose the number of dimensions that add up to a sufficiently large portion of the variance (e.g., 95%).
Unless, of course, you are reducing dimensionality for data visualization — in that case you will
generally want to reduce the dimensionality down to 2 or 3.
The following code computes PCA without reducing dimensionality, then computes the minimum number
of dimensions required to preserve 95% of the training set’s variance:
!bc pycod
pca = PCA()
pca.fit(X)
cumsum = np.cumsum(pca.explained_variance_ratio_)
d = np.argmax(cumsum >= 0.95) + 1
You could then set n_components=d and run PCA again. However, there is a much better option: instead
of specifying the number of principal components you want to preserve, you can set n_components to be
!ec
You could then set $n\_components=d$ and run PCA again. However, there is a much better option: instead
of specifying the number of principal components you want to preserve, you can set $n\_components$ to be
a float between 0.0 and 1.0, indicating the ratio of variance you wish to preserve:
!bc pycod
pca = PCA(n_components=0.95)
X_reduced = pca.fit_transform(X)
!ec
Obviously after dimensionality reduction, the training set takes up much less space. For example, try
applying PCA to the MNIST dataset while preserving 95% of its variance. You should find that each
instance will have just over 150 features, instead of the original 784 features. So while most of the
variance is preserved, the dataset is now less than 20% of its original size! This is a reasonable
compression ratio, and you can see how this can speed up a classification algorithm (such as an SVM
classifier) tremendously.
It is also possible to decompress the reduced dataset back to 784 dimensions by applying the inverse
transformation of the PCA projection. Of course this won’t give you back the original data, since the
projection lost a bit of information (within the 5% variance that was dropped), but it will likely be quite
close to the original data. The mean squared distance between the original data and the reconstructed data
(compressed and then decompressed) is called the reconstruction error. For example, the following code
compresses the MNIST dataset down to 154 dimensions, then uses the inverse_transform() method to
decompress it back to 784 dimensions. Figure 8-9 shows a few digits from the original training set (on the
left), and the corresponding digits after compression and decompression. You can see that there is a slight
image quality loss, but the digits are still mostly intact.
pca = PCA(n_components = 154)
X_mnist_reduced = pca.fit_transform(X_mnist)
X_mnist_recovered = pca.inverse_transform(X_mnist_reduced)
Figure
Incremental PCA
!split
===== Incremental PCA =====
One problem with the preceding implementation of PCA is that it requires the whole training set to fit in
memory in order for the SVD algorithm to run. Fortunately, Incremental PCA (IPCA) algorithms have
been developed: you can split the training set into mini-batches and feed an IPCA algorithm one minibatch
at a time. This is useful for large training sets, and also to apply PCA online (i.e., on the fly, as new
instances arrive).
The following code splits the MNIST dataset into 100 mini-batches (using NumPy’s array_split()
function) and feeds them to Scikit-Learn’s IncrementalPCA class5 to reduce the dimensionality of the
MNIST dataset down to 154 dimensions (just like before). Note that you must call the partial_fit()
method with each mini-batch rather than the fit() method with the whole training set:
from sklearn.decomposition import IncrementalPCA
n_batches = 100
inc_pca = IncrementalPCA(n_components=154)
for X_batch in np.array_split(X_mnist, n_batches):
inc_pca.partial_fit(X_batch)
X_mnist_reduced = inc_pca.transform(X_mnist)
!split
===== Randomized PCA =====
Alternatively, you can use NumPy’s memmap class, which allows you to manipulate a large array stored in
a binary file on disk as if it were entirely in memory; the class loads only the data it needs in memory,
when it needs it. Since the IncrementalPCA class uses only a small part of the array at any given time,
the memory usage remains under control. This makes it possible to call the usual fit() method, as you
can see in the following code:
X_mm = np.memmap(filename, dtype="float32", mode="readonly", shape=(m, n))
batch_size = m // n_batches
inc_pca = IncrementalPCA(n_components=154, batch_size=batch_size)
inc_pca.fit(X_mm)
Randomized PCA
Scikit-Learn offers yet another option to perform PCA, called Randomized PCA. This is a stochastic
algorithm that quickly finds an approximation of the first d principal components. Its computational
complexity is O(m × d2) + O(d3), instead of O(m × n2) + O(n3), so it is dramatically faster than the
previous algorithms when d is much smaller than n.
rnd_pca = PCA(n_components=154, svd_solver="randomized")
X_reduced = rnd_pca.fit_transform(X_mnist)
complexity is $O(m \times d^2)+O(d^3)$, instead of $O(m \times n^2) + O(n^3)$, so it is dramatically faster than the
previous algorithms when $d$ is much smaller than $n$.
!eblock
@@ -155,7 +115,6 @@ X_reduced = rnd_pca.fit_transform(X_mnist)
===== Kernel PCA =====
!bblock
Kernel PCA
The kernel trick is a mathematical technique that implicitly maps instances into a
very high-dimensional space (called the feature space), enabling nonlinear classification and regression
with Support Vector Machines. Recall that a linear decision boundary in the high-dimensional feature
@@ -165,10 +124,11 @@ projections for dimensionality reduction. This is called Kernel PCA (kPCA). It i
preserving clusters of instances after projection, or sometimes even unrolling datasets that lie close to a
twisted manifold.
For example, the following code uses Scikit-Learn’s KernelPCA class to perform kPCA with an
!bc pycod
from sklearn.decomposition import KernelPCA
rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.04)
X_reduced = rbf_pca.fit_transform(X)
Figure 8-
!ec
!eblock
@@ -176,12 +136,11 @@ Figure 8-
!split
===== LLE =====
Locally Linear Embedding (LLE)8 is another very powerful nonlinear dimensionality reduction
Locally Linear Embedding (LLE) is another very powerful nonlinear dimensionality reduction
(NLDR) technique. It is a Manifold Learning technique that does not rely on projections like the previous
algorithms. In a nutshell, LLE works by first measuring how each training instance linearly relates to its
closest neighbors (c.n.), and then looking for a low-dimensional representation of the training set where
these local relationships are best preserved (more details shortly). This makes it particularly good at
unrolling twisted manifolds, especially when there is not too much noise.
these local relationships are best preserved (more details shortly).
@@ -190,17 +149,9 @@ unrolling twisted manifolds, especially when there is not too much noise.
There are many other dimensionality reduction techniques, several of which are available in Scikit-Learn.
Here are some of the most popular:
Multidimensional Scaling (MDS) reduces dimensionality while trying to preserve the distances
between the instances (see Figure 8-13).
Isomap creates a graph by connecting each instance to its nearest neighbors, then reduces
dimensionality while trying to preserve the geodesic distances9 between the instances.
t-Distributed Stochastic Neighbor Embedding (t-SNE) reduces dimensionality while trying to keep
similar instances close and dissimilar instances apart. It is mostly used for visualization, in
particular to visualize clusters of instances in high-dimensional space (e.g., to visualize the MNIST
images in 2D).
Linear Discriminant Analysis (LDA) is actually a classification algorithm, but during training it
learns the most discriminative axes between the classes, and these axes can then be used to define a
hyperplane onto which to project the data. The benefit is that the projection will keep classes as far
apart as possible, so LDA is a good technique to reduce dimensionality before running another
classification algorithm such as an SVM classifier
* _Multidimensional Scaling (MDS)_ reduces dimensionality while trying to preserve the distances between the instances.
* _Isomap_ creates a graph by connecting each instance to its nearest neighbors, then reduces dimensionality while trying to preserve the geodesic distances between the instances.
* _t-Distributed Stochastic Neighbor Embedding_ (t-SNE) reduces dimensionality while trying to keep similar instances close and dissimilar instances apart. It is mostly used for visualization, in particular to visualize clusters of instances in high-dimensional space (e.g., to visualize the MNIST images in 2D).
* Linear Discriminant Analysis (LDA) is actually a classification algorithm, but during training it learns the most discriminative axes between the classes, and these axes can then be used to define a hyperplane onto which to project the data. The benefit is that the projection will keep classes as far apart as possible, so LDA is a good technique to reduce dimensionality before running another classification algorithm such as a Support Vector Machine (SVM) classifier discussed in the SVM lectures.
doc/web/course.do.txt
+10 -10
View File
@@ -250,23 +250,23 @@ Acronyms for textbooks and references to chapter
|----------------------------------------------------------------------------------------------------------------------------|
| Week 38 | Optimization methods | Exercises and project 1 | HTF chapter 5 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" | Work on project 1, deadline October 1|
|----------------------------------------------------------------------------------------------------------------------------|
| Week 39 | Statistics, Bayesian statistics | Project 1 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Bayesian/html/Bayesian-bs.html" | Work on Project 1|
| Week 39 | Logistic regression and optimization | Project 1 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Bayesian/html/Bayesian-bs.html" | Work on Project 1|
|----------------------------------------------------------------------------------------------------------------------------|
| Week 40 | Statistics, Monte Carlo and Randow walks | Presentation of project 2, deadline November 5 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" | Deadline project 1, October 1|
| Week 40 | Neural Networks | Presentation of project 2, deadline November 5 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" | Deadline project 1, October 1|
|----------------------------------------------------------------------------------------------------------------------------|
| Week 41 | Statistics, Monte Carlo, Gibbs and Metropolis sampling | Project 2 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" | Work on project 2 |
| Week 41 | Neural Networks | Project 2 | "Lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" | Work on project 2 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 42 | Neural networks | Project 2 | HTF chapter 11 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
| Week 42 | Neural Networks | Project 2 | HTF chapter 11 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 43 | Neural networks | Project 2 | HTF chapter 11 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
| Week 43 | Dimensionality reduction and support vector machines | Project 2 | HTF chapters 3 and 12 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 44 | Neural networks | Project 2 | HTF chapter 11 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
| Week 44 | SVM and tree and forest models | Project 2 | HTF chapter 9 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" | Work on project 2 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 45 | Support Vector Machines | Presentation and discussion of project 3 | HTF chapter 12 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/svm/html/svm-bs.html" | Deadline project 2 November 5 |
| Week 45 | Unsupervised learning, Boltzmann machines | Presentation and discussion of project 3 | HTF chapter 14 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/BM/html/BM-bs.html" | Deadline project 2 November 5 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 46 | Decision trees | Project 3 | HTF chapter 9 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/DecisionTrees/html/DecisionTrees-bs.html" | Work on project 3 |
| Week 46 | Bayesian statitics | Project 3 | TBA | Work on project 3 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 47 | Unsupervised learning, Boltzmann machines | Project 3 | HTF chapter 14 and "lecture notes":"https://compphysics.github.io/MachineLearning/doc/pub/BM/html/BM-bs.html" | Work on project 3 |
| Week 47 | Bayesian statistics | Project 3 | TBA | Work on project 3 |
|----------------------------------------------------------------------------------------------------------------------------|
| Week 48 | Unsupervised learning, summary of course and final workshop | Project 3 | Lecture notes | Final workshop with presentation of project 3 |
| Week 48 | Summary of course and final workshop | Project 3 | Lecture notes | Final workshop with presentation of project 3 on November 30|
|----------------------------------------------------------------------------------------------------------------------------|
doc/web/course.html
+16 -16
View File
@@ -836,24 +836,24 @@ Acronyms for textbooks and references to chapter
<table border="1">
<thead>
<tr><th align="center">Week and days</th> <th align="center"> Topics to be covered </th> <th align="center"> Projects, exercises and deadlines </th> <th align="center"> Reading assignments </th> <th align="center"> Lab activities </th> </tr>
<tr><th align="center">Week and days</th> <th align="center"> Topics to be covered </th> <th align="center"> Projects, exercises and deadlines </th> <th align="center"> Reading assignments </th> <th align="center"> Lab activities </th> </tr>
</thead>
<tbody>
<tr><td align="center"> Week 34 </td> <td align="center"> Introduction and regression analysis </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapters 1-3 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> No lab first week </td> </tr>
<tr><td align="center"> Week 35 </td> <td align="center"> Regression analysis </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapter 3 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Introduction to Git, GitHub and Python software, Python technicalities and work on exercises </td> </tr>
<tr><td align="center"> Week 36 </td> <td align="center"> Regression analysis and nearest neighbors </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapters 3, 4 and 13 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on exercises </td> </tr>
<tr><td align="center"> Week 37 </td> <td align="center"> Classification and logistic regression </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/Projects/2018/Project1/html/Project1-bs.html" target="_self">Presentation of Project 1, deadline October 1</a> </td> <td align="center"> HTF chapter 4 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 1 </td> </tr>
<tr><td align="center"> Week 38 </td> <td align="center"> Optimization methods </td> <td align="center"> Exercises and project 1 </td> <td align="center"> HTF chapter 5 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 1, deadline October 1 </td> </tr>
<tr><td align="center"> Week 39 </td> <td align="center"> Statistics, Bayesian statistics </td> <td align="center"> Project 1 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Bayesian/html/Bayesian-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Work on Project 1 </td> </tr>
<tr><td align="center"> Week 40 </td> <td align="center"> Statistics, Monte Carlo and Randow walks </td> <td align="center"> Presentation of project 2, deadline November 5 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Deadline project 1, October 1 </td> </tr>
<tr><td align="center"> Week 41 </td> <td align="center"> Statistics, Monte Carlo, Gibbs and Metropolis sampling </td> <td align="center"> Project 2 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 42 </td> <td align="center"> Neural networks </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapter 11 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 43 </td> <td align="center"> Neural networks </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapter 11 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 44 </td> <td align="center"> Neural networks </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapter 11 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 45 </td> <td align="center"> Support Vector Machines </td> <td align="center"> Presentation and discussion of project 3 </td> <td align="center"> HTF chapter 12 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/svm/html/svm-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Deadline project 2 November 5 </td> </tr>
<tr><td align="center"> Week 46 </td> <td align="center"> Decision trees </td> <td align="center"> Project 3 </td> <td align="center"> HTF chapter 9 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/DecisionTrees/html/DecisionTrees-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 3 </td> </tr>
<tr><td align="center"> Week 47 </td> <td align="center"> Unsupervised learning, Boltzmann machines </td> <td align="center"> Project 3 </td> <td align="center"> HTF chapter 14 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/BM/html/BM-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 3 </td> </tr>
<tr><td align="center"> Week 48 </td> <td align="center"> Unsupervised learning, summary of course and final workshop </td> <td align="center"> Project 3 </td> <td align="center"> Lecture notes </td> <td align="center"> Final workshop with presentation of project 3 </td> </tr>
<tr><td align="center"> Week 34 </td> <td align="center"> Introduction and regression analysis </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapters 1-3 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> No lab first week </td> </tr>
<tr><td align="center"> Week 35 </td> <td align="center"> Regression analysis </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapter 3 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Introduction to Git, GitHub and Python software, Python technicalities and work on exercises </td> </tr>
<tr><td align="center"> Week 36 </td> <td align="center"> Regression analysis and nearest neighbors </td> <td align="center"> Exercises TBD </td> <td align="center"> HTF chapters 3, 4 and 13 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on exercises </td> </tr>
<tr><td align="center"> Week 37 </td> <td align="center"> Classification and logistic regression </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/Projects/2018/Project1/html/Project1-bs.html" target="_self">Presentation of Project 1, deadline October 1</a> </td> <td align="center"> HTF chapter 4 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 1 </td> </tr>
<tr><td align="center"> Week 38 </td> <td align="center"> Optimization methods </td> <td align="center"> Exercises and project 1 </td> <td align="center"> HTF chapter 5 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 1, deadline October 1 </td> </tr>
<tr><td align="center"> Week 39 </td> <td align="center"> Logistic regression and optimization </td> <td align="center"> Project 1 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Bayesian/html/Bayesian-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Work on Project 1 </td> </tr>
<tr><td align="center"> Week 40 </td> <td align="center"> Neural Networks </td> <td align="center"> Presentation of project 2, deadline November 5 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Deadline project 1, October 1 </td> </tr>
<tr><td align="center"> Week 41 </td> <td align="center"> Neural Networks </td> <td align="center"> Project 2 </td> <td align="center"> <a href="https://compphysics.github.io/MachineLearning/doc/pub/Statistics/html/Statistics-bs.html" target="_self">Lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 42 </td> <td align="center"> Neural Networks </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapter 11 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 43 </td> <td align="center"> Dimensionality reduction and support vector machines </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapters 3 and 12 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 44 </td> <td align="center"> SVM and tree and forest models </td> <td align="center"> Project 2 </td> <td align="center"> HTF chapter 9 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/NeuralNet/html/NeuralNet-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Work on project 2 </td> </tr>
<tr><td align="center"> Week 45 </td> <td align="center"> Unsupervised learning, Boltzmann machines </td> <td align="center"> Presentation and discussion of project 3 </td> <td align="center"> HTF chapter 14 and <a href="https://compphysics.github.io/MachineLearning/doc/pub/BM/html/BM-bs.html" target="_self">lecture notes</a> </td> <td align="center"> Deadline project 2 November 5 </td> </tr>
<tr><td align="center"> Week 46 </td> <td align="center"> Bayesian statitics </td> <td align="center"> Project 3 </td> <td align="center"> TBA </td> <td align="center"> Work on project 3 </td> </tr>
<tr><td align="center"> Week 47 </td> <td align="center"> Bayesian statistics </td> <td align="center"> Project 3 </td> <td align="center"> TBA </td> <td align="center"> Work on project 3 </td> </tr>
<tr><td align="center"> Week 48 </td> <td align="center"> Summary of course and final workshop </td> <td align="center"> Project 3 </td> <td align="center"> Lecture notes </td> <td align="center"> Final workshop with presentation of project 3 on November 30 </td> </tr>
</tbody>
</table>