Files
FYS-STK4155/doc/pub/svm/html/svm-reveal.html
T
2018-11-02 04:49:53 +01:00

451 lines
22 KiB
HTML

\
<!DOCTYPE html>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
<meta name="description" content="Data Analysis and Machine Learning: Support Vector Machines">
<title>Data Analysis and Machine Learning: Support Vector Machines</title>
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
<link rel="stylesheet" href="reveal.js/css/reveal.css">
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
<!--
<link rel="stylesheet" href="reveal.js/css/reveal.css">
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
-->
<!-- For syntax highlighting -->
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
<!-- Printing and PDF exports -->
<script>
var link = document.createElement( 'link' );
link.rel = 'stylesheet';
link.type = 'text/css';
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
document.getElementsByTagName( 'head' )[0].appendChild( link );
</script>
<style type="text/css">
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
.reveal .alert-text-small { font-size: 80%; }
.reveal .alert-text-large { font-size: 130%; }
.reveal .alert-text-normal { font-size: 90%; }
.reveal .alert {
padding:8px 35px 8px 14px; margin-bottom:18px;
text-shadow:0 1px 0 rgba(255,255,255,0.5);
border:5px solid #bababa;
-webkit-border-radius: 14px; -moz-border-radius: 14px;
border-radius:14px;
background-position: 10px 10px;
background-repeat: no-repeat;
background-size: 38px;
padding-left: 30px; /* 55px; if icon */
}
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
/*.reveal .alert li {margin-top: 1em}*/
.reveal .alert-block p+p {margin-top:5px}
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
</style>
<!-- Styles for table layout of slides -->
<style type="text/css">
td.padding {
padding-top:20px;
padding-bottom:20px;
padding-right:50px;
padding-left:50px;
}
</style>
</head>
<body>
<div class="reveal">
<!-- Any section element inside the <div class="slides"> container
is displayed as a slide -->
<div class="slides">
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<section>
<!-- ------------------- main content ---------------------- -->
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Support Vector Machines</h1></center> <!-- document title -->
<p>
<!-- author(s): Morten Hjorth-Jensen -->
<center>
<b>Morten Hjorth-Jensen</b> [1, 2]
</center>
<p>&nbsp;<br>
<!-- institution(s) -->
<center>[1] <b>Department of Physics, University of Oslo</b></center>
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>&nbsp;<br>
<center><h4>Nov 2, 2018</h4></center> <!-- date -->
<br>
<p>
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
</section>
<section>
<h2 id="___sec0">Support Vector Machines, overarching aims </h2>
<p>
A Support Vector Machine (SVM) is a very powerful and versatile
Machine Learning model, capable of performing linear or nonlinear
classification, regression, and even outlier detection. It is one of
the most popular models in Machine Learning, and anyone interested in
Machine Learning should have it in their toolbox. SVMs are
particularly well suited for classification of complex but small-sized or
medium-sized datasets.
<p>
The basic mathematics relies on the definition of hyperplanes and the definition of a <b>margin</b> which separates
classes (in case of classification problems) of variables. It is also used for regression problems.
<p>
With SVMs we distinguish between hard margin and soft margins. The latter introduces a so-called softening parameter to be discussed below.
We distringuish also between linearn and non-linear approaches.
<p>
<b>These notes will be updated shortly with more material</b>
</section>
<section>
<h2 id="___sec1">Strength and weakness </h2>
When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
</section>
<section>
<h2 id="___sec2">Examples with kernels </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> set_matplotlib_formats, display
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">mglearn</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">cycler</span> <span style="color: #8B008B; font-weight: bold">import</span> cycler
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_blobs
X, y = make_blobs(centers=<span style="color: #B452CD">4</span>, random_state=<span style="color: #B452CD">8</span>)
y = y % <span style="color: #B452CD">2</span>
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
plt.xlabel(<span style="color: #CD5555">&quot;Feature 0&quot;</span>)
plt.ylabel(<span style="color: #CD5555">&quot;Feature 1&quot;</span>)
plt.show()
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
linear_svm = LinearSVC().fit(X, y)
mglearn.plots.plot_2d_separator(linear_svm, X)
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
plt.xlabel(<span style="color: #CD5555">&quot;Feature 0&quot;</span>)
plt.ylabel(<span style="color: #CD5555">&quot;Feature 1&quot;</span>)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># add the squared first feature</span>
X_new = np.hstack([X, X[:, <span style="color: #B452CD">1</span>:] ** <span style="color: #B452CD">2</span>])
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D, axes3d
figure = plt.figure()
<span style="color: #228B22"># visualize in 3D</span>
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
<span style="color: #228B22"># plot first all the points with y==0, then all with y == 1</span>
mask = y == <span style="color: #B452CD">0</span>
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">&#39;b&#39;</span>,
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">&#39;k&#39;</span>)
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">&#39;r&#39;</span>, marker=<span style="color: #CD5555">&#39;^&#39;</span>,
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">&#39;k&#39;</span>)
ax.set_xlabel(<span style="color: #CD5555">&quot;feature0&quot;</span>)
ax.set_ylabel(<span style="color: #CD5555">&quot;feature1&quot;</span>)
ax.set_zlabel(<span style="color: #CD5555">&quot;feature1 ** 2&quot;</span>)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>linear_svm_3d = LinearSVC().fit(X_new, y)
coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
<span style="color: #228B22"># show linear decision boundary</span>
figure = plt.figure()
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
xx = np.linspace(X_new[:, <span style="color: #B452CD">0</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">0</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
yy = np.linspace(X_new[:, <span style="color: #B452CD">1</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">1</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
XX, YY = np.meshgrid(xx, yy)
ZZ = (coef[<span style="color: #B452CD">0</span>] * XX + coef[<span style="color: #B452CD">1</span>] * YY + intercept) / -coef[<span style="color: #B452CD">2</span>]
ax.plot_surface(XX, YY, ZZ, rstride=<span style="color: #B452CD">8</span>, cstride=<span style="color: #B452CD">8</span>, alpha=<span style="color: #B452CD">0.3</span>)
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">&#39;b&#39;</span>,
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">&#39;k&#39;</span>)
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">&#39;r&#39;</span>, marker=<span style="color: #CD5555">&#39;^&#39;</span>,
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">&#39;k&#39;</span>)
ax.set_xlabel(<span style="color: #CD5555">&quot;feature0&quot;</span>)
ax.set_ylabel(<span style="color: #CD5555">&quot;feature1&quot;</span>)
ax.set_zlabel(<span style="color: #CD5555">&quot;feature1 ** 2&quot;</span>)
ZZ = YY ** <span style="color: #B452CD">2</span>
dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), <span style="color: #B452CD">0</span>, dec.max()],
cmap=mglearn.cm2, alpha=<span style="color: #B452CD">0.5</span>)
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
plt.xlabel(<span style="color: #CD5555">&quot;Feature 0&quot;</span>)
plt.ylabel(<span style="color: #CD5555">&quot;Feature 1&quot;</span>)
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
X, y = mglearn.tools.make_handcrafted_dataset()
svm = SVC(kernel=<span style="color: #CD5555">&#39;rbf&#39;</span>, C=<span style="color: #B452CD">10</span>, gamma=<span style="color: #B452CD">0.1</span>).fit(X, y)
mglearn.plots.plot_2d_separator(svm, X, eps=.<span style="color: #B452CD">5</span>)
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
<span style="color: #228B22"># plot support vectors</span>
sv = svm.support_vectors_
<span style="color: #228B22"># class labels of support vectors are given by the sign of the dual coefficients</span>
sv_labels = svm.dual_coef_.ravel() &gt; <span style="color: #B452CD">0</span>
mglearn.discrete_scatter(sv[:, <span style="color: #B452CD">0</span>], sv[:, <span style="color: #B452CD">1</span>], sv_labels, s=<span style="color: #B452CD">15</span>, markeredgewidth=<span style="color: #B452CD">3</span>)
plt.xlabel(<span style="color: #CD5555">&quot;Feature 0&quot;</span>)
plt.ylabel(<span style="color: #CD5555">&quot;Feature 1&quot;</span>)
fig, axes = plt.subplots(<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, figsize=(<span style="color: #B452CD">15</span>, <span style="color: #B452CD">10</span>))
<span style="color: #8B008B; font-weight: bold">for</span> ax, C <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(axes, [-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">3</span>]):
<span style="color: #8B008B; font-weight: bold">for</span> a, gamma <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(ax, <span style="color: #658b00">range</span>(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">2</span>)):
mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
axes[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>].legend([<span style="color: #CD5555">&quot;class 0&quot;</span>, <span style="color: #CD5555">&quot;class 1&quot;</span>, <span style="color: #CD5555">&quot;sv class 0&quot;</span>, <span style="color: #CD5555">&quot;sv class 1&quot;</span>],
ncol=<span style="color: #B452CD">4</span>, loc=(.<span style="color: #B452CD">9</span>, <span style="color: #B452CD">1.2</span>))
</pre></div>
</section>
</div> <!-- class="slides" -->
</div> <!-- class="reveal" -->
<script src="reveal.js/lib/js/head.min.js"></script>
<script src="reveal.js/js/reveal.js"></script>
<script>
// Full list of configuration options available here:
// https://github.com/hakimel/reveal.js#configuration
Reveal.initialize({
// Display navigation controls in the bottom right corner
controls: true,
// Display progress bar (below the horiz. slider)
progress: true,
// Display the page number of the current slide
slideNumber: true,
// Push each slide change to the browser history
history: false,
// Enable keyboard shortcuts for navigation
keyboard: true,
// Enable the slide overview mode
overview: true,
// Vertical centering of slides
//center: true,
center: false,
// Enables touch navigation on devices with touch input
touch: true,
// Loop the presentation
loop: false,
// Change the presentation direction to be RTL
rtl: false,
// Turns fragments on and off globally
fragments: true,
// Flags if the presentation is running in an embedded mode,
// i.e. contained within a limited portion of the screen
embedded: false,
// Number of milliseconds between automatically proceeding to the
// next slide, disabled when set to 0, this value can be overwritten
// by using a data-autoslide attribute on your slides
autoSlide: 0,
// Stop auto-sliding after user input
autoSlideStoppable: true,
// Enable slide navigation via mouse wheel
mouseWheel: false,
// Hides the address bar on mobile devices
hideAddressBar: true,
// Opens links in an iframe preview overlay
previewLinks: false,
// Transition style
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
// Transition speed
transitionSpeed: 'default', // default/fast/slow
// Transition style for full page slide backgrounds
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
// Number of slides away from the current that are visible
viewDistance: 3,
// Parallax background image
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
// Parallax background size
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
});
Reveal.initialize({
dependencies: [
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
// Interpret Markdown in <section> elements
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
// Syntax highlight for <code> elements
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
// Zoom in and out with Alt+click
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
// Speaker notes
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
// Remote control your reveal.js presentation using a touch device
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
// MathJax
//{ src: 'reveal.js/plugin/math/math.js', async: true }
]
});
Reveal.initialize({
// The "normal" size of the presentation, aspect ratio will be preserved
// when the presentation is scaled to fit different resolutions. Can be
// specified using percentage units.
width: 1170, // original: 960,
height: 700,
// Factor of the display size that should remain empty around the content
margin: 0.1,
// Bounds for smallest/largest possible scale to apply to content
minScale: 0.2,
maxScale: 1.0
});
</script>
<!-- begin footer logo
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
<img src="somelogo.png">
</div>
end footer logo -->
</body>
</html>