sum update
This commit is contained in:
mhjensen
+118
-88
@@ -41,14 +41,28 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Support Vector Machines, overarching aims', 2, None, '___sec0'),
|
||||
('SVMs, basics with scikit-learn', 2, None, '___sec1'),
|
||||
('SVMs, adding polynomial features', 2, None, '___sec2'),
|
||||
('SVMs, polynomials and kernels', 2, None, '___sec3'),
|
||||
('SVMs, regression', 2, None, '___sec4')]}
|
||||
('Strength and weakness', 2, None, '___sec1'),
|
||||
('Examples with kernels', 2, None, '___sec2')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "AMS" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -67,10 +81,8 @@ end of tocinfo -->
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="#___sec0" style="font-size: 80%;">Support Vector Machines, overarching aims</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">SVMs, basics with scikit-learn</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">SVMs, adding polynomial features</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">SVMs, polynomials and kernels</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">SVMs, regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">Strength and weakness</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Examples with kernels</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -104,7 +116,7 @@ end of tocinfo -->
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Nov 1, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- potential-jumbotron-button -->
|
||||
@@ -137,106 +149,124 @@ We distringuish also between linearn and non-linear approaches.
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec1" class="anchor">SVMs, basics with scikit-learn </h2>
|
||||
<h2 id="___sec1" class="anchor">Strength and weakness </h2>
|
||||
When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
|
||||
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
|
||||
|
||||
<p>
|
||||
The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
|
||||
model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
|
||||
Iris-Virginica flowers.
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec2" class="anchor">Examples with kernels </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> set_matplotlib_formats, display
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">mglearn</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">cycler</span> <span style="color: #008000; font-weight: bold">import</span> cycler
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVC
|
||||
iris <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_iris()
|
||||
X <span style="color: #666666">=</span> iris[<span style="color: #BA2121">"data"</span>][:, (<span style="color: #666666">2</span>, <span style="color: #666666">3</span>)] <span style="color: #408080; font-style: italic"># petal length, petal width</span>
|
||||
y <span style="color: #666666">=</span> (iris[<span style="color: #BA2121">"target"</span>] <span style="color: #666666">==</span> <span style="color: #666666">2</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float64) <span style="color: #408080; font-style: italic"># Iris-Virginica</span>
|
||||
svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"linear_svc"</span>, LinearSVC(C<span style="color: #666666">=1</span>, loss<span style="color: #666666">=</span><span style="color: #BA2121">"hinge"</span>)),
|
||||
))
|
||||
svm_clf<span style="color: #666666">.</span>fit(X_scaled, y)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_blobs
|
||||
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_blobs(centers<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=8</span>)
|
||||
y <span style="color: #666666">=</span> y <span style="color: #666666">%</span> <span style="color: #666666">2</span>
|
||||
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVC
|
||||
linear_svm <span style="color: #666666">=</span> LinearSVC()<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_2d_separator(linear_svm, X)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Alternatively, you could use the SVC class, using <b>SVC(kernel="linear", C=1)</b>, but it is much slower,
|
||||
especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
|
||||
class, with <b>SGDClassifier(loss="hinge", alpha=1/(m*C))</b>. This applies regular Stochastic
|
||||
Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
|
||||
LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
|
||||
training), or to handle online classification tasks.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec2" class="anchor">SVMs, adding polynomial features </h2>
|
||||
|
||||
<p>
|
||||
Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
|
||||
are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
|
||||
features, such as polynomial features. In some cases this can result in a linearly
|
||||
separable dataset.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
polynomial_svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"poly_features"</span>, PolynomialFeatures(degree<span style="color: #666666">=3</span>)),
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"svm_clf"</span>, LinearSVC(C<span style="color: #666666">=10</span>, loss<span style="color: #666666">=</span><span style="color: #BA2121">"hinge"</span>))
|
||||
))
|
||||
polynomial_svm_clf<span style="color: #666666">.</span>fit(X, y)
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># add the squared first feature</span>
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>hstack([X, X[:, <span style="color: #666666">1</span>:] <span style="color: #666666">**</span> <span style="color: #666666">2</span>])
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">mpl_toolkits.mplot3d</span> <span style="color: #008000; font-weight: bold">import</span> Axes3D, axes3d
|
||||
figure <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
<span style="color: #408080; font-style: italic"># visualize in 3D</span>
|
||||
ax <span style="color: #666666">=</span> Axes3D(figure, elev<span style="color: #666666">=-152</span>, azim<span style="color: #666666">=-26</span>)
|
||||
<span style="color: #408080; font-style: italic"># plot first all the points with y==0, then all with y == 1</span>
|
||||
mask <span style="color: #666666">=</span> y <span style="color: #666666">==</span> <span style="color: #666666">0</span>
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[mask, <span style="color: #666666">0</span>], X_new[mask, <span style="color: #666666">1</span>], X_new[mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'b'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">0</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">1</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'r'</span>, marker<span style="color: #666666">=</span><span style="color: #BA2121">'^'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"feature0"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"feature1"</span>)
|
||||
ax<span style="color: #666666">.</span>set_zlabel(<span style="color: #BA2121">"feature1 ** 2"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec3" class="anchor">SVMs, polynomials and kernels </h2>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>linear_svm_3d <span style="color: #666666">=</span> LinearSVC()<span style="color: #666666">.</span>fit(X_new, y)
|
||||
coef, intercept <span style="color: #666666">=</span> linear_svm_3d<span style="color: #666666">.</span>coef_<span style="color: #666666">.</span>ravel(), linear_svm_3d<span style="color: #666666">.</span>intercept_
|
||||
|
||||
<p>
|
||||
Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
|
||||
algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
|
||||
and with a high polynomial degree it creates a huge number of features, making the model too slow.
|
||||
Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
|
||||
kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
|
||||
polynomial features, even with very high-degree polynomials, without actually having to add them. So
|
||||
there is no combinatorial explosion of the number of features since you don’t actually add any features.
|
||||
This trick is implemented by the SVC class.
|
||||
<span style="color: #408080; font-style: italic"># show linear decision boundary</span>
|
||||
figure <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> Axes3D(figure, elev<span style="color: #666666">=-152</span>, azim<span style="color: #666666">=-26</span>)
|
||||
xx <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(X_new[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">2</span>, X_new[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">2</span>, <span style="color: #666666">50</span>)
|
||||
yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(X_new[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">2</span>, X_new[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">2</span>, <span style="color: #666666">50</span>)
|
||||
|
||||
XX, YY <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(xx, yy)
|
||||
ZZ <span style="color: #666666">=</span> (coef[<span style="color: #666666">0</span>] <span style="color: #666666">*</span> XX <span style="color: #666666">+</span> coef[<span style="color: #666666">1</span>] <span style="color: #666666">*</span> YY <span style="color: #666666">+</span> intercept) <span style="color: #666666">/</span> <span style="color: #666666">-</span>coef[<span style="color: #666666">2</span>]
|
||||
ax<span style="color: #666666">.</span>plot_surface(XX, YY, ZZ, rstride<span style="color: #666666">=8</span>, cstride<span style="color: #666666">=8</span>, alpha<span style="color: #666666">=0.3</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[mask, <span style="color: #666666">0</span>], X_new[mask, <span style="color: #666666">1</span>], X_new[mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'b'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">0</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">1</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'r'</span>, marker<span style="color: #666666">=</span><span style="color: #BA2121">'^'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"feature0"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"feature1"</span>)
|
||||
ax<span style="color: #666666">.</span>set_zlabel(<span style="color: #BA2121">"feature1 ** 2"</span>)
|
||||
|
||||
ZZ <span style="color: #666666">=</span> YY <span style="color: #666666">**</span> <span style="color: #666666">2</span>
|
||||
dec <span style="color: #666666">=</span> linear_svm_3d<span style="color: #666666">.</span>decision_function(np<span style="color: #666666">.</span>c_[XX<span style="color: #666666">.</span>ravel(), YY<span style="color: #666666">.</span>ravel(), ZZ<span style="color: #666666">.</span>ravel()])
|
||||
plt<span style="color: #666666">.</span>contourf(XX, YY, dec<span style="color: #666666">.</span>reshape(XX<span style="color: #666666">.</span>shape), levels<span style="color: #666666">=</span>[dec<span style="color: #666666">.</span>min(), <span style="color: #666666">0</span>, dec<span style="color: #666666">.</span>max()],
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, alpha<span style="color: #666666">=0.5</span>)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
poly_kernel_svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"svm_clf"</span>, SVC(kernel<span style="color: #666666">=</span><span style="color: #BA2121">"poly"</span>, degree<span style="color: #666666">=3</span>, coef0<span style="color: #666666">=1</span>, C<span style="color: #666666">=5</span>))
|
||||
))
|
||||
poly_kernel_svm_clf<span style="color: #666666">.</span>fit(X, y)
|
||||
X, y <span style="color: #666666">=</span> mglearn<span style="color: #666666">.</span>tools<span style="color: #666666">.</span>make_handcrafted_dataset()
|
||||
svm <span style="color: #666666">=</span> SVC(kernel<span style="color: #666666">=</span><span style="color: #BA2121">'rbf'</span>, C<span style="color: #666666">=10</span>, gamma<span style="color: #666666">=0.1</span>)<span style="color: #666666">.</span>fit(X, y)
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_2d_separator(svm, X, eps<span style="color: #666666">=.5</span>)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
<span style="color: #408080; font-style: italic"># plot support vectors</span>
|
||||
sv <span style="color: #666666">=</span> svm<span style="color: #666666">.</span>support_vectors_
|
||||
<span style="color: #408080; font-style: italic"># class labels of support vectors are given by the sign of the dual coefficients</span>
|
||||
sv_labels <span style="color: #666666">=</span> svm<span style="color: #666666">.</span>dual_coef_<span style="color: #666666">.</span>ravel() <span style="color: #666666">></span> <span style="color: #666666">0</span>
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(sv[:, <span style="color: #666666">0</span>], sv[:, <span style="color: #666666">1</span>], sv_labels, s<span style="color: #666666">=15</span>, markeredgewidth<span style="color: #666666">=3</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
|
||||
fig, axes <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(<span style="color: #666666">3</span>, <span style="color: #666666">3</span>, figsize<span style="color: #666666">=</span>(<span style="color: #666666">15</span>, <span style="color: #666666">10</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> ax, C <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(axes, [<span style="color: #666666">-1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>]):
|
||||
<span style="color: #008000; font-weight: bold">for</span> a, gamma <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(ax, <span style="color: #008000">range</span>(<span style="color: #666666">-1</span>, <span style="color: #666666">2</span>)):
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_svm(log_C<span style="color: #666666">=</span>C, log_gamma<span style="color: #666666">=</span>gamma, ax<span style="color: #666666">=</span>a)
|
||||
|
||||
axes[<span style="color: #666666">0</span>, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>legend([<span style="color: #BA2121">"class 0"</span>, <span style="color: #BA2121">"class 1"</span>, <span style="color: #BA2121">"sv class 0"</span>, <span style="color: #BA2121">"sv class 1"</span>],
|
||||
ncol<span style="color: #666666">=4</span>, loc<span style="color: #666666">=</span>(<span style="color: #666666">.9</span>, <span style="color: #666666">1.2</span>))
|
||||
</pre></div>
|
||||
<p>
|
||||
This code trains an SVM classifier using a 3rd-degree polynomial kernel.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec4" class="anchor">SVMs, regression </h2>
|
||||
|
||||
<p>
|
||||
You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
|
||||
shows how to use the regression option (more material to come)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVR
|
||||
svm_reg <span style="color: #666666">=</span> LinearSVR(epsilon<span style="color: #666666">=1.5</span>)
|
||||
svm_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
To tackle nonlinear regression tasks, you can use a kernelized SVM model
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -107,6 +107,22 @@ td.padding {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -132,7 +148,7 @@ td.padding {
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Nov 1, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -168,107 +184,123 @@ We distringuish also between linearn and non-linear approaches.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec1">SVMs, basics with scikit-learn </h2>
|
||||
<h2 id="___sec1">Strength and weakness </h2>
|
||||
When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
|
||||
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec2">Examples with kernels </h2>
|
||||
|
||||
<p>
|
||||
The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
|
||||
model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
|
||||
Iris-Virginica flowers.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> set_matplotlib_formats, display
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">mglearn</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">cycler</span> <span style="color: #8B008B; font-weight: bold">import</span> cycler
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
|
||||
iris = datasets.load_iris()
|
||||
X = iris[<span style="color: #CD5555">"data"</span>][:, (<span style="color: #B452CD">2</span>, <span style="color: #B452CD">3</span>)] <span style="color: #228B22"># petal length, petal width</span>
|
||||
y = (iris[<span style="color: #CD5555">"target"</span>] == <span style="color: #B452CD">2</span>).astype(np.float64) <span style="color: #228B22"># Iris-Virginica</span>
|
||||
svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"linear_svc"</span>, LinearSVC(C=<span style="color: #B452CD">1</span>, loss=<span style="color: #CD5555">"hinge"</span>)),
|
||||
))
|
||||
svm_clf.fit(X_scaled, y)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_blobs
|
||||
|
||||
|
||||
X, y = make_blobs(centers=<span style="color: #B452CD">4</span>, random_state=<span style="color: #B452CD">8</span>)
|
||||
y = y % <span style="color: #B452CD">2</span>
|
||||
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
|
||||
linear_svm = LinearSVC().fit(X, y)
|
||||
|
||||
mglearn.plots.plot_2d_separator(linear_svm, X)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Alternatively, you could use the SVC class, using <b>SVC(kernel="linear", C=1)</b>, but it is much slower,
|
||||
especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
|
||||
class, with <b>SGDClassifier(loss="hinge", alpha=1/(m*C))</b>. This applies regular Stochastic
|
||||
Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
|
||||
LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
|
||||
training), or to handle online classification tasks.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec2">SVMs, adding polynomial features </h2>
|
||||
|
||||
<p>
|
||||
Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
|
||||
are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
|
||||
features, such as polynomial features. In some cases this can result in a linearly
|
||||
separable dataset.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
|
||||
polynomial_svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"poly_features"</span>, PolynomialFeatures(degree=<span style="color: #B452CD">3</span>)),
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"svm_clf"</span>, LinearSVC(C=<span style="color: #B452CD">10</span>, loss=<span style="color: #CD5555">"hinge"</span>))
|
||||
))
|
||||
polynomial_svm_clf.fit(X, y)
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># add the squared first feature</span>
|
||||
X_new = np.hstack([X, X[:, <span style="color: #B452CD">1</span>:] ** <span style="color: #B452CD">2</span>])
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D, axes3d
|
||||
figure = plt.figure()
|
||||
<span style="color: #228B22"># visualize in 3D</span>
|
||||
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
|
||||
<span style="color: #228B22"># plot first all the points with y==0, then all with y == 1</span>
|
||||
mask = y == <span style="color: #B452CD">0</span>
|
||||
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'b'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'r'</span>, marker=<span style="color: #CD5555">'^'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.set_xlabel(<span style="color: #CD5555">"feature0"</span>)
|
||||
ax.set_ylabel(<span style="color: #CD5555">"feature1"</span>)
|
||||
ax.set_zlabel(<span style="color: #CD5555">"feature1 ** 2"</span>)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec3">SVMs, polynomials and kernels </h2>
|
||||
|
||||
<p>
|
||||
Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
|
||||
algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
|
||||
and with a high polynomial degree it creates a huge number of features, making the model too slow.
|
||||
Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
|
||||
kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
|
||||
polynomial features, even with very high-degree polynomials, without actually having to add them. So
|
||||
there is no combinatorial explosion of the number of features since you don’t actually add any features.
|
||||
This trick is implemented by the SVC class.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>linear_svm_3d = LinearSVC().fit(X_new, y)
|
||||
coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
|
||||
|
||||
<span style="color: #228B22"># show linear decision boundary</span>
|
||||
figure = plt.figure()
|
||||
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
|
||||
xx = np.linspace(X_new[:, <span style="color: #B452CD">0</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">0</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
|
||||
yy = np.linspace(X_new[:, <span style="color: #B452CD">1</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">1</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
|
||||
|
||||
XX, YY = np.meshgrid(xx, yy)
|
||||
ZZ = (coef[<span style="color: #B452CD">0</span>] * XX + coef[<span style="color: #B452CD">1</span>] * YY + intercept) / -coef[<span style="color: #B452CD">2</span>]
|
||||
ax.plot_surface(XX, YY, ZZ, rstride=<span style="color: #B452CD">8</span>, cstride=<span style="color: #B452CD">8</span>, alpha=<span style="color: #B452CD">0.3</span>)
|
||||
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'b'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'r'</span>, marker=<span style="color: #CD5555">'^'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
|
||||
ax.set_xlabel(<span style="color: #CD5555">"feature0"</span>)
|
||||
ax.set_ylabel(<span style="color: #CD5555">"feature1"</span>)
|
||||
ax.set_zlabel(<span style="color: #CD5555">"feature1 ** 2"</span>)
|
||||
|
||||
ZZ = YY ** <span style="color: #B452CD">2</span>
|
||||
dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
|
||||
plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), <span style="color: #B452CD">0</span>, dec.max()],
|
||||
cmap=mglearn.cm2, alpha=<span style="color: #B452CD">0.5</span>)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
poly_kernel_svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"poly"</span>, degree=<span style="color: #B452CD">3</span>, coef0=<span style="color: #B452CD">1</span>, C=<span style="color: #B452CD">5</span>))
|
||||
))
|
||||
poly_kernel_svm_clf.fit(X, y)
|
||||
X, y = mglearn.tools.make_handcrafted_dataset()
|
||||
svm = SVC(kernel=<span style="color: #CD5555">'rbf'</span>, C=<span style="color: #B452CD">10</span>, gamma=<span style="color: #B452CD">0.1</span>).fit(X, y)
|
||||
mglearn.plots.plot_2d_separator(svm, X, eps=.<span style="color: #B452CD">5</span>)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
<span style="color: #228B22"># plot support vectors</span>
|
||||
sv = svm.support_vectors_
|
||||
<span style="color: #228B22"># class labels of support vectors are given by the sign of the dual coefficients</span>
|
||||
sv_labels = svm.dual_coef_.ravel() > <span style="color: #B452CD">0</span>
|
||||
mglearn.discrete_scatter(sv[:, <span style="color: #B452CD">0</span>], sv[:, <span style="color: #B452CD">1</span>], sv_labels, s=<span style="color: #B452CD">15</span>, markeredgewidth=<span style="color: #B452CD">3</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
|
||||
fig, axes = plt.subplots(<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, figsize=(<span style="color: #B452CD">15</span>, <span style="color: #B452CD">10</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> ax, C <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(axes, [-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">3</span>]):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> a, gamma <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(ax, <span style="color: #658b00">range</span>(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">2</span>)):
|
||||
mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
|
||||
|
||||
axes[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>].legend([<span style="color: #CD5555">"class 0"</span>, <span style="color: #CD5555">"class 1"</span>, <span style="color: #CD5555">"sv class 0"</span>, <span style="color: #CD5555">"sv class 1"</span>],
|
||||
ncol=<span style="color: #B452CD">4</span>, loc=(.<span style="color: #B452CD">9</span>, <span style="color: #B452CD">1.2</span>))
|
||||
</pre></div>
|
||||
<p>
|
||||
This code trains an SVM classifier using a 3rd-degree polynomial kernel.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec4">SVMs, regression </h2>
|
||||
|
||||
<p>
|
||||
You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
|
||||
shows how to use the regression option (more material to come)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVR
|
||||
svm_reg = LinearSVR(epsilon=<span style="color: #B452CD">1.5</span>)
|
||||
svm_reg.fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
To tackle nonlinear regression tasks, you can use a kernelized SVM model
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
@@ -35,14 +35,28 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Support Vector Machines, overarching aims', 2, None, '___sec0'),
|
||||
('SVMs, basics with scikit-learn', 2, None, '___sec1'),
|
||||
('SVMs, adding polynomial features', 2, None, '___sec2'),
|
||||
('SVMs, polynomials and kernels', 2, None, '___sec3'),
|
||||
('SVMs, regression', 2, None, '___sec4')]}
|
||||
('Strength and weakness', 2, None, '___sec1'),
|
||||
('Examples with kernels', 2, None, '___sec2')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "AMS" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
|
||||
@@ -64,7 +78,7 @@ end of tocinfo -->
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Nov 1, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -94,106 +108,124 @@ We distringuish also between linearn and non-linear approaches.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec1">SVMs, basics with scikit-learn </h2>
|
||||
<h2 id="___sec1">Strength and weakness </h2>
|
||||
When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
|
||||
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
|
||||
|
||||
<p>
|
||||
The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
|
||||
model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
|
||||
Iris-Virginica flowers.
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Examples with kernels </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">IPython.display</span> <span style="color: #8B008B; font-weight: bold">import</span> set_matplotlib_formats, display
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pandas</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">pd</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">mglearn</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">cycler</span> <span style="color: #8B008B; font-weight: bold">import</span> cycler
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
|
||||
iris = datasets.load_iris()
|
||||
X = iris[<span style="color: #CD5555">"data"</span>][:, (<span style="color: #B452CD">2</span>, <span style="color: #B452CD">3</span>)] <span style="color: #228B22"># petal length, petal width</span>
|
||||
y = (iris[<span style="color: #CD5555">"target"</span>] == <span style="color: #B452CD">2</span>).astype(np.float64) <span style="color: #228B22"># Iris-Virginica</span>
|
||||
svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"linear_svc"</span>, LinearSVC(C=<span style="color: #B452CD">1</span>, loss=<span style="color: #CD5555">"hinge"</span>)),
|
||||
))
|
||||
svm_clf.fit(X_scaled, y)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_blobs
|
||||
|
||||
|
||||
X, y = make_blobs(centers=<span style="color: #B452CD">4</span>, random_state=<span style="color: #B452CD">8</span>)
|
||||
y = y % <span style="color: #B452CD">2</span>
|
||||
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
|
||||
linear_svm = LinearSVC().fit(X, y)
|
||||
|
||||
mglearn.plots.plot_2d_separator(linear_svm, X)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Alternatively, you could use the SVC class, using <b>SVC(kernel="linear", C=1)</b>, but it is much slower,
|
||||
especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
|
||||
class, with <b>SGDClassifier(loss="hinge", alpha=1/(m*C))</b>. This applies regular Stochastic
|
||||
Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
|
||||
LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
|
||||
training), or to handle online classification tasks.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">SVMs, adding polynomial features </h2>
|
||||
|
||||
<p>
|
||||
Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
|
||||
are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
|
||||
features, such as polynomial features. In some cases this can result in a linearly
|
||||
separable dataset.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
|
||||
polynomial_svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"poly_features"</span>, PolynomialFeatures(degree=<span style="color: #B452CD">3</span>)),
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"svm_clf"</span>, LinearSVC(C=<span style="color: #B452CD">10</span>, loss=<span style="color: #CD5555">"hinge"</span>))
|
||||
))
|
||||
polynomial_svm_clf.fit(X, y)
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># add the squared first feature</span>
|
||||
X_new = np.hstack([X, X[:, <span style="color: #B452CD">1</span>:] ** <span style="color: #B452CD">2</span>])
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">mpl_toolkits.mplot3d</span> <span style="color: #8B008B; font-weight: bold">import</span> Axes3D, axes3d
|
||||
figure = plt.figure()
|
||||
<span style="color: #228B22"># visualize in 3D</span>
|
||||
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
|
||||
<span style="color: #228B22"># plot first all the points with y==0, then all with y == 1</span>
|
||||
mask = y == <span style="color: #B452CD">0</span>
|
||||
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'b'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'r'</span>, marker=<span style="color: #CD5555">'^'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.set_xlabel(<span style="color: #CD5555">"feature0"</span>)
|
||||
ax.set_ylabel(<span style="color: #CD5555">"feature1"</span>)
|
||||
ax.set_zlabel(<span style="color: #CD5555">"feature1 ** 2"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">SVMs, polynomials and kernels </h2>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>linear_svm_3d = LinearSVC().fit(X_new, y)
|
||||
coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
|
||||
|
||||
<p>
|
||||
Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
|
||||
algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
|
||||
and with a high polynomial degree it creates a huge number of features, making the model too slow.
|
||||
Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
|
||||
kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
|
||||
polynomial features, even with very high-degree polynomials, without actually having to add them. So
|
||||
there is no combinatorial explosion of the number of features since you don’t actually add any features.
|
||||
This trick is implemented by the SVC class.
|
||||
<span style="color: #228B22"># show linear decision boundary</span>
|
||||
figure = plt.figure()
|
||||
ax = Axes3D(figure, elev=-<span style="color: #B452CD">152</span>, azim=-<span style="color: #B452CD">26</span>)
|
||||
xx = np.linspace(X_new[:, <span style="color: #B452CD">0</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">0</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
|
||||
yy = np.linspace(X_new[:, <span style="color: #B452CD">1</span>].min() - <span style="color: #B452CD">2</span>, X_new[:, <span style="color: #B452CD">1</span>].max() + <span style="color: #B452CD">2</span>, <span style="color: #B452CD">50</span>)
|
||||
|
||||
XX, YY = np.meshgrid(xx, yy)
|
||||
ZZ = (coef[<span style="color: #B452CD">0</span>] * XX + coef[<span style="color: #B452CD">1</span>] * YY + intercept) / -coef[<span style="color: #B452CD">2</span>]
|
||||
ax.plot_surface(XX, YY, ZZ, rstride=<span style="color: #B452CD">8</span>, cstride=<span style="color: #B452CD">8</span>, alpha=<span style="color: #B452CD">0.3</span>)
|
||||
ax.scatter(X_new[mask, <span style="color: #B452CD">0</span>], X_new[mask, <span style="color: #B452CD">1</span>], X_new[mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'b'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
ax.scatter(X_new[~mask, <span style="color: #B452CD">0</span>], X_new[~mask, <span style="color: #B452CD">1</span>], X_new[~mask, <span style="color: #B452CD">2</span>], c=<span style="color: #CD5555">'r'</span>, marker=<span style="color: #CD5555">'^'</span>,
|
||||
cmap=mglearn.cm2, s=<span style="color: #B452CD">60</span>, edgecolor=<span style="color: #CD5555">'k'</span>)
|
||||
|
||||
ax.set_xlabel(<span style="color: #CD5555">"feature0"</span>)
|
||||
ax.set_ylabel(<span style="color: #CD5555">"feature1"</span>)
|
||||
ax.set_zlabel(<span style="color: #CD5555">"feature1 ** 2"</span>)
|
||||
|
||||
ZZ = YY ** <span style="color: #B452CD">2</span>
|
||||
dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
|
||||
plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), <span style="color: #B452CD">0</span>, dec.max()],
|
||||
cmap=mglearn.cm2, alpha=<span style="color: #B452CD">0.5</span>)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
poly_kernel_svm_clf = Pipeline((
|
||||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"poly"</span>, degree=<span style="color: #B452CD">3</span>, coef0=<span style="color: #B452CD">1</span>, C=<span style="color: #B452CD">5</span>))
|
||||
))
|
||||
poly_kernel_svm_clf.fit(X, y)
|
||||
X, y = mglearn.tools.make_handcrafted_dataset()
|
||||
svm = SVC(kernel=<span style="color: #CD5555">'rbf'</span>, C=<span style="color: #B452CD">10</span>, gamma=<span style="color: #B452CD">0.1</span>).fit(X, y)
|
||||
mglearn.plots.plot_2d_separator(svm, X, eps=.<span style="color: #B452CD">5</span>)
|
||||
mglearn.discrete_scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], y)
|
||||
<span style="color: #228B22"># plot support vectors</span>
|
||||
sv = svm.support_vectors_
|
||||
<span style="color: #228B22"># class labels of support vectors are given by the sign of the dual coefficients</span>
|
||||
sv_labels = svm.dual_coef_.ravel() > <span style="color: #B452CD">0</span>
|
||||
mglearn.discrete_scatter(sv[:, <span style="color: #B452CD">0</span>], sv[:, <span style="color: #B452CD">1</span>], sv_labels, s=<span style="color: #B452CD">15</span>, markeredgewidth=<span style="color: #B452CD">3</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"Feature 0"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Feature 1"</span>)
|
||||
|
||||
fig, axes = plt.subplots(<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, figsize=(<span style="color: #B452CD">15</span>, <span style="color: #B452CD">10</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> ax, C <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(axes, [-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">3</span>]):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> a, gamma <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(ax, <span style="color: #658b00">range</span>(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">2</span>)):
|
||||
mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
|
||||
|
||||
axes[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>].legend([<span style="color: #CD5555">"class 0"</span>, <span style="color: #CD5555">"class 1"</span>, <span style="color: #CD5555">"sv class 0"</span>, <span style="color: #CD5555">"sv class 1"</span>],
|
||||
ncol=<span style="color: #B452CD">4</span>, loc=(.<span style="color: #B452CD">9</span>, <span style="color: #B452CD">1.2</span>))
|
||||
</pre></div>
|
||||
<p>
|
||||
This code trains an SVM classifier using a 3rd-degree polynomial kernel.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">SVMs, regression </h2>
|
||||
|
||||
<p>
|
||||
You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
|
||||
shows how to use the regression option (more material to come)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVR
|
||||
svm_reg = LinearSVR(epsilon=<span style="color: #B452CD">1.5</span>)
|
||||
svm_reg.fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
To tackle nonlinear regression tasks, you can use a kernelized SVM model
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
+116
-84
@@ -40,14 +40,28 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Support Vector Machines, overarching aims', 2, None, '___sec0'),
|
||||
('SVMs, basics with scikit-learn', 2, None, '___sec1'),
|
||||
('SVMs, adding polynomial features', 2, None, '___sec2'),
|
||||
('SVMs, polynomials and kernels', 2, None, '___sec3'),
|
||||
('SVMs, regression', 2, None, '___sec4')]}
|
||||
('Strength and weakness', 2, None, '___sec1'),
|
||||
('Examples with kernels', 2, None, '___sec2')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "AMS" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
|
||||
@@ -69,7 +83,7 @@ end of tocinfo -->
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Nov 1, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Nov 2, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -99,106 +113,124 @@ We distringuish also between linearn and non-linear approaches.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec1">SVMs, basics with scikit-learn </h2>
|
||||
<h2 id="___sec1">Strength and weakness </h2>
|
||||
When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
|
||||
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
|
||||
|
||||
<p>
|
||||
The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
|
||||
model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
|
||||
Iris-Virginica flowers.
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Examples with kernels </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> set_matplotlib_formats, display
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">mglearn</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">cycler</span> <span style="color: #008000; font-weight: bold">import</span> cycler
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVC
|
||||
iris <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_iris()
|
||||
X <span style="color: #666666">=</span> iris[<span style="color: #BA2121">"data"</span>][:, (<span style="color: #666666">2</span>, <span style="color: #666666">3</span>)] <span style="color: #408080; font-style: italic"># petal length, petal width</span>
|
||||
y <span style="color: #666666">=</span> (iris[<span style="color: #BA2121">"target"</span>] <span style="color: #666666">==</span> <span style="color: #666666">2</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>float64) <span style="color: #408080; font-style: italic"># Iris-Virginica</span>
|
||||
svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"linear_svc"</span>, LinearSVC(C<span style="color: #666666">=1</span>, loss<span style="color: #666666">=</span><span style="color: #BA2121">"hinge"</span>)),
|
||||
))
|
||||
svm_clf<span style="color: #666666">.</span>fit(X_scaled, y)
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_blobs
|
||||
|
||||
|
||||
X, y <span style="color: #666666">=</span> make_blobs(centers<span style="color: #666666">=4</span>, random_state<span style="color: #666666">=8</span>)
|
||||
y <span style="color: #666666">=</span> y <span style="color: #666666">%</span> <span style="color: #666666">2</span>
|
||||
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVC
|
||||
linear_svm <span style="color: #666666">=</span> LinearSVC()<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_2d_separator(linear_svm, X)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Alternatively, you could use the SVC class, using <b>SVC(kernel="linear", C=1)</b>, but it is much slower,
|
||||
especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
|
||||
class, with <b>SGDClassifier(loss="hinge", alpha=1/(m*C))</b>. This applies regular Stochastic
|
||||
Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
|
||||
LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
|
||||
training), or to handle online classification tasks.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">SVMs, adding polynomial features </h2>
|
||||
|
||||
<p>
|
||||
Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
|
||||
are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
|
||||
features, such as polynomial features. In some cases this can result in a linearly
|
||||
separable dataset.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_moons
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.pipeline</span> <span style="color: #008000; font-weight: bold">import</span> Pipeline
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> PolynomialFeatures
|
||||
polynomial_svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"poly_features"</span>, PolynomialFeatures(degree<span style="color: #666666">=3</span>)),
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"svm_clf"</span>, LinearSVC(C<span style="color: #666666">=10</span>, loss<span style="color: #666666">=</span><span style="color: #BA2121">"hinge"</span>))
|
||||
))
|
||||
polynomial_svm_clf<span style="color: #666666">.</span>fit(X, y)
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># add the squared first feature</span>
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>hstack([X, X[:, <span style="color: #666666">1</span>:] <span style="color: #666666">**</span> <span style="color: #666666">2</span>])
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">mpl_toolkits.mplot3d</span> <span style="color: #008000; font-weight: bold">import</span> Axes3D, axes3d
|
||||
figure <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
<span style="color: #408080; font-style: italic"># visualize in 3D</span>
|
||||
ax <span style="color: #666666">=</span> Axes3D(figure, elev<span style="color: #666666">=-152</span>, azim<span style="color: #666666">=-26</span>)
|
||||
<span style="color: #408080; font-style: italic"># plot first all the points with y==0, then all with y == 1</span>
|
||||
mask <span style="color: #666666">=</span> y <span style="color: #666666">==</span> <span style="color: #666666">0</span>
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[mask, <span style="color: #666666">0</span>], X_new[mask, <span style="color: #666666">1</span>], X_new[mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'b'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">0</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">1</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'r'</span>, marker<span style="color: #666666">=</span><span style="color: #BA2121">'^'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"feature0"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"feature1"</span>)
|
||||
ax<span style="color: #666666">.</span>set_zlabel(<span style="color: #BA2121">"feature1 ** 2"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">SVMs, polynomials and kernels </h2>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>linear_svm_3d <span style="color: #666666">=</span> LinearSVC()<span style="color: #666666">.</span>fit(X_new, y)
|
||||
coef, intercept <span style="color: #666666">=</span> linear_svm_3d<span style="color: #666666">.</span>coef_<span style="color: #666666">.</span>ravel(), linear_svm_3d<span style="color: #666666">.</span>intercept_
|
||||
|
||||
<p>
|
||||
Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
|
||||
algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
|
||||
and with a high polynomial degree it creates a huge number of features, making the model too slow.
|
||||
Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
|
||||
kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
|
||||
polynomial features, even with very high-degree polynomials, without actually having to add them. So
|
||||
there is no combinatorial explosion of the number of features since you don’t actually add any features.
|
||||
This trick is implemented by the SVC class.
|
||||
<span style="color: #408080; font-style: italic"># show linear decision boundary</span>
|
||||
figure <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
||||
ax <span style="color: #666666">=</span> Axes3D(figure, elev<span style="color: #666666">=-152</span>, azim<span style="color: #666666">=-26</span>)
|
||||
xx <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(X_new[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">2</span>, X_new[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">2</span>, <span style="color: #666666">50</span>)
|
||||
yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(X_new[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">2</span>, X_new[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">2</span>, <span style="color: #666666">50</span>)
|
||||
|
||||
XX, YY <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(xx, yy)
|
||||
ZZ <span style="color: #666666">=</span> (coef[<span style="color: #666666">0</span>] <span style="color: #666666">*</span> XX <span style="color: #666666">+</span> coef[<span style="color: #666666">1</span>] <span style="color: #666666">*</span> YY <span style="color: #666666">+</span> intercept) <span style="color: #666666">/</span> <span style="color: #666666">-</span>coef[<span style="color: #666666">2</span>]
|
||||
ax<span style="color: #666666">.</span>plot_surface(XX, YY, ZZ, rstride<span style="color: #666666">=8</span>, cstride<span style="color: #666666">=8</span>, alpha<span style="color: #666666">=0.3</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[mask, <span style="color: #666666">0</span>], X_new[mask, <span style="color: #666666">1</span>], X_new[mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'b'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
ax<span style="color: #666666">.</span>scatter(X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">0</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">1</span>], X_new[<span style="color: #666666">~</span>mask, <span style="color: #666666">2</span>], c<span style="color: #666666">=</span><span style="color: #BA2121">'r'</span>, marker<span style="color: #666666">=</span><span style="color: #BA2121">'^'</span>,
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, s<span style="color: #666666">=60</span>, edgecolor<span style="color: #666666">=</span><span style="color: #BA2121">'k'</span>)
|
||||
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"feature0"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"feature1"</span>)
|
||||
ax<span style="color: #666666">.</span>set_zlabel(<span style="color: #BA2121">"feature1 ** 2"</span>)
|
||||
|
||||
ZZ <span style="color: #666666">=</span> YY <span style="color: #666666">**</span> <span style="color: #666666">2</span>
|
||||
dec <span style="color: #666666">=</span> linear_svm_3d<span style="color: #666666">.</span>decision_function(np<span style="color: #666666">.</span>c_[XX<span style="color: #666666">.</span>ravel(), YY<span style="color: #666666">.</span>ravel(), ZZ<span style="color: #666666">.</span>ravel()])
|
||||
plt<span style="color: #666666">.</span>contourf(XX, YY, dec<span style="color: #666666">.</span>reshape(XX<span style="color: #666666">.</span>shape), levels<span style="color: #666666">=</span>[dec<span style="color: #666666">.</span>min(), <span style="color: #666666">0</span>, dec<span style="color: #666666">.</span>max()],
|
||||
cmap<span style="color: #666666">=</span>mglearn<span style="color: #666666">.</span>cm2, alpha<span style="color: #666666">=0.5</span>)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
||||
poly_kernel_svm_clf <span style="color: #666666">=</span> Pipeline((
|
||||
(<span style="color: #BA2121">"scaler"</span>, StandardScaler()),
|
||||
(<span style="color: #BA2121">"svm_clf"</span>, SVC(kernel<span style="color: #666666">=</span><span style="color: #BA2121">"poly"</span>, degree<span style="color: #666666">=3</span>, coef0<span style="color: #666666">=1</span>, C<span style="color: #666666">=5</span>))
|
||||
))
|
||||
poly_kernel_svm_clf<span style="color: #666666">.</span>fit(X, y)
|
||||
X, y <span style="color: #666666">=</span> mglearn<span style="color: #666666">.</span>tools<span style="color: #666666">.</span>make_handcrafted_dataset()
|
||||
svm <span style="color: #666666">=</span> SVC(kernel<span style="color: #666666">=</span><span style="color: #BA2121">'rbf'</span>, C<span style="color: #666666">=10</span>, gamma<span style="color: #666666">=0.1</span>)<span style="color: #666666">.</span>fit(X, y)
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_2d_separator(svm, X, eps<span style="color: #666666">=.5</span>)
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], y)
|
||||
<span style="color: #408080; font-style: italic"># plot support vectors</span>
|
||||
sv <span style="color: #666666">=</span> svm<span style="color: #666666">.</span>support_vectors_
|
||||
<span style="color: #408080; font-style: italic"># class labels of support vectors are given by the sign of the dual coefficients</span>
|
||||
sv_labels <span style="color: #666666">=</span> svm<span style="color: #666666">.</span>dual_coef_<span style="color: #666666">.</span>ravel() <span style="color: #666666">></span> <span style="color: #666666">0</span>
|
||||
mglearn<span style="color: #666666">.</span>discrete_scatter(sv[:, <span style="color: #666666">0</span>], sv[:, <span style="color: #666666">1</span>], sv_labels, s<span style="color: #666666">=15</span>, markeredgewidth<span style="color: #666666">=3</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Feature 0"</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Feature 1"</span>)
|
||||
|
||||
fig, axes <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(<span style="color: #666666">3</span>, <span style="color: #666666">3</span>, figsize<span style="color: #666666">=</span>(<span style="color: #666666">15</span>, <span style="color: #666666">10</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> ax, C <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(axes, [<span style="color: #666666">-1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">3</span>]):
|
||||
<span style="color: #008000; font-weight: bold">for</span> a, gamma <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(ax, <span style="color: #008000">range</span>(<span style="color: #666666">-1</span>, <span style="color: #666666">2</span>)):
|
||||
mglearn<span style="color: #666666">.</span>plots<span style="color: #666666">.</span>plot_svm(log_C<span style="color: #666666">=</span>C, log_gamma<span style="color: #666666">=</span>gamma, ax<span style="color: #666666">=</span>a)
|
||||
|
||||
axes[<span style="color: #666666">0</span>, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>legend([<span style="color: #BA2121">"class 0"</span>, <span style="color: #BA2121">"class 1"</span>, <span style="color: #BA2121">"sv class 0"</span>, <span style="color: #BA2121">"sv class 1"</span>],
|
||||
ncol<span style="color: #666666">=4</span>, loc<span style="color: #666666">=</span>(<span style="color: #666666">.9</span>, <span style="color: #666666">1.2</span>))
|
||||
</pre></div>
|
||||
<p>
|
||||
This code trains an SVM classifier using a 3rd-degree polynomial kernel.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">SVMs, regression </h2>
|
||||
|
||||
<p>
|
||||
You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
|
||||
shows how to use the regression option (more material to come)
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> LinearSVR
|
||||
svm_reg <span style="color: #666666">=</span> LinearSVR(epsilon<span style="color: #666666">=1.5</span>)
|
||||
svm_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
</pre></div>
|
||||
<p>
|
||||
To tackle nonlinear regression tasks, you can use a kernelized SVM model
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
Binary file not shown.
+93
-83
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Nov 1, 2018**\n",
|
||||
"Date: **Nov 2, 2018**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -34,11 +34,13 @@
|
||||
"\n",
|
||||
"**These notes will be updated shortly with more material**\n",
|
||||
"\n",
|
||||
"## SVMs, basics with scikit-learn\n",
|
||||
"\n",
|
||||
"The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM\n",
|
||||
"model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect\n",
|
||||
"Iris-Virginica flowers."
|
||||
"## Strength and weakness\n",
|
||||
"When we implement a linear support vector machine, the main parameter is the constant $C$. Small values of $C$ mean simple models.\n",
|
||||
"These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Examples with kernels"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -49,38 +51,34 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"from IPython.display import set_matplotlib_formats, display\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn import datasets\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.preprocessing import StandardScaler\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import mglearn\n",
|
||||
"from cycler import cycler\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import LinearSVC\n",
|
||||
"iris = datasets.load_iris()\n",
|
||||
"X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n",
|
||||
"y = (iris[\"target\"] == 2).astype(np.float64) # Iris-Virginica\n",
|
||||
"svm_clf = Pipeline((\n",
|
||||
"(\"scaler\", StandardScaler()),\n",
|
||||
"(\"linear_svc\", LinearSVC(C=1, loss=\"hinge\")),\n",
|
||||
"))\n",
|
||||
"svm_clf.fit(X_scaled, y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Alternatively, you could use the SVC class, using **SVC(kernel=\"linear\", C=1)**, but it is much slower,\n",
|
||||
"especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier\n",
|
||||
"class, with **SGDClassifier(loss=\"hinge\", alpha=1/(m*C))**. This applies regular Stochastic\n",
|
||||
"Gradient Descentto train a linear SVM classifier. It does not converge as fast as the\n",
|
||||
"LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core\n",
|
||||
"training), or to handle online classification tasks.\n",
|
||||
"from sklearn.datasets import make_blobs\n",
|
||||
"\n",
|
||||
"## SVMs, adding polynomial features\n",
|
||||
"\n",
|
||||
"Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets\n",
|
||||
"are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more\n",
|
||||
"features, such as polynomial features. In some cases this can result in a linearly\n",
|
||||
"separable dataset."
|
||||
"X, y = make_blobs(centers=4, random_state=8)\n",
|
||||
"y = y % 2\n",
|
||||
"\n",
|
||||
"mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
|
||||
"plt.xlabel(\"Feature 0\")\n",
|
||||
"plt.ylabel(\"Feature 1\")\n",
|
||||
"plt.show()\n",
|
||||
"\n",
|
||||
"from sklearn.svm import LinearSVC\n",
|
||||
"linear_svm = LinearSVC().fit(X, y)\n",
|
||||
"\n",
|
||||
"mglearn.plots.plot_2d_separator(linear_svm, X)\n",
|
||||
"mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
|
||||
"plt.xlabel(\"Feature 0\")\n",
|
||||
"plt.ylabel(\"Feature 1\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -91,31 +89,23 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures\n",
|
||||
"polynomial_svm_clf = Pipeline((\n",
|
||||
"(\"poly_features\", PolynomialFeatures(degree=3)),\n",
|
||||
"(\"scaler\", StandardScaler()),\n",
|
||||
"(\"svm_clf\", LinearSVC(C=10, loss=\"hinge\"))\n",
|
||||
"))\n",
|
||||
"polynomial_svm_clf.fit(X, y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## SVMs, polynomials and kernels\n",
|
||||
"# add the squared first feature\n",
|
||||
"X_new = np.hstack([X, X[:, 1:] ** 2])\n",
|
||||
"\n",
|
||||
"Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning\n",
|
||||
"algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,\n",
|
||||
"and with a high polynomial degree it creates a huge number of features, making the model too slow.\n",
|
||||
"Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the\n",
|
||||
"kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many\n",
|
||||
"polynomial features, even with very high-degree polynomials, without actually having to add them. So\n",
|
||||
"there is no combinatorial explosion of the number of features since you don’t actually add any features.\n",
|
||||
"This trick is implemented by the SVC class."
|
||||
"\n",
|
||||
"from mpl_toolkits.mplot3d import Axes3D, axes3d\n",
|
||||
"figure = plt.figure()\n",
|
||||
"# visualize in 3D\n",
|
||||
"ax = Axes3D(figure, elev=-152, azim=-26)\n",
|
||||
"# plot first all the points with y==0, then all with y == 1\n",
|
||||
"mask = y == 0\n",
|
||||
"ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
|
||||
" cmap=mglearn.cm2, s=60, edgecolor='k')\n",
|
||||
"ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
|
||||
" cmap=mglearn.cm2, s=60, edgecolor='k')\n",
|
||||
"ax.set_xlabel(\"feature0\")\n",
|
||||
"ax.set_ylabel(\"feature1\")\n",
|
||||
"ax.set_zlabel(\"feature1 ** 2\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -126,25 +116,34 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.svm import SVC\n",
|
||||
"poly_kernel_svm_clf = Pipeline((\n",
|
||||
"(\"scaler\", StandardScaler()),\n",
|
||||
"(\"svm_clf\", SVC(kernel=\"poly\", degree=3, coef0=1, C=5))\n",
|
||||
"))\n",
|
||||
"poly_kernel_svm_clf.fit(X, y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This code trains an SVM classifier using a 3rd-degree polynomial kernel.\n",
|
||||
"linear_svm_3d = LinearSVC().fit(X_new, y)\n",
|
||||
"coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_\n",
|
||||
"\n",
|
||||
"# show linear decision boundary\n",
|
||||
"figure = plt.figure()\n",
|
||||
"ax = Axes3D(figure, elev=-152, azim=-26)\n",
|
||||
"xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)\n",
|
||||
"yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)\n",
|
||||
"\n",
|
||||
"## SVMs, regression\n",
|
||||
"XX, YY = np.meshgrid(xx, yy)\n",
|
||||
"ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]\n",
|
||||
"ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)\n",
|
||||
"ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
|
||||
" cmap=mglearn.cm2, s=60, edgecolor='k')\n",
|
||||
"ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
|
||||
" cmap=mglearn.cm2, s=60, edgecolor='k')\n",
|
||||
"\n",
|
||||
"You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code\n",
|
||||
"shows how to use the regression option (more material to come)"
|
||||
"ax.set_xlabel(\"feature0\")\n",
|
||||
"ax.set_ylabel(\"feature1\")\n",
|
||||
"ax.set_zlabel(\"feature1 ** 2\")\n",
|
||||
"\n",
|
||||
"ZZ = YY ** 2\n",
|
||||
"dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])\n",
|
||||
"plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],\n",
|
||||
" cmap=mglearn.cm2, alpha=0.5)\n",
|
||||
"mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
|
||||
"plt.xlabel(\"Feature 0\")\n",
|
||||
"plt.ylabel(\"Feature 1\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -155,16 +154,27 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.svm import LinearSVR\n",
|
||||
"svm_reg = LinearSVR(epsilon=1.5)\n",
|
||||
"svm_reg.fit(X, y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"To tackle nonlinear regression tasks, you can use a kernelized SVM model"
|
||||
"from sklearn.svm import SVC\n",
|
||||
"X, y = mglearn.tools.make_handcrafted_dataset() \n",
|
||||
"svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)\n",
|
||||
"mglearn.plots.plot_2d_separator(svm, X, eps=.5)\n",
|
||||
"mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
|
||||
"# plot support vectors\n",
|
||||
"sv = svm.support_vectors_\n",
|
||||
"# class labels of support vectors are given by the sign of the dual coefficients\n",
|
||||
"sv_labels = svm.dual_coef_.ravel() > 0\n",
|
||||
"mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)\n",
|
||||
"plt.xlabel(\"Feature 0\")\n",
|
||||
"plt.ylabel(\"Feature 1\")\n",
|
||||
"\n",
|
||||
"fig, axes = plt.subplots(3, 3, figsize=(15, 10))\n",
|
||||
"\n",
|
||||
"for ax, C in zip(axes, [-1, 0, 3]):\n",
|
||||
" for a, gamma in zip(ax, range(-1, 2)):\n",
|
||||
" mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)\n",
|
||||
" \n",
|
||||
"axes[0, 0].legend([\"class 0\", \"class 1\", \"sv class 0\", \"sv class 1\"],\n",
|
||||
" ncol=4, loc=(.9, 1.2))"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -21,87 +21,117 @@ We distringuish also between linearn and non-linear approaches.
|
||||
|
||||
_These notes will be updated shortly with more material_
|
||||
|
||||
!split
|
||||
===== SVMs, basics with scikit-learn =====
|
||||
|
||||
The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
|
||||
model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
|
||||
Iris-Virginica flowers.
|
||||
!split
|
||||
===== Strength and weakness =====
|
||||
When we implement a linear support vector machine, the main parameter is the constant $C$. Small values of $C$ mean simple models.
|
||||
These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
|
||||
|
||||
|
||||
!split
|
||||
===== Examples with kernels =====
|
||||
|
||||
!bc pycod
|
||||
from IPython.display import set_matplotlib_formats, display
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from sklearn import datasets
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import matplotlib.pyplot as plt
|
||||
import mglearn
|
||||
from cycler import cycler
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.svm import LinearSVC
|
||||
iris = datasets.load_iris()
|
||||
X = iris["data"][:, (2, 3)] # petal length, petal width
|
||||
y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
|
||||
svm_clf = Pipeline((
|
||||
("scaler", StandardScaler()),
|
||||
("linear_svc", LinearSVC(C=1, loss="hinge")),
|
||||
))
|
||||
svm_clf.fit(X_scaled, y)
|
||||
from sklearn.datasets import make_blobs
|
||||
|
||||
|
||||
X, y = make_blobs(centers=4, random_state=8)
|
||||
y = y % 2
|
||||
|
||||
mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
|
||||
plt.xlabel("Feature 0")
|
||||
plt.ylabel("Feature 1")
|
||||
plt.show()
|
||||
|
||||
from sklearn.svm import LinearSVC
|
||||
linear_svm = LinearSVC().fit(X, y)
|
||||
|
||||
mglearn.plots.plot_2d_separator(linear_svm, X)
|
||||
mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
|
||||
plt.xlabel("Feature 0")
|
||||
plt.ylabel("Feature 1")
|
||||
!ec
|
||||
|
||||
Alternatively, you could use the SVC class, using _SVC(kernel="linear", C=1)_, but it is much slower,
|
||||
especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
|
||||
class, with _SGDClassifier(loss="hinge", alpha=1/(m*C))_. This applies regular Stochastic
|
||||
Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
|
||||
LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
|
||||
training), or to handle online classification tasks.
|
||||
|
||||
!split
|
||||
===== SVMs, adding polynomial features =====
|
||||
|
||||
Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
|
||||
are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
|
||||
features, such as polynomial features. In some cases this can result in a linearly
|
||||
separable dataset.
|
||||
|
||||
!bc pycod
|
||||
from sklearn.datasets import make_moons
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
polynomial_svm_clf = Pipeline((
|
||||
("poly_features", PolynomialFeatures(degree=3)),
|
||||
("scaler", StandardScaler()),
|
||||
("svm_clf", LinearSVC(C=10, loss="hinge"))
|
||||
))
|
||||
polynomial_svm_clf.fit(X, y)
|
||||
# add the squared first feature
|
||||
X_new = np.hstack([X, X[:, 1:] ** 2])
|
||||
|
||||
|
||||
from mpl_toolkits.mplot3d import Axes3D, axes3d
|
||||
figure = plt.figure()
|
||||
# visualize in 3D
|
||||
ax = Axes3D(figure, elev=-152, azim=-26)
|
||||
# plot first all the points with y==0, then all with y == 1
|
||||
mask = y == 0
|
||||
ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
|
||||
cmap=mglearn.cm2, s=60, edgecolor='k')
|
||||
ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
|
||||
cmap=mglearn.cm2, s=60, edgecolor='k')
|
||||
ax.set_xlabel("feature0")
|
||||
ax.set_ylabel("feature1")
|
||||
ax.set_zlabel("feature1 ** 2")
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
linear_svm_3d = LinearSVC().fit(X_new, y)
|
||||
coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
|
||||
|
||||
!split
|
||||
===== SVMs, polynomials and kernels =====
|
||||
# show linear decision boundary
|
||||
figure = plt.figure()
|
||||
ax = Axes3D(figure, elev=-152, azim=-26)
|
||||
xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
|
||||
yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
|
||||
|
||||
Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
|
||||
algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
|
||||
and with a high polynomial degree it creates a huge number of features, making the model too slow.
|
||||
Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
|
||||
kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
|
||||
polynomial features, even with very high-degree polynomials, without actually having to add them. So
|
||||
there is no combinatorial explosion of the number of features since you don’t actually add any features.
|
||||
This trick is implemented by the SVC class.
|
||||
XX, YY = np.meshgrid(xx, yy)
|
||||
ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
|
||||
ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
|
||||
ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
|
||||
cmap=mglearn.cm2, s=60, edgecolor='k')
|
||||
ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
|
||||
cmap=mglearn.cm2, s=60, edgecolor='k')
|
||||
|
||||
ax.set_xlabel("feature0")
|
||||
ax.set_ylabel("feature1")
|
||||
ax.set_zlabel("feature1 ** 2")
|
||||
|
||||
ZZ = YY ** 2
|
||||
dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
|
||||
plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
|
||||
cmap=mglearn.cm2, alpha=0.5)
|
||||
mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
|
||||
plt.xlabel("Feature 0")
|
||||
plt.ylabel("Feature 1")
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
from sklearn.svm import SVC
|
||||
poly_kernel_svm_clf = Pipeline((
|
||||
("scaler", StandardScaler()),
|
||||
("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
|
||||
))
|
||||
poly_kernel_svm_clf.fit(X, y)
|
||||
X, y = mglearn.tools.make_handcrafted_dataset()
|
||||
svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
|
||||
mglearn.plots.plot_2d_separator(svm, X, eps=.5)
|
||||
mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
|
||||
# plot support vectors
|
||||
sv = svm.support_vectors_
|
||||
# class labels of support vectors are given by the sign of the dual coefficients
|
||||
sv_labels = svm.dual_coef_.ravel() > 0
|
||||
mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
|
||||
plt.xlabel("Feature 0")
|
||||
plt.ylabel("Feature 1")
|
||||
|
||||
fig, axes = plt.subplots(3, 3, figsize=(15, 10))
|
||||
|
||||
for ax, C in zip(axes, [-1, 0, 3]):
|
||||
for a, gamma in zip(ax, range(-1, 2)):
|
||||
mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
|
||||
|
||||
axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
|
||||
ncol=4, loc=(.9, 1.2))
|
||||
|
||||
!ec
|
||||
This code trains an SVM classifier using a 3rd-degree polynomial kernel.
|
||||
|
||||
|
||||
!split
|
||||
===== SVMs, regression =====
|
||||
|
||||
You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
|
||||
shows how to use the regression option (more material to come)
|
||||
!bc pycod
|
||||
from sklearn.svm import LinearSVR
|
||||
svm_reg = LinearSVR(epsilon=1.5)
|
||||
svm_reg.fit(X, y)
|
||||
!ec
|
||||
To tackle nonlinear regression tasks, you can use a kernelized SVM model
|
||||
|
||||
Reference in New Issue
Block a user