-
@@ -137,106 +149,124 @@ We distringuish also between linearn and non-linear approaches.
-
-The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM -model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect -Iris-Virginica flowers. + + +
-
import numpy as np
-from sklearn import datasets
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+from IPython.display import set_matplotlib_formats, display
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import mglearn
+from cycler import cycler
+from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
-iris = datasets.load_iris()
-X = iris["data"][:, (2, 3)] # petal length, petal width
-y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
-svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("linear_svc", LinearSVC(C=1, loss="hinge")),
-))
-svm_clf.fit(X_scaled, y)
+from sklearn.datasets import make_blobs
+
+
+X, y = make_blobs(centers=4, random_state=8)
+y = y % 2
+
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+plt.show()
+
+from sklearn.svm import LinearSVC
+linear_svm = LinearSVC().fit(X, y)
+
+mglearn.plots.plot_2d_separator(linear_svm, X)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
-
-Alternatively, you could use the SVC class, using SVC(kernel="linear", C=1), but it is much slower,
-especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
-class, with SGDClassifier(loss="hinge", alpha=1/(m*C)). This applies regular Stochastic
-Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
-LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
-training), or to handle online classification tasks.
-
-
-
-
-
SVMs, adding polynomial features
-
-
-Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
-are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
-features, such as polynomial features. In some cases this can result in a linearly
-separable dataset.
-
-
from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-polynomial_svm_clf = Pipeline((
-("poly_features", PolynomialFeatures(degree=3)),
-("scaler", StandardScaler()),
-("svm_clf", LinearSVC(C=10, loss="hinge"))
-))
-polynomial_svm_clf.fit(X, y)
+# add the squared first feature
+X_new = np.hstack([X, X[:, 1:] ** 2])
+
+
+from mpl_toolkits.mplot3d import Axes3D, axes3d
+figure = plt.figure()
+# visualize in 3D
+ax = Axes3D(figure, elev=-152, azim=-26)
+# plot first all the points with y==0, then all with y == 1
+mask = y == 0
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
-
-
SVMs, polynomials and kernels
+
+linear_svm_3d = LinearSVC().fit(X_new, y)
+coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
-
-Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
-algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
-and with a high polynomial degree it creates a huge number of features, making the model too slow.
-Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
-kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
-polynomial features, even with very high-degree polynomials, without actually having to add them. So
-there is no combinatorial explosion of the number of features since you don’t actually add any features.
-This trick is implemented by the SVC class.
+# show linear decision boundary
+figure = plt.figure()
+ax = Axes3D(figure, elev=-152, azim=-26)
+xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
+yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
+XX, YY = np.meshgrid(xx, yy)
+ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
+ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
+
+ZZ = YY ** 2
+dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
+plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
+ cmap=mglearn.cm2, alpha=0.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
from sklearn.svm import SVC
-poly_kernel_svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
-))
-poly_kernel_svm_clf.fit(X, y)
+X, y = mglearn.tools.make_handcrafted_dataset()
+svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
+mglearn.plots.plot_2d_separator(svm, X, eps=.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+# plot support vectors
+sv = svm.support_vectors_
+# class labels of support vectors are given by the sign of the dual coefficients
+sv_labels = svm.dual_coef_.ravel() > 0
+mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
+fig, axes = plt.subplots(3, 3, figsize=(15, 10))
+
+for ax, C in zip(axes, [-1, 0, 3]):
+ for a, gamma in zip(ax, range(-1, 2)):
+ mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
+
+axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
+ ncol=4, loc=(.9, 1.2))
-This code trains an SVM classifier using a 3rd-degree polynomial kernel.
-
-
-
-
-
SVMs, regression
-
-
-You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
-shows how to use the regression option (more material to come)
-
-
-
-
from sklearn.svm import LinearSVR
-svm_reg = LinearSVR(epsilon=1.5)
-svm_reg.fit(X, y)
-
-
-To tackle nonlinear regression tasks, you can use a kernelized SVM model
diff --git a/doc/pub/svm/html/svm-reveal.html b/doc/pub/svm/html/svm-reveal.html
index 1782b55ba..9ba90f136 100644
--- a/doc/pub/svm/html/svm-reveal.html
+++ b/doc/pub/svm/html/svm-reveal.html
@@ -107,6 +107,22 @@ td.padding {
+
+
+
+
+
+
+
@@ -132,7 +148,7 @@ td.padding {
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 1, 2018
Nov 2, 2018
@@ -168,107 +184,123 @@ We distringuish also between linearn and non-linear approaches.
-SVMs, basics with scikit-learn
+Strength and weakness
+When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
+These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
+
+
+
+
+Examples with kernels
-
-The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
-model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
-Iris-Virginica flowers.
-
import numpy as np
-from sklearn import datasets
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+from IPython.display import set_matplotlib_formats, display
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import mglearn
+from cycler import cycler
+from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
-iris = datasets.load_iris()
-X = iris["data"][:, (2, 3)] # petal length, petal width
-y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
-svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("linear_svc", LinearSVC(C=1, loss="hinge")),
-))
-svm_clf.fit(X_scaled, y)
+from sklearn.datasets import make_blobs
+
+
+X, y = make_blobs(centers=4, random_state=8)
+y = y % 2
+
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+plt.show()
+
+from sklearn.svm import LinearSVC
+linear_svm = LinearSVC().fit(X, y)
+
+mglearn.plots.plot_2d_separator(linear_svm, X)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
-
-Alternatively, you could use the SVC class, using SVC(kernel="linear", C=1), but it is much slower,
-especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
-class, with SGDClassifier(loss="hinge", alpha=1/(m*C)). This applies regular Stochastic
-Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
-LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
-training), or to handle online classification tasks.
-
-
-
-
-SVMs, adding polynomial features
-
-
-Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
-are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
-features, such as polynomial features. In some cases this can result in a linearly
-separable dataset.
-
-
from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-polynomial_svm_clf = Pipeline((
-("poly_features", PolynomialFeatures(degree=3)),
-("scaler", StandardScaler()),
-("svm_clf", LinearSVC(C=10, loss="hinge"))
-))
-polynomial_svm_clf.fit(X, y)
+# add the squared first feature
+X_new = np.hstack([X, X[:, 1:] ** 2])
+
+
+from mpl_toolkits.mplot3d import Axes3D, axes3d
+figure = plt.figure()
+# visualize in 3D
+ax = Axes3D(figure, elev=-152, azim=-26)
+# plot first all the points with y==0, then all with y == 1
+mask = y == 0
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
-
-
-
-
-SVMs, polynomials and kernels
-
-Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
-algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
-and with a high polynomial degree it creates a huge number of features, making the model too slow.
-Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
-kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
-polynomial features, even with very high-degree polynomials, without actually having to add them. So
-there is no combinatorial explosion of the number of features since you don’t actually add any features.
-This trick is implemented by the SVC class.
+
+
linear_svm_3d = LinearSVC().fit(X_new, y)
+coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
+
+# show linear decision boundary
+figure = plt.figure()
+ax = Axes3D(figure, elev=-152, azim=-26)
+xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
+yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
+
+XX, YY = np.meshgrid(xx, yy)
+ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
+ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
+
+ZZ = YY ** 2
+dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
+plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
+ cmap=mglearn.cm2, alpha=0.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
from sklearn.svm import SVC
-poly_kernel_svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
-))
-poly_kernel_svm_clf.fit(X, y)
+X, y = mglearn.tools.make_handcrafted_dataset()
+svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
+mglearn.plots.plot_2d_separator(svm, X, eps=.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+# plot support vectors
+sv = svm.support_vectors_
+# class labels of support vectors are given by the sign of the dual coefficients
+sv_labels = svm.dual_coef_.ravel() > 0
+mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
+fig, axes = plt.subplots(3, 3, figsize=(15, 10))
+
+for ax, C in zip(axes, [-1, 0, 3]):
+ for a, gamma in zip(ax, range(-1, 2)):
+ mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
+
+axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
+ ncol=4, loc=(.9, 1.2))
-
-This code trains an SVM classifier using a 3rd-degree polynomial kernel.
-
-
-
-
-SVMs, regression
-
-
-You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
-shows how to use the regression option (more material to come)
-
-
-
-
from sklearn.svm import LinearSVR
-svm_reg = LinearSVR(epsilon=1.5)
-svm_reg.fit(X, y)
-
-
-To tackle nonlinear regression tasks, you can use a kernelized SVM model
diff --git a/doc/pub/svm/html/svm-solarized.html b/doc/pub/svm/html/svm-solarized.html
index f6107c8cf..f7cdf487e 100644
--- a/doc/pub/svm/html/svm-solarized.html
+++ b/doc/pub/svm/html/svm-solarized.html
@@ -35,14 +35,28 @@ div { text-align: justify; text-justify: inter-word; }
+
+
+
+
+
+
+
@@ -64,7 +78,7 @@ end of tocinfo -->
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 1, 2018
Nov 2, 2018
@@ -94,106 +108,124 @@ We distringuish also between linearn and non-linear approaches.
-
SVMs, basics with scikit-learn
+Strength and weakness
+When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
+These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
-The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
-model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
-Iris-Virginica flowers.
+
+
+
Examples with kernels
+
-
import numpy as np
-from sklearn import datasets
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+from IPython.display import set_matplotlib_formats, display
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import mglearn
+from cycler import cycler
+from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
-iris = datasets.load_iris()
-X = iris["data"][:, (2, 3)] # petal length, petal width
-y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
-svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("linear_svc", LinearSVC(C=1, loss="hinge")),
-))
-svm_clf.fit(X_scaled, y)
+from sklearn.datasets import make_blobs
+
+
+X, y = make_blobs(centers=4, random_state=8)
+y = y % 2
+
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+plt.show()
+
+from sklearn.svm import LinearSVC
+linear_svm = LinearSVC().fit(X, y)
+
+mglearn.plots.plot_2d_separator(linear_svm, X)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
-
-Alternatively, you could use the SVC class, using SVC(kernel="linear", C=1), but it is much slower,
-especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
-class, with SGDClassifier(loss="hinge", alpha=1/(m*C)). This applies regular Stochastic
-Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
-LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
-training), or to handle online classification tasks.
-
-
-
-
-
SVMs, adding polynomial features
-
-
-Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
-are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
-features, such as polynomial features. In some cases this can result in a linearly
-separable dataset.
-
-
from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-polynomial_svm_clf = Pipeline((
-("poly_features", PolynomialFeatures(degree=3)),
-("scaler", StandardScaler()),
-("svm_clf", LinearSVC(C=10, loss="hinge"))
-))
-polynomial_svm_clf.fit(X, y)
+# add the squared first feature
+X_new = np.hstack([X, X[:, 1:] ** 2])
+
+
+from mpl_toolkits.mplot3d import Axes3D, axes3d
+figure = plt.figure()
+# visualize in 3D
+ax = Axes3D(figure, elev=-152, azim=-26)
+# plot first all the points with y==0, then all with y == 1
+mask = y == 0
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
-
-
SVMs, polynomials and kernels
+
+linear_svm_3d = LinearSVC().fit(X_new, y)
+coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
-
-Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
-algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
-and with a high polynomial degree it creates a huge number of features, making the model too slow.
-Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
-kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
-polynomial features, even with very high-degree polynomials, without actually having to add them. So
-there is no combinatorial explosion of the number of features since you don’t actually add any features.
-This trick is implemented by the SVC class.
+# show linear decision boundary
+figure = plt.figure()
+ax = Axes3D(figure, elev=-152, azim=-26)
+xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
+yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
+XX, YY = np.meshgrid(xx, yy)
+ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
+ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
+
+ZZ = YY ** 2
+dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
+plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
+ cmap=mglearn.cm2, alpha=0.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
from sklearn.svm import SVC
-poly_kernel_svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
-))
-poly_kernel_svm_clf.fit(X, y)
+X, y = mglearn.tools.make_handcrafted_dataset()
+svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
+mglearn.plots.plot_2d_separator(svm, X, eps=.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+# plot support vectors
+sv = svm.support_vectors_
+# class labels of support vectors are given by the sign of the dual coefficients
+sv_labels = svm.dual_coef_.ravel() > 0
+mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
+fig, axes = plt.subplots(3, 3, figsize=(15, 10))
+
+for ax, C in zip(axes, [-1, 0, 3]):
+ for a, gamma in zip(ax, range(-1, 2)):
+ mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
+
+axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
+ ncol=4, loc=(.9, 1.2))
-This code trains an SVM classifier using a 3rd-degree polynomial kernel.
-
-
-
-
-
SVMs, regression
-
-
-You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
-shows how to use the regression option (more material to come)
-
-
-
-
from sklearn.svm import LinearSVR
-svm_reg = LinearSVR(epsilon=1.5)
-svm_reg.fit(X, y)
-
-
-To tackle nonlinear regression tasks, you can use a kernelized SVM model
diff --git a/doc/pub/svm/html/svm.html b/doc/pub/svm/html/svm.html
index cf2fe755b..d2c492c95 100644
--- a/doc/pub/svm/html/svm.html
+++ b/doc/pub/svm/html/svm.html
@@ -40,14 +40,28 @@ div { text-align: justify; text-justify: inter-word; }
+
+
+
+
+
+
+
@@ -69,7 +83,7 @@ end of tocinfo -->
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
-
Nov 1, 2018
Nov 2, 2018
@@ -99,106 +113,124 @@ We distringuish also between linearn and non-linear approaches.
-
SVMs, basics with scikit-learn
+Strength and weakness
+When we implement a linear support vector machine, the main parameter is the constant \( C \). Small values of \( C \) mean simple models.
+These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
-The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
-model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
-Iris-Virginica flowers.
+
+
+
Examples with kernels
+
-
import numpy as np
-from sklearn import datasets
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+from IPython.display import set_matplotlib_formats, display
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import mglearn
+from cycler import cycler
+from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
-iris = datasets.load_iris()
-X = iris["data"][:, (2, 3)] # petal length, petal width
-y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
-svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("linear_svc", LinearSVC(C=1, loss="hinge")),
-))
-svm_clf.fit(X_scaled, y)
+from sklearn.datasets import make_blobs
+
+
+X, y = make_blobs(centers=4, random_state=8)
+y = y % 2
+
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+plt.show()
+
+from sklearn.svm import LinearSVC
+linear_svm = LinearSVC().fit(X, y)
+
+mglearn.plots.plot_2d_separator(linear_svm, X)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
-
-Alternatively, you could use the SVC class, using SVC(kernel="linear", C=1), but it is much slower,
-especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
-class, with SGDClassifier(loss="hinge", alpha=1/(m*C)). This applies regular Stochastic
-Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
-LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
-training), or to handle online classification tasks.
-
-
-
-
-
SVMs, adding polynomial features
-
-
-Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
-are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
-features, such as polynomial features. In some cases this can result in a linearly
-separable dataset.
-
-
from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-polynomial_svm_clf = Pipeline((
-("poly_features", PolynomialFeatures(degree=3)),
-("scaler", StandardScaler()),
-("svm_clf", LinearSVC(C=10, loss="hinge"))
-))
-polynomial_svm_clf.fit(X, y)
+# add the squared first feature
+X_new = np.hstack([X, X[:, 1:] ** 2])
+
+
+from mpl_toolkits.mplot3d import Axes3D, axes3d
+figure = plt.figure()
+# visualize in 3D
+ax = Axes3D(figure, elev=-152, azim=-26)
+# plot first all the points with y==0, then all with y == 1
+mask = y == 0
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
-
-
SVMs, polynomials and kernels
+
+linear_svm_3d = LinearSVC().fit(X_new, y)
+coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
-
-Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
-algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
-and with a high polynomial degree it creates a huge number of features, making the model too slow.
-Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
-kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
-polynomial features, even with very high-degree polynomials, without actually having to add them. So
-there is no combinatorial explosion of the number of features since you don’t actually add any features.
-This trick is implemented by the SVC class.
+# show linear decision boundary
+figure = plt.figure()
+ax = Axes3D(figure, elev=-152, azim=-26)
+xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
+yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
+XX, YY = np.meshgrid(xx, yy)
+ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
+ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
+
+ZZ = YY ** 2
+dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
+plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
+ cmap=mglearn.cm2, alpha=0.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
from sklearn.svm import SVC
-poly_kernel_svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
-))
-poly_kernel_svm_clf.fit(X, y)
+X, y = mglearn.tools.make_handcrafted_dataset()
+svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
+mglearn.plots.plot_2d_separator(svm, X, eps=.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+# plot support vectors
+sv = svm.support_vectors_
+# class labels of support vectors are given by the sign of the dual coefficients
+sv_labels = svm.dual_coef_.ravel() > 0
+mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
+fig, axes = plt.subplots(3, 3, figsize=(15, 10))
+
+for ax, C in zip(axes, [-1, 0, 3]):
+ for a, gamma in zip(ax, range(-1, 2)):
+ mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
+
+axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
+ ncol=4, loc=(.9, 1.2))
-This code trains an SVM classifier using a 3rd-degree polynomial kernel.
-
-
-
-
-
SVMs, regression
-
-
-You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
-shows how to use the regression option (more material to come)
-
-
-
-
from sklearn.svm import LinearSVR
-svm_reg = LinearSVR(epsilon=1.5)
-svm_reg.fit(X, y)
-
-
-To tackle nonlinear regression tasks, you can use a kernelized SVM model
diff --git a/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz b/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz
index d5c9e5f73..982e68638 100644
Binary files a/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz and b/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz differ
diff --git a/doc/pub/svm/ipynb/svm.ipynb b/doc/pub/svm/ipynb/svm.ipynb
index 2876278d5..77e0cf0e0 100644
--- a/doc/pub/svm/ipynb/svm.ipynb
+++ b/doc/pub/svm/ipynb/svm.ipynb
@@ -10,7 +10,7 @@
" \n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
"\n",
- "Date: **Nov 1, 2018**\n",
+ "Date: **Nov 2, 2018**\n",
"\n",
"Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
"\n",
@@ -34,11 +34,13 @@
"\n",
"**These notes will be updated shortly with more material**\n",
"\n",
- "## SVMs, basics with scikit-learn\n",
"\n",
- "The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM\n",
- "model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect\n",
- "Iris-Virginica flowers."
+ "## Strength and weakness\n",
+ "When we implement a linear support vector machine, the main parameter is the constant $C$. Small values of $C$ mean simple models.\n",
+ "These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions. \n",
+ "\n",
+ "\n",
+ "## Examples with kernels"
]
},
{
@@ -49,38 +51,34 @@
},
"outputs": [],
"source": [
+ "%matplotlib inline\n",
+ "\n",
+ "from IPython.display import set_matplotlib_formats, display\n",
+ "import pandas as pd\n",
"import numpy as np\n",
- "from sklearn import datasets\n",
- "from sklearn.pipeline import Pipeline\n",
- "from sklearn.preprocessing import StandardScaler\n",
+ "import matplotlib.pyplot as plt\n",
+ "import mglearn\n",
+ "from cycler import cycler\n",
+ "from sklearn.linear_model import LogisticRegression\n",
"from sklearn.svm import LinearSVC\n",
- "iris = datasets.load_iris()\n",
- "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n",
- "y = (iris[\"target\"] == 2).astype(np.float64) # Iris-Virginica\n",
- "svm_clf = Pipeline((\n",
- "(\"scaler\", StandardScaler()),\n",
- "(\"linear_svc\", LinearSVC(C=1, loss=\"hinge\")),\n",
- "))\n",
- "svm_clf.fit(X_scaled, y)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Alternatively, you could use the SVC class, using **SVC(kernel=\"linear\", C=1)**, but it is much slower,\n",
- "especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier\n",
- "class, with **SGDClassifier(loss=\"hinge\", alpha=1/(m*C))**. This applies regular Stochastic\n",
- "Gradient Descentto train a linear SVM classifier. It does not converge as fast as the\n",
- "LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core\n",
- "training), or to handle online classification tasks.\n",
+ "from sklearn.datasets import make_blobs\n",
"\n",
- "## SVMs, adding polynomial features\n",
"\n",
- "Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets\n",
- "are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more\n",
- "features, such as polynomial features. In some cases this can result in a linearly\n",
- "separable dataset."
+ "X, y = make_blobs(centers=4, random_state=8)\n",
+ "y = y % 2\n",
+ "\n",
+ "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
+ "plt.xlabel(\"Feature 0\")\n",
+ "plt.ylabel(\"Feature 1\")\n",
+ "plt.show()\n",
+ "\n",
+ "from sklearn.svm import LinearSVC\n",
+ "linear_svm = LinearSVC().fit(X, y)\n",
+ "\n",
+ "mglearn.plots.plot_2d_separator(linear_svm, X)\n",
+ "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
+ "plt.xlabel(\"Feature 0\")\n",
+ "plt.ylabel(\"Feature 1\")"
]
},
{
@@ -91,31 +89,23 @@
},
"outputs": [],
"source": [
- "from sklearn.datasets import make_moons\n",
- "from sklearn.pipeline import Pipeline\n",
- "from sklearn.preprocessing import PolynomialFeatures\n",
- "polynomial_svm_clf = Pipeline((\n",
- "(\"poly_features\", PolynomialFeatures(degree=3)),\n",
- "(\"scaler\", StandardScaler()),\n",
- "(\"svm_clf\", LinearSVC(C=10, loss=\"hinge\"))\n",
- "))\n",
- "polynomial_svm_clf.fit(X, y)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## SVMs, polynomials and kernels\n",
+ "# add the squared first feature\n",
+ "X_new = np.hstack([X, X[:, 1:] ** 2])\n",
"\n",
- "Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning\n",
- "algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,\n",
- "and with a high polynomial degree it creates a huge number of features, making the model too slow.\n",
- "Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the\n",
- "kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many\n",
- "polynomial features, even with very high-degree polynomials, without actually having to add them. So\n",
- "there is no combinatorial explosion of the number of features since you don’t actually add any features.\n",
- "This trick is implemented by the SVC class."
+ "\n",
+ "from mpl_toolkits.mplot3d import Axes3D, axes3d\n",
+ "figure = plt.figure()\n",
+ "# visualize in 3D\n",
+ "ax = Axes3D(figure, elev=-152, azim=-26)\n",
+ "# plot first all the points with y==0, then all with y == 1\n",
+ "mask = y == 0\n",
+ "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
+ " cmap=mglearn.cm2, s=60, edgecolor='k')\n",
+ "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
+ " cmap=mglearn.cm2, s=60, edgecolor='k')\n",
+ "ax.set_xlabel(\"feature0\")\n",
+ "ax.set_ylabel(\"feature1\")\n",
+ "ax.set_zlabel(\"feature1 ** 2\")"
]
},
{
@@ -126,25 +116,34 @@
},
"outputs": [],
"source": [
- "from sklearn.svm import SVC\n",
- "poly_kernel_svm_clf = Pipeline((\n",
- "(\"scaler\", StandardScaler()),\n",
- "(\"svm_clf\", SVC(kernel=\"poly\", degree=3, coef0=1, C=5))\n",
- "))\n",
- "poly_kernel_svm_clf.fit(X, y)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This code trains an SVM classifier using a 3rd-degree polynomial kernel.\n",
+ "linear_svm_3d = LinearSVC().fit(X_new, y)\n",
+ "coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_\n",
"\n",
+ "# show linear decision boundary\n",
+ "figure = plt.figure()\n",
+ "ax = Axes3D(figure, elev=-152, azim=-26)\n",
+ "xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)\n",
+ "yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)\n",
"\n",
- "## SVMs, regression\n",
+ "XX, YY = np.meshgrid(xx, yy)\n",
+ "ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]\n",
+ "ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)\n",
+ "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
+ " cmap=mglearn.cm2, s=60, edgecolor='k')\n",
+ "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
+ " cmap=mglearn.cm2, s=60, edgecolor='k')\n",
"\n",
- "You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code\n",
- "shows how to use the regression option (more material to come)"
+ "ax.set_xlabel(\"feature0\")\n",
+ "ax.set_ylabel(\"feature1\")\n",
+ "ax.set_zlabel(\"feature1 ** 2\")\n",
+ "\n",
+ "ZZ = YY ** 2\n",
+ "dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])\n",
+ "plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],\n",
+ " cmap=mglearn.cm2, alpha=0.5)\n",
+ "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
+ "plt.xlabel(\"Feature 0\")\n",
+ "plt.ylabel(\"Feature 1\")"
]
},
{
@@ -155,16 +154,27 @@
},
"outputs": [],
"source": [
- "from sklearn.svm import LinearSVR\n",
- "svm_reg = LinearSVR(epsilon=1.5)\n",
- "svm_reg.fit(X, y)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "To tackle nonlinear regression tasks, you can use a kernelized SVM model"
+ "from sklearn.svm import SVC\n",
+ "X, y = mglearn.tools.make_handcrafted_dataset() \n",
+ "svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)\n",
+ "mglearn.plots.plot_2d_separator(svm, X, eps=.5)\n",
+ "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
+ "# plot support vectors\n",
+ "sv = svm.support_vectors_\n",
+ "# class labels of support vectors are given by the sign of the dual coefficients\n",
+ "sv_labels = svm.dual_coef_.ravel() > 0\n",
+ "mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)\n",
+ "plt.xlabel(\"Feature 0\")\n",
+ "plt.ylabel(\"Feature 1\")\n",
+ "\n",
+ "fig, axes = plt.subplots(3, 3, figsize=(15, 10))\n",
+ "\n",
+ "for ax, C in zip(axes, [-1, 0, 3]):\n",
+ " for a, gamma in zip(ax, range(-1, 2)):\n",
+ " mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)\n",
+ " \n",
+ "axes[0, 0].legend([\"class 0\", \"class 1\", \"sv class 0\", \"sv class 1\"],\n",
+ " ncol=4, loc=(.9, 1.2))"
]
}
],
diff --git a/doc/pub/svm/pdf/svm-beamer-handouts2x3.pdf b/doc/pub/svm/pdf/svm-beamer-handouts2x3.pdf
index 9ca94c518..b7b72e45b 100644
Binary files a/doc/pub/svm/pdf/svm-beamer-handouts2x3.pdf and b/doc/pub/svm/pdf/svm-beamer-handouts2x3.pdf differ
diff --git a/doc/pub/svm/pdf/svm-beamer.pdf b/doc/pub/svm/pdf/svm-beamer.pdf
index a6d8ec491..8d15a7898 100644
Binary files a/doc/pub/svm/pdf/svm-beamer.pdf and b/doc/pub/svm/pdf/svm-beamer.pdf differ
diff --git a/doc/pub/svm/pdf/svm-minted.pdf b/doc/pub/svm/pdf/svm-minted.pdf
index a773ef0fd..d68d6be07 100644
Binary files a/doc/pub/svm/pdf/svm-minted.pdf and b/doc/pub/svm/pdf/svm-minted.pdf differ
diff --git a/doc/src/SupportVMachines/svm.do.txt b/doc/src/SupportVMachines/svm.do.txt
index c9777f557..c67f69955 100644
--- a/doc/src/SupportVMachines/svm.do.txt
+++ b/doc/src/SupportVMachines/svm.do.txt
@@ -21,87 +21,117 @@ We distringuish also between linearn and non-linear approaches.
_These notes will be updated shortly with more material_
-!split
-===== SVMs, basics with scikit-learn =====
-The following Scikit-Learn code loads the iris dataset, scales the features, and then trains a linear SVM
-model (using the LinearSVC class with C = 0.1 and the hinge loss function, described shortly) to detect
-Iris-Virginica flowers.
+!split
+===== Strength and weakness =====
+When we implement a linear support vector machine, the main parameter is the constant $C$. Small values of $C$ mean simple models.
+These models are fast to train and also fast to predict and scale to very large data sets and work well with sparse data. Linear support vector machines make it easy to understand how a prediction is made, however it is often not easy to understand why coefficients are the way they are. These models work also well in higer dimensions.
+
+
+!split
+===== Examples with kernels =====
+
!bc pycod
+from IPython.display import set_matplotlib_formats, display
+import pandas as pd
import numpy as np
-from sklearn import datasets
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+import matplotlib.pyplot as plt
+import mglearn
+from cycler import cycler
+from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
-iris = datasets.load_iris()
-X = iris["data"][:, (2, 3)] # petal length, petal width
-y = (iris["target"] == 2).astype(np.float64) # Iris-Virginica
-svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("linear_svc", LinearSVC(C=1, loss="hinge")),
-))
-svm_clf.fit(X_scaled, y)
+from sklearn.datasets import make_blobs
+
+
+X, y = make_blobs(centers=4, random_state=8)
+y = y % 2
+
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+plt.show()
+
+from sklearn.svm import LinearSVC
+linear_svm = LinearSVC().fit(X, y)
+
+mglearn.plots.plot_2d_separator(linear_svm, X)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
!ec
-Alternatively, you could use the SVC class, using _SVC(kernel="linear", C=1)_, but it is much slower,
-especially with large training sets, so it is not recommended. Another option is to use the SGDClassifier
-class, with _SGDClassifier(loss="hinge", alpha=1/(m*C))_. This applies regular Stochastic
-Gradient Descentto train a linear SVM classifier. It does not converge as fast as the
-LinearSVC class, but it can be useful to handle huge datasets that do not fit in memory (out-of-core
-training), or to handle online classification tasks.
-
-!split
-===== SVMs, adding polynomial features =====
-
-Although linear SVM classifiers are efficient and work surprisingly well in many cases, many datasets
-are not even close to being linearly separable. One approach to handling nonlinear datasets is to add more
-features, such as polynomial features. In some cases this can result in a linearly
-separable dataset.
-
!bc pycod
-from sklearn.datasets import make_moons
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-polynomial_svm_clf = Pipeline((
-("poly_features", PolynomialFeatures(degree=3)),
-("scaler", StandardScaler()),
-("svm_clf", LinearSVC(C=10, loss="hinge"))
-))
-polynomial_svm_clf.fit(X, y)
+# add the squared first feature
+X_new = np.hstack([X, X[:, 1:] ** 2])
+
+
+from mpl_toolkits.mplot3d import Axes3D, axes3d
+figure = plt.figure()
+# visualize in 3D
+ax = Axes3D(figure, elev=-152, azim=-26)
+# plot first all the points with y==0, then all with y == 1
+mask = y == 0
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
!ec
+!bc pycod
+linear_svm_3d = LinearSVC().fit(X_new, y)
+coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_
-!split
-===== SVMs, polynomials and kernels =====
+# show linear decision boundary
+figure = plt.figure()
+ax = Axes3D(figure, elev=-152, azim=-26)
+xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)
+yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)
-Adding polynomial features is simple to implement and can work great with all sorts of Machine Learning
-algorithms (not just SVMs), but at a low polynomial degree it cannot deal with very complex datasets,
-and with a high polynomial degree it creates a huge number of features, making the model too slow.
-Fortunately, when using SVMs you can apply an almost miraculous mathematical technique called the
-kernel trick discussed during the lectures. It makes it possible to get the same result as if you added many
-polynomial features, even with very high-degree polynomials, without actually having to add them. So
-there is no combinatorial explosion of the number of features since you don’t actually add any features.
-This trick is implemented by the SVC class.
+XX, YY = np.meshgrid(xx, yy)
+ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]
+ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)
+ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',
+ cmap=mglearn.cm2, s=60, edgecolor='k')
+
+ax.set_xlabel("feature0")
+ax.set_ylabel("feature1")
+ax.set_zlabel("feature1 ** 2")
+
+ZZ = YY ** 2
+dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])
+plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],
+ cmap=mglearn.cm2, alpha=0.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+!ec
!bc pycod
from sklearn.svm import SVC
-poly_kernel_svm_clf = Pipeline((
-("scaler", StandardScaler()),
-("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
-))
-poly_kernel_svm_clf.fit(X, y)
+X, y = mglearn.tools.make_handcrafted_dataset()
+svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)
+mglearn.plots.plot_2d_separator(svm, X, eps=.5)
+mglearn.discrete_scatter(X[:, 0], X[:, 1], y)
+# plot support vectors
+sv = svm.support_vectors_
+# class labels of support vectors are given by the sign of the dual coefficients
+sv_labels = svm.dual_coef_.ravel() > 0
+mglearn.discrete_scatter(sv[:, 0], sv[:, 1], sv_labels, s=15, markeredgewidth=3)
+plt.xlabel("Feature 0")
+plt.ylabel("Feature 1")
+
+fig, axes = plt.subplots(3, 3, figsize=(15, 10))
+
+for ax, C in zip(axes, [-1, 0, 3]):
+ for a, gamma in zip(ax, range(-1, 2)):
+ mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)
+
+axes[0, 0].legend(["class 0", "class 1", "sv class 0", "sv class 1"],
+ ncol=4, loc=(.9, 1.2))
+
!ec
-This code trains an SVM classifier using a 3rd-degree polynomial kernel.
-
-
-!split
-===== SVMs, regression =====
-
-You can use Scikit-Learn’s LinearSVR class to perform linear SVM Regression. The following code
-shows how to use the regression option (more material to come)
-!bc pycod
-from sklearn.svm import LinearSVR
-svm_reg = LinearSVR(epsilon=1.5)
-svm_reg.fit(X, y)
-!ec
-To tackle nonlinear regression tasks, you can use a kernelized SVM model