-
@@ -194,7 +196,7 @@ MathJax.Hub.Config({
We assume here that our data set can be well separated into two domains, where a straight line does the job in the separating the two -classes. Here the two classes are represented by either crosses or +classes. Here the two classes are represented by either squares or circles. +
+ +
from sklearn import datasets
+from sklearn.svm import SVC, LinearSVC
+from sklearn.linear_model import SGDClassifier
+from sklearn.preprocessing import StandardScaler
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+iris = datasets.load_iris()
+X = iris["data"][:, (2, 3)] # petal length, petal width
+y = iris["target"]
+
+setosa_or_versicolor = (y == 0) | (y == 1)
+X = X[setosa_or_versicolor]
+y = y[setosa_or_versicolor]
+
+
+
+C = 5
+alpha = 1 / (C * len(X))
+
+lin_clf = LinearSVC(loss="hinge", C=C, random_state=42)
+svm_clf = SVC(kernel="linear", C=C)
+sgd_clf = SGDClassifier(loss="hinge", learning_rate="constant", eta0=0.001, alpha=alpha,
+ max_iter=100000, random_state=42)
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+lin_clf.fit(X_scaled, y)
+svm_clf.fit(X_scaled, y)
+sgd_clf.fit(X_scaled, y)
+
+print("LinearSVC: ", lin_clf.intercept_, lin_clf.coef_)
+print("SVC: ", svm_clf.intercept_, svm_clf.coef_)
+print("SGDClassifier(alpha={:.5f}):".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
+
+# Compute the slope and bias of each decision boundary
+w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]
+b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]
+w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]
+b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]
+w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]
+b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]
+
+# Transform the decision boundary lines back to the original scale
+line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])
+line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])
+line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])
+
+# Plot all three decision boundaries
+plt.figure(figsize=(11, 4))
+plt.plot(line1[:, 0], line1[:, 1], "k:", label="LinearSVC")
+plt.plot(line2[:, 0], line2[:, 1], "b--", linewidth=2, label="SVC")
+plt.plot(line3[:, 0], line3[:, 1], "r-", label="SGDClassifier")
+plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs") # label="Iris-Versicolor"
+plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo") # label="Iris-Setosa"
+plt.xlabel("Petal length", fontsize=14)
+plt.ylabel("Petal width", fontsize=14)
+plt.legend(loc="upper center", fontsize=14)
+plt.axis([0, 5.5, 0, 2])
+
+plt.show()
+
@@ -184,7 +255,7 @@ circles.
+ + +
import numpy as np
+import os
+
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
+
@@ -195,6 +247,7 @@ we need to introduce for example a polynomial transformation to a two-dimensiona
-A mathematical (quadratic) optimization problem, or just optimization problem, has the form -$$ -\begin{align*} - &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber - &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f. -\end{align*} -$$ -subject to some constraints for say a selected set \( i=1,2,\dots, n \). -In our case we are optimizing with respect to the Lagrangian multipliers \( \lambda_i \), and the -vector \( \boldsymbol{\lambda}=[\lambda_1, \lambda_2,\dots, \lambda_n] \) is the optimization variable we are dealing with. + +
from __future__ import division, print_function, unicode_literals
-
-In our case we are particularly interested in a class of optimization problems called convex optmization problems.
-In our discussion on gradient descent methods we discussed at length the definition of a convex function.
+import numpy as np
+np.random.seed(42)
-
-Convex optimization problems play a central role in applied mathematics and we recommend strongly Boyd and Vandenberghe's text on the topics.
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
@@ -193,6 +366,7 @@ Convex optimization problems play a central role in applied mathematics and we r
-If we use Python as programming language and wish to venture beyond -scikit-learn, tensorflow and similar software which makes our -lives so much easier, we need to dive into the wonderful world of -quadratic programming. We can, if we wish, solve the minimization -problem using say standard gradient methods or conjugate gradient -methods. However, these methods tend to exhibit a rather slow -converge. So, welcome to the promised land of quadratic programming. +A mathematical (quadratic) optimization problem, or just optimization problem, has the form +$$ +\begin{align*} + &\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber + &\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f. +\end{align*} +$$ + +subject to some constraints for say a selected set \( i=1,2,\dots, n \). +In our case we are optimizing with respect to the Lagrangian multipliers \( \lambda_i \), and the +vector \( \boldsymbol{\lambda}=[\lambda_1, \lambda_2,\dots, \lambda_n] \) is the optimization variable we are dealing with.
-The functions we need are contained in the quadratic programming package CVXOPT and we need to import it together with numpy as +In our case we are particularly interested in a class of optimization problems called convex optmization problems. +In our discussion on gradient descent methods we discussed at length the definition of a convex function.
- - -
import numpy
-import cvxopt
--This will make our life much easier. You don't need t write your own optimizer. +Convex optimization problems play a central role in applied mathematics and we recommend strongly Boyd and Vandenberghe's text on the topics.
@@ -193,6 +194,7 @@ This will make our life much easier. You don't need t write your own optimizer.
-We remind ourselves about the general problem we want to solve -$$ -\begin{align*} - &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}\boldsymbol{x}^T\boldsymbol{P}\boldsymbol{x}+\boldsymbol{q}^T\boldsymbol{x},\\ \nonumber - &\mathrm{subject\hspace{0.1cm} to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{x} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{x}=f. -\end{align*} -$$ +If we use Python as programming language and wish to venture beyond +scikit-learn, tensorflow and similar software which makes our +lives so much easier, we need to dive into the wonderful world of +quadratic programming. We can, if we wish, solve the minimization +problem using say standard gradient methods or conjugate gradient +methods. However, these methods tend to exhibit a rather slow +converge. So, welcome to the promised land of quadratic programming.
-Let us show how to perform the optmization using a simple case. Assume we want to optimize the following problem -$$ -\begin{align*} - &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}x^2+5x+3y \\ \nonumber - &\mathrm{subject to} \\ \nonumber - &x, y \geq 0 \\ \nonumber - &x+3y \geq 15 \\ \nonumber - &2x+5y \leq 100 \\ \nonumber - &3x+4y \leq 80. \\ \nonumber -\end{align*} -$$ +The functions we need are contained in the quadratic programming package CVXOPT and we need to import it together with numpy as -The minimization problem can be rewritten in terms of vectors and matrices as (with \( x \) and \( y \) being the unknowns) -$$ -\frac{1}{2}\begin{bmatrix} x\\ y \end{bmatrix}^T \begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} + \begin{bmatrix}3\\ 4 \end{bmatrix}^T \begin{bmatrix}x \\ y \end{bmatrix}. -$$ - -Similarly, we can now set up the inequalities (we need to change \( \geq \) to \( \leq \) by multiplying with \( -1 \) on bot sides) as the following matrix-vector equation -$$ -\begin{bmatrix} -1 & 0 \\ 0 & -1 \\ -1 & -3 \\ 2 & 5 \\ 3 & 4\end{bmatrix}\begin{bmatrix} x \\ y\end{bmatrix} \preceq \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}. -$$ - -We have collapsed all the inequalities into a single matrix \( \boldsymbol{G} \). We see also that our matrix -$$ -\boldsymbol{P} =\begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} -$$ - -is clearly positive semi-definite (all eigenvalues larger or equal zero). -Finally, the vector \( \boldsymbol{h} \) is defined as -$$ -\boldsymbol{h} = \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}. -$$ - -
-Since we don't have any equalities the matrix \( \boldsymbol{A} \) is set to zero -The following code solves the equations for us
-
# Import the necessary packages
-import numpy
-from cvxopt import matrix
-from cvxopt import solvers
-P = matrix(numpy.diag([1,0]), tc=’d’)
-q = matrix(numpy.array([3,4]), tc=’d’)
-G = matrix(numpy.array([[-1,0],[0,-1],[-1,-3],[2,5],[3,4]]), tc=’d’)
-h = matrix(numpy.array([0,0,-15,100,80]), tc=’d’)
-# Construct the QP, invoke solver
-sol = solvers.qp(P,q,G,h)
-# Extract optimal value and solution
-sol[’x’]
-sol[’primal objective’]
+import numpy
+import cvxopt
+
+This will make our life much easier. You don't need t write your own optimizer.
+
@@ -234,6 +194,7 @@ sol[’primal objective’]
-We are now ready to return to our setup of the optmization problem for a more realistic case. Introducint the slack parameter \( C \) we have +We remind ourselves about the general problem we want to solve $$ -\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\ -y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2K(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\ -\dots & \dots & \dots & \dots & \dots \\ -\dots & \dots & \dots & \dots & \dots \\ -y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\ -\end{bmatrix}\boldsymbol{\lambda}-\mathbb{I}\boldsymbol{\lambda}, +\begin{align*} + &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}\boldsymbol{x}^T\boldsymbol{P}\boldsymbol{x}+\boldsymbol{q}^T\boldsymbol{x},\\ \nonumber + &\mathrm{subject\hspace{0.1cm} to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{x} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{x}=f. +\end{align*} $$ -subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and -\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \). -With the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \). -
-code will be added +Let us show how to perform the optmization using a simple case. Assume we want to optimize the following problem +$$ +\begin{align*} + &\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}x^2+5x+3y \\ \nonumber + &\mathrm{subject to} \\ \nonumber + &x, y \geq 0 \\ \nonumber + &x+3y \geq 15 \\ \nonumber + &2x+5y \leq 100 \\ \nonumber + &3x+4y \leq 80. \\ \nonumber +\end{align*} +$$ +The minimization problem can be rewritten in terms of vectors and matrices as (with \( x \) and \( y \) being the unknowns) +$$ +\frac{1}{2}\begin{bmatrix} x\\ y \end{bmatrix}^T \begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} + \begin{bmatrix}3\\ 4 \end{bmatrix}^T \begin{bmatrix}x \\ y \end{bmatrix}. +$$ + +Similarly, we can now set up the inequalities (we need to change \( \geq \) to \( \leq \) by multiplying with \( -1 \) on bot sides) as the following matrix-vector equation +$$ +\begin{bmatrix} -1 & 0 \\ 0 & -1 \\ -1 & -3 \\ 2 & 5 \\ 3 & 4\end{bmatrix}\begin{bmatrix} x \\ y\end{bmatrix} \preceq \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}. +$$ + +We have collapsed all the inequalities into a single matrix \( \boldsymbol{G} \). We see also that our matrix +$$ +\boldsymbol{P} =\begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} +$$ + +is clearly positive semi-definite (all eigenvalues larger or equal zero). +Finally, the vector \( \boldsymbol{h} \) is defined as +$$ +\boldsymbol{h} = \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}. +$$ + +
+Since we don't have any equalities the matrix \( \boldsymbol{A} \) is set to zero +The following code solves the equations for us +
+ + +
# Import the necessary packages
+import numpy
+from cvxopt import matrix
+from cvxopt import solvers
+P = matrix(numpy.diag([1,0]), tc=’d’)
+q = matrix(numpy.array([3,4]), tc=’d’)
+G = matrix(numpy.array([[-1,0],[0,-1],[-1,-3],[2,5],[3,4]]), tc=’d’)
+h = matrix(numpy.array([0,0,-15,100,80]), tc=’d’)
+# Construct the QP, invoke solver
+sol = solvers.qp(P,q,G,h)
+# Extract optimal value and solution
+sol[’x’]
+sol[’primal objective’]
+
@@ -188,6 +235,7 @@ With the slack constants this leads to the additional constraint \( 0\leq \lamb
+We are now ready to return to our setup of the optmization problem for a more realistic case. Introducint the slack parameter \( C \) we have +$$ +\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\ +y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2K(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\ +\dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots \\ +y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\ +\end{bmatrix}\boldsymbol{\lambda}-\mathbb{I}\boldsymbol{\lambda}, +$$ + +subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and +\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \). +With the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \). + +
+code will be added + +
diff --git a/doc/pub/svm/html/._svm-bs026.html b/doc/pub/svm/html/._svm-bs026.html new file mode 100644 index 000000000..256c542f6 --- /dev/null +++ b/doc/pub/svm/html/._svm-bs026.html @@ -0,0 +1,197 @@ + + + + + + + +
+ + + + +
+ +
+ + +-
@@ -194,7 +196,7 @@ MathJax.Hub.Config({
-
@@ -200,13 +200,83 @@ The theory behind support vector machines (SVM hereafter) is based on the mathematical description of so-called hyperplanes. Let us start with a two-dimensional case. This will also allow us to introduce our first SVM examples. These will be tailored to the case of two specific -classes, as displayed in the figure here. +classes, as displayed in the figure here based on the usage of the petal data.
We assume here that our data set can be well separated into two domains, where a straight line does the job in the separating the two -classes. Here the two classes are represented by either crosses or +classes. Here the two classes are represented by either squares or circles. +
+ + +
from sklearn import datasets
+from sklearn.svm import SVC, LinearSVC
+from sklearn.linear_model import SGDClassifier
+from sklearn.preprocessing import StandardScaler
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+iris = datasets.load_iris()
+X = iris["data"][:, (2, 3)] # petal length, petal width
+y = iris["target"]
+
+setosa_or_versicolor = (y == 0) | (y == 1)
+X = X[setosa_or_versicolor]
+y = y[setosa_or_versicolor]
+
+
+
+C = 5
+alpha = 1 / (C * len(X))
+
+lin_clf = LinearSVC(loss="hinge", C=C, random_state=42)
+svm_clf = SVC(kernel="linear", C=C)
+sgd_clf = SGDClassifier(loss="hinge", learning_rate="constant", eta0=0.001, alpha=alpha,
+ max_iter=100000, random_state=42)
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+lin_clf.fit(X_scaled, y)
+svm_clf.fit(X_scaled, y)
+sgd_clf.fit(X_scaled, y)
+
+print("LinearSVC: ", lin_clf.intercept_, lin_clf.coef_)
+print("SVC: ", svm_clf.intercept_, svm_clf.coef_)
+print("SGDClassifier(alpha={:.5f}):".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
+
+# Compute the slope and bias of each decision boundary
+w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]
+b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]
+w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]
+b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]
+w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]
+b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]
+
+# Transform the decision boundary lines back to the original scale
+line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])
+line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])
+line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])
+
+# Plot all three decision boundaries
+plt.figure(figsize=(11, 4))
+plt.plot(line1[:, 0], line1[:, 1], "k:", label="LinearSVC")
+plt.plot(line2[:, 0], line2[:, 1], "b--", linewidth=2, label="SVC")
+plt.plot(line3[:, 0], line3[:, 1], "r-", label="SGDClassifier")
+plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs") # label="Iris-Versicolor"
+plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo") # label="Iris-Setosa"
+plt.xlabel("Petal length", fontsize=14)
+plt.ylabel("Petal width", fontsize=14)
+plt.legend(loc="upper center", fontsize=14)
+plt.axis([0, 5.5, 0, 2])
+
+plt.show()
+The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that we need to introduce for example a polynomial transformation to a two-dimensional training set. + +
+ + +
import numpy as np
+import os
+
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
++ + +
from __future__ import division, print_function, unicode_literals
+
+import numpy as np
+np.random.seed(42)
+
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -987,7 +1305,7 @@ Convex optimization problems play a central role in applied mathematics and we r
If we use Python as programming language and wish to venture beyond
@@ -1013,7 +1331,7 @@ This will make our life much easier. You don't need t write your own optimizer.
We remind ourselves about the general problem we want to solve
@@ -1094,7 +1412,7 @@ sol[’primal obj
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducint the slack parameter \( C \) we have
@@ -1119,7 +1437,7 @@ With the slack constants this leads to the additional constraint \( 0\leq \lamb
-
We assume here that our data set can be well separated into two
domains, where a straight line does the job in the separating the two
-classes. Here the two classes are represented by either crosses or
+classes. Here the two classes are represented by either squares or
circles.
+
+
+
+
+
+
+
+
+
+
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -814,7 +1130,7 @@ Convex optimization problems play a central role in applied mathematics and we r
If we use Python as programming language and wish to venture beyond
@@ -840,7 +1156,7 @@ This will make our life much easier. You don't need t write your own optimizer.
We remind ourselves about the general problem we want to solve
@@ -908,7 +1224,7 @@ sol[’primal obj
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducint the slack parameter \( C \) we have
@@ -931,7 +1247,7 @@ With the slack constants this leads to the additional constraint \( 0\leq \lamb
-
We assume here that our data set can be well separated into two
domains, where a straight line does the job in the separating the two
-classes. Here the two classes are represented by either crosses or
+classes. Here the two classes are represented by either squares or
circles.
+
+
+
+
+
+
+
+
+
+
A mathematical (quadratic) optimization problem, or just optimization problem, has the form
@@ -819,7 +1135,7 @@ Convex optimization problems play a central role in applied mathematics and we r
If we use Python as programming language and wish to venture beyond
@@ -845,7 +1161,7 @@ This will make our life much easier. You don't need t write your own optimizer.
We remind ourselves about the general problem we want to solve
@@ -913,7 +1229,7 @@ sol[’primal objective’]
We are now ready to return to our setup of the optmization problem for a more realistic case. Introducint the slack parameter \( C \) we have
@@ -936,7 +1252,7 @@ With the slack constants this leads to the additional constraint \( 0\leq \lamb
How do we solve these problems?
+How do we solve these problems?
A simplex example
+A simplex example
Back to the more realistic cases
+Back to the more realistic cases
Multiclass problems and regression with SVMs
+Multiclass problems and regression with SVMs
This material will be added later.
Nov 7, 2018
Nov 8, 2018
@@ -155,14 +156,83 @@ The theory behind support vector machines (SVM hereafter) is based on
the mathematical description of so-called hyperplanes. Let us start
with a two-dimensional case. This will also allow us to introduce our
first SVM examples. These will be tailored to the case of two specific
-classes, as displayed in the figure here.
+classes, as displayed in the figure here based on the usage of the petal data.
from sklearn import datasets
+from sklearn.svm import SVC, LinearSVC
+from sklearn.linear_model import SGDClassifier
+from sklearn.preprocessing import StandardScaler
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+iris = datasets.load_iris()
+X = iris["data"][:, (2, 3)] # petal length, petal width
+y = iris["target"]
+
+setosa_or_versicolor = (y == 0) | (y == 1)
+X = X[setosa_or_versicolor]
+y = y[setosa_or_versicolor]
+
+
+
+C = 5
+alpha = 1 / (C * len(X))
+
+lin_clf = LinearSVC(loss="hinge", C=C, random_state=42)
+svm_clf = SVC(kernel="linear", C=C)
+sgd_clf = SGDClassifier(loss="hinge", learning_rate="constant", eta0=0.001, alpha=alpha,
+ max_iter=100000, random_state=42)
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+lin_clf.fit(X_scaled, y)
+svm_clf.fit(X_scaled, y)
+sgd_clf.fit(X_scaled, y)
+
+print("LinearSVC: ", lin_clf.intercept_, lin_clf.coef_)
+print("SVC: ", svm_clf.intercept_, svm_clf.coef_)
+print("SGDClassifier(alpha={:.5f}):".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
+
+# Compute the slope and bias of each decision boundary
+w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]
+b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]
+w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]
+b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]
+w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]
+b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]
+
+# Transform the decision boundary lines back to the original scale
+line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])
+line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])
+line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])
+
+# Plot all three decision boundaries
+plt.figure(figsize=(11, 4))
+plt.plot(line1[:, 0], line1[:, 1], "k:", label="LinearSVC")
+plt.plot(line2[:, 0], line2[:, 1], "b--", linewidth=2, label="SVC")
+plt.plot(line3[:, 0], line3[:, 1], "r-", label="SGDClassifier")
+plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs") # label="Iris-Versicolor"
+plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo") # label="Iris-Setosa"
+plt.xlabel("Petal length", fontsize=14)
+plt.ylabel("Petal width", fontsize=14)
+plt.legend(loc="upper center", fontsize=14)
+plt.axis([0, 5.5, 0, 2])
+
+plt.show()
+
@@ -677,6 +747,56 @@ obtain a separation between the classes which is almost linear.
The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that
we need to introduce for example a polynomial transformation to a two-dimensional training set.
+import numpy as np
+import os
+
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
+
@@ -789,7 +909,203 @@ well in practice.
-Mathematical optimization of convex functions
+The moons example
+from __future__ import division, print_function, unicode_literals
+
+import numpy as np
+np.random.seed(42)
+
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
+
+Mathematical optimization of convex functions
-How do we solve these problems?
+How do we solve these problems?
-A simplex example
+A simplex example
-Back to the more realistic cases
+Back to the more realistic cases
-Multiclass problems and regression with SVMs
+Multiclass problems and regression with SVMs
This material will be added later.
diff --git a/doc/pub/svm/html/svm.html b/doc/pub/svm/html/svm.html
index 7b4651af8..a41c5358e 100644
--- a/doc/pub/svm/html/svm.html
+++ b/doc/pub/svm/html/svm.html
@@ -62,17 +62,18 @@ div { text-align: justify; text-justify: inter-word; }
('The equations', 2, None, '___sec17'),
('The problem to solve', 2, None, '___sec18'),
("Different kernels and Mercer's theorem", 2, None, '___sec19'),
+ ('The moons example', 2, None, '___sec20'),
('Mathematical optimization of convex functions',
2,
None,
- '___sec20'),
- ('How do we solve these problems?', 2, None, '___sec21'),
- ('A simplex example', 2, None, '___sec22'),
- ('Back to the more realistic cases', 2, None, '___sec23'),
+ '___sec21'),
+ ('How do we solve these problems?', 2, None, '___sec22'),
+ ('A simplex example', 2, None, '___sec23'),
+ ('Back to the more realistic cases', 2, None, '___sec24'),
('Multiclass problems and regression with SVMs',
2,
None,
- '___sec24')]}
+ '___sec25')]}
end of tocinfo -->
@@ -114,7 +115,7 @@ MathJax.Hub.Config({
Nov 7, 2018
Nov 8, 2018
@@ -160,14 +161,83 @@ The theory behind support vector machines (SVM hereafter) is based on
the mathematical description of so-called hyperplanes. Let us start
with a two-dimensional case. This will also allow us to introduce our
first SVM examples. These will be tailored to the case of two specific
-classes, as displayed in the figure here.
+classes, as displayed in the figure here based on the usage of the petal data.
from sklearn import datasets
+from sklearn.svm import SVC, LinearSVC
+from sklearn.linear_model import SGDClassifier
+from sklearn.preprocessing import StandardScaler
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+iris = datasets.load_iris()
+X = iris["data"][:, (2, 3)] # petal length, petal width
+y = iris["target"]
+
+setosa_or_versicolor = (y == 0) | (y == 1)
+X = X[setosa_or_versicolor]
+y = y[setosa_or_versicolor]
+
+
+
+C = 5
+alpha = 1 / (C * len(X))
+
+lin_clf = LinearSVC(loss="hinge", C=C, random_state=42)
+svm_clf = SVC(kernel="linear", C=C)
+sgd_clf = SGDClassifier(loss="hinge", learning_rate="constant", eta0=0.001, alpha=alpha,
+ max_iter=100000, random_state=42)
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+lin_clf.fit(X_scaled, y)
+svm_clf.fit(X_scaled, y)
+sgd_clf.fit(X_scaled, y)
+
+print("LinearSVC: ", lin_clf.intercept_, lin_clf.coef_)
+print("SVC: ", svm_clf.intercept_, svm_clf.coef_)
+print("SGDClassifier(alpha={:.5f}):".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
+
+# Compute the slope and bias of each decision boundary
+w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]
+b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]
+w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]
+b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]
+w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]
+b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]
+
+# Transform the decision boundary lines back to the original scale
+line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])
+line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])
+line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])
+
+# Plot all three decision boundaries
+plt.figure(figsize=(11, 4))
+plt.plot(line1[:, 0], line1[:, 1], "k:", label="LinearSVC")
+plt.plot(line2[:, 0], line2[:, 1], "b--", linewidth=2, label="SVC")
+plt.plot(line3[:, 0], line3[:, 1], "r-", label="SGDClassifier")
+plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs") # label="Iris-Versicolor"
+plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo") # label="Iris-Setosa"
+plt.xlabel("Petal length", fontsize=14)
+plt.ylabel("Petal width", fontsize=14)
+plt.legend(loc="upper center", fontsize=14)
+plt.axis([0, 5.5, 0, 2])
+
+plt.show()
+
@@ -682,6 +752,56 @@ obtain a separation between the classes which is almost linear.
The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that
we need to introduce for example a polynomial transformation to a two-dimensional training set.
+import numpy as np
+import os
+
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
+
@@ -794,7 +914,203 @@ well in practice.
-Mathematical optimization of convex functions
+The moons example
+from __future__ import division, print_function, unicode_literals
+
+import numpy as np
+np.random.seed(42)
+
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
+
+Mathematical optimization of convex functions
-How do we solve these problems?
+How do we solve these problems?
-A simplex example
+A simplex example
-Back to the more realistic cases
+Back to the more realistic cases
-Multiclass problems and regression with SVMs
+Multiclass problems and regression with SVMs
This material will be added later.
diff --git a/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz b/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz
index dc344b8f0..3aff4b71f 100644
Binary files a/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz and b/doc/pub/svm/ipynb/ipynb-svm-src.tar.gz differ
diff --git a/doc/pub/svm/ipynb/svm.ipynb b/doc/pub/svm/ipynb/svm.ipynb
index cde854a55..a2e4f867a 100644
--- a/doc/pub/svm/ipynb/svm.ipynb
+++ b/doc/pub/svm/ipynb/svm.ipynb
@@ -10,7 +10,7 @@
" \n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
"\n",
- "Date: **Nov 7, 2018**\n",
+ "Date: **Nov 8, 2018**\n",
"\n",
"Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
"\n",
@@ -49,13 +49,96 @@
"the mathematical description of so-called hyperplanes. Let us start\n",
"with a two-dimensional case. This will also allow us to introduce our\n",
"first SVM examples. These will be tailored to the case of two specific\n",
- "classes, as displayed in the figure here.\n",
+ "classes, as displayed in the figure here based on the usage of the petal data.\n",
"\n",
"We assume here that our data set can be well separated into two\n",
"domains, where a straight line does the job in the separating the two\n",
- "classes. Here the two classes are represented by either crosses or\n",
- "circles.\n",
+ "classes. Here the two classes are represented by either squares or\n",
+ "circles."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "%matplotlib inline\n",
"\n",
+ "from sklearn import datasets\n",
+ "from sklearn.svm import SVC, LinearSVC\n",
+ "from sklearn.linear_model import SGDClassifier\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "plt.rcParams['axes.labelsize'] = 14\n",
+ "plt.rcParams['xtick.labelsize'] = 12\n",
+ "plt.rcParams['ytick.labelsize'] = 12\n",
+ "\n",
+ "\n",
+ "iris = datasets.load_iris()\n",
+ "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n",
+ "y = iris[\"target\"]\n",
+ "\n",
+ "setosa_or_versicolor = (y == 0) | (y == 1)\n",
+ "X = X[setosa_or_versicolor]\n",
+ "y = y[setosa_or_versicolor]\n",
+ "\n",
+ "\n",
+ "\n",
+ "C = 5\n",
+ "alpha = 1 / (C * len(X))\n",
+ "\n",
+ "lin_clf = LinearSVC(loss=\"hinge\", C=C, random_state=42)\n",
+ "svm_clf = SVC(kernel=\"linear\", C=C)\n",
+ "sgd_clf = SGDClassifier(loss=\"hinge\", learning_rate=\"constant\", eta0=0.001, alpha=alpha,\n",
+ " max_iter=100000, random_state=42)\n",
+ "\n",
+ "scaler = StandardScaler()\n",
+ "X_scaled = scaler.fit_transform(X)\n",
+ "\n",
+ "lin_clf.fit(X_scaled, y)\n",
+ "svm_clf.fit(X_scaled, y)\n",
+ "sgd_clf.fit(X_scaled, y)\n",
+ "\n",
+ "print(\"LinearSVC: \", lin_clf.intercept_, lin_clf.coef_)\n",
+ "print(\"SVC: \", svm_clf.intercept_, svm_clf.coef_)\n",
+ "print(\"SGDClassifier(alpha={:.5f}):\".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)\n",
+ "\n",
+ "# Compute the slope and bias of each decision boundary\n",
+ "w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]\n",
+ "b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]\n",
+ "w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]\n",
+ "b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]\n",
+ "w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]\n",
+ "b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]\n",
+ "\n",
+ "# Transform the decision boundary lines back to the original scale\n",
+ "line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])\n",
+ "line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])\n",
+ "line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])\n",
+ "\n",
+ "# Plot all three decision boundaries\n",
+ "plt.figure(figsize=(11, 4))\n",
+ "plt.plot(line1[:, 0], line1[:, 1], \"k:\", label=\"LinearSVC\")\n",
+ "plt.plot(line2[:, 0], line2[:, 1], \"b--\", linewidth=2, label=\"SVC\")\n",
+ "plt.plot(line3[:, 0], line3[:, 1], \"r-\", label=\"SGDClassifier\")\n",
+ "plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\") # label=\"Iris-Versicolor\"\n",
+ "plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\") # label=\"Iris-Setosa\"\n",
+ "plt.xlabel(\"Petal length\", fontsize=14)\n",
+ "plt.ylabel(\"Petal width\", fontsize=14)\n",
+ "plt.legend(loc=\"upper center\", fontsize=14)\n",
+ "plt.axis([0, 5.5, 0, 2])\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## What is a hyperplane?\n",
"\n",
"The aim of the SVM algorithm is to find a hyperplane in an $p$-dimensional space, where $p$ is the number of features that distinctly classifies the data points. \n",
@@ -1079,8 +1162,69 @@
"obtain a separation between the classes which is almost linear. \n",
"\n",
"The change of basis, from $x\\rightarrow z=\\phi(x)$ leads to the same type of equations to be solved, except that\n",
- "we need to introduce for example a polynomial transformation to a two-dimensional training set.\n",
+ "we need to introduce for example a polynomial transformation to a two-dimensional training set."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import os\n",
"\n",
+ "np.random.seed(42)\n",
+ "\n",
+ "# To plot pretty figures\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "plt.rcParams['axes.labelsize'] = 14\n",
+ "plt.rcParams['xtick.labelsize'] = 12\n",
+ "plt.rcParams['ytick.labelsize'] = 12\n",
+ "\n",
+ "\n",
+ "from sklearn.svm import SVC\n",
+ "from sklearn import datasets\n",
+ "\n",
+ "\n",
+ "\n",
+ "X1D = np.linspace(-4, 4, 9).reshape(-1, 1)\n",
+ "X2D = np.c_[X1D, X1D**2]\n",
+ "y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])\n",
+ "\n",
+ "plt.figure(figsize=(11, 4))\n",
+ "\n",
+ "plt.subplot(121)\n",
+ "plt.grid(True, which='both')\n",
+ "plt.axhline(y=0, color='k')\n",
+ "plt.plot(X1D[:, 0][y==0], np.zeros(4), \"bs\")\n",
+ "plt.plot(X1D[:, 0][y==1], np.zeros(5), \"g^\")\n",
+ "plt.gca().get_yaxis().set_ticks([])\n",
+ "plt.xlabel(r\"$x_1$\", fontsize=20)\n",
+ "plt.axis([-4.5, 4.5, -0.2, 0.2])\n",
+ "\n",
+ "plt.subplot(122)\n",
+ "plt.grid(True, which='both')\n",
+ "plt.axhline(y=0, color='k')\n",
+ "plt.axvline(x=0, color='k')\n",
+ "plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], \"bs\")\n",
+ "plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], \"g^\")\n",
+ "plt.xlabel(r\"$x_1$\", fontsize=20)\n",
+ "plt.ylabel(r\"$x_2$\", fontsize=20, rotation=0)\n",
+ "plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])\n",
+ "plt.plot([-4.5, 4.5], [6.5, 6.5], \"r--\", linewidth=3)\n",
+ "plt.axis([-4.5, 4.5, -1, 17])\n",
+ "plt.subplots_adjust(right=1)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## The equations\n",
"\n",
"Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with $x_i$ and $y_i$ as variables)"
@@ -1277,6 +1421,211 @@
"well in practice.\n",
"\n",
"\n",
+ "## The moons example"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "from __future__ import division, print_function, unicode_literals\n",
+ "\n",
+ "import numpy as np\n",
+ "np.random.seed(42)\n",
+ "\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "plt.rcParams['axes.labelsize'] = 14\n",
+ "plt.rcParams['xtick.labelsize'] = 12\n",
+ "plt.rcParams['ytick.labelsize'] = 12\n",
+ "\n",
+ "\n",
+ "from sklearn.svm import SVC\n",
+ "from sklearn import datasets\n",
+ "\n",
+ "\n",
+ "\n",
+ "from sklearn.pipeline import Pipeline\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "from sklearn.svm import LinearSVC\n",
+ "\n",
+ "\n",
+ "from sklearn.datasets import make_moons\n",
+ "X, y = make_moons(n_samples=100, noise=0.15, random_state=42)\n",
+ "\n",
+ "def plot_dataset(X, y, axes):\n",
+ " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"bs\")\n",
+ " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"g^\")\n",
+ " plt.axis(axes)\n",
+ " plt.grid(True, which='both')\n",
+ " plt.xlabel(r\"$x_1$\", fontsize=20)\n",
+ " plt.ylabel(r\"$x_2$\", fontsize=20, rotation=0)\n",
+ "\n",
+ "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n",
+ "plt.show()\n",
+ "\n",
+ "from sklearn.datasets import make_moons\n",
+ "from sklearn.pipeline import Pipeline\n",
+ "from sklearn.preprocessing import PolynomialFeatures\n",
+ "\n",
+ "polynomial_svm_clf = Pipeline([\n",
+ " (\"poly_features\", PolynomialFeatures(degree=3)),\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " (\"svm_clf\", LinearSVC(C=10, loss=\"hinge\", random_state=42))\n",
+ " ])\n",
+ "\n",
+ "polynomial_svm_clf.fit(X, y)\n",
+ "\n",
+ "def plot_predictions(clf, axes):\n",
+ " x0s = np.linspace(axes[0], axes[1], 100)\n",
+ " x1s = np.linspace(axes[2], axes[3], 100)\n",
+ " x0, x1 = np.meshgrid(x0s, x1s)\n",
+ " X = np.c_[x0.ravel(), x1.ravel()]\n",
+ " y_pred = clf.predict(X).reshape(x0.shape)\n",
+ " y_decision = clf.decision_function(X).reshape(x0.shape)\n",
+ " plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)\n",
+ " plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)\n",
+ "\n",
+ "plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])\n",
+ "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n",
+ "\n",
+ "plt.show()\n",
+ "\n",
+ "\n",
+ "from sklearn.svm import SVC\n",
+ "\n",
+ "poly_kernel_svm_clf = Pipeline([\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " (\"svm_clf\", SVC(kernel=\"poly\", degree=3, coef0=1, C=5))\n",
+ " ])\n",
+ "poly_kernel_svm_clf.fit(X, y)\n",
+ "\n",
+ "poly100_kernel_svm_clf = Pipeline([\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " (\"svm_clf\", SVC(kernel=\"poly\", degree=10, coef0=100, C=5))\n",
+ " ])\n",
+ "poly100_kernel_svm_clf.fit(X, y)\n",
+ "\n",
+ "plt.figure(figsize=(11, 4))\n",
+ "\n",
+ "plt.subplot(121)\n",
+ "plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])\n",
+ "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n",
+ "plt.title(r\"$d=3, r=1, C=5$\", fontsize=18)\n",
+ "\n",
+ "plt.subplot(122)\n",
+ "plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])\n",
+ "plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n",
+ "plt.title(r\"$d=10, r=100, C=5$\", fontsize=18)\n",
+ "\n",
+ "plt.show()\n",
+ "\n",
+ "def gaussian_rbf(x, landmark, gamma):\n",
+ " return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)\n",
+ "\n",
+ "gamma = 0.3\n",
+ "\n",
+ "x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)\n",
+ "x2s = gaussian_rbf(x1s, -2, gamma)\n",
+ "x3s = gaussian_rbf(x1s, 1, gamma)\n",
+ "\n",
+ "XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]\n",
+ "yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])\n",
+ "\n",
+ "plt.figure(figsize=(11, 4))\n",
+ "\n",
+ "plt.subplot(121)\n",
+ "plt.grid(True, which='both')\n",
+ "plt.axhline(y=0, color='k')\n",
+ "plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c=\"red\")\n",
+ "plt.plot(X1D[:, 0][yk==0], np.zeros(4), \"bs\")\n",
+ "plt.plot(X1D[:, 0][yk==1], np.zeros(5), \"g^\")\n",
+ "plt.plot(x1s, x2s, \"g--\")\n",
+ "plt.plot(x1s, x3s, \"b:\")\n",
+ "plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])\n",
+ "plt.xlabel(r\"$x_1$\", fontsize=20)\n",
+ "plt.ylabel(r\"Similarity\", fontsize=14)\n",
+ "plt.annotate(r'$\\mathbf{x}$',\n",
+ " xy=(X1D[3, 0], 0),\n",
+ " xytext=(-0.5, 0.20),\n",
+ " ha=\"center\",\n",
+ " arrowprops=dict(facecolor='black', shrink=0.1),\n",
+ " fontsize=18,\n",
+ " )\n",
+ "plt.text(-2, 0.9, \"$x_2$\", ha=\"center\", fontsize=20)\n",
+ "plt.text(1, 0.9, \"$x_3$\", ha=\"center\", fontsize=20)\n",
+ "plt.axis([-4.5, 4.5, -0.1, 1.1])\n",
+ "\n",
+ "plt.subplot(122)\n",
+ "plt.grid(True, which='both')\n",
+ "plt.axhline(y=0, color='k')\n",
+ "plt.axvline(x=0, color='k')\n",
+ "plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], \"bs\")\n",
+ "plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], \"g^\")\n",
+ "plt.xlabel(r\"$x_2$\", fontsize=20)\n",
+ "plt.ylabel(r\"$x_3$ \", fontsize=20, rotation=0)\n",
+ "plt.annotate(r'$\\phi\\left(\\mathbf{x}\\right)$',\n",
+ " xy=(XK[3, 0], XK[3, 1]),\n",
+ " xytext=(0.65, 0.50),\n",
+ " ha=\"center\",\n",
+ " arrowprops=dict(facecolor='black', shrink=0.1),\n",
+ " fontsize=18,\n",
+ " )\n",
+ "plt.plot([-0.1, 1.1], [0.57, -0.1], \"r--\", linewidth=3)\n",
+ "plt.axis([-0.1, 1.1, -0.1, 1.1])\n",
+ " \n",
+ "plt.subplots_adjust(right=1)\n",
+ "\n",
+ "plt.show()\n",
+ "\n",
+ "\n",
+ "x1_example = X1D[3, 0]\n",
+ "for landmark in (-2, 1):\n",
+ " k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)\n",
+ " print(\"Phi({}, {}) = {}\".format(x1_example, landmark, k))\n",
+ "\n",
+ "rbf_kernel_svm_clf = Pipeline([\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " (\"svm_clf\", SVC(kernel=\"rbf\", gamma=5, C=0.001))\n",
+ " ])\n",
+ "rbf_kernel_svm_clf.fit(X, y)\n",
+ "\n",
+ "\n",
+ "from sklearn.svm import SVC\n",
+ "\n",
+ "gamma1, gamma2 = 0.1, 5\n",
+ "C1, C2 = 0.001, 1000\n",
+ "hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)\n",
+ "\n",
+ "svm_clfs = []\n",
+ "for gamma, C in hyperparams:\n",
+ " rbf_kernel_svm_clf = Pipeline([\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " (\"svm_clf\", SVC(kernel=\"rbf\", gamma=gamma, C=C))\n",
+ " ])\n",
+ " rbf_kernel_svm_clf.fit(X, y)\n",
+ " svm_clfs.append(rbf_kernel_svm_clf)\n",
+ "\n",
+ "plt.figure(figsize=(11, 7))\n",
+ "\n",
+ "for i, svm_clf in enumerate(svm_clfs):\n",
+ " plt.subplot(221 + i)\n",
+ " plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])\n",
+ " plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])\n",
+ " gamma, C = hyperparams[i]\n",
+ " plt.title(r\"$\\gamma = {}, C = {}$\".format(gamma, C), fontsize=16)\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Mathematical optimization of convex functions\n",
"\n",
"A mathematical (quadratic) optimization problem, or just optimization problem, has the form"
@@ -1324,7 +1673,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 4,
"metadata": {
"collapsed": false
},
@@ -1456,7 +1805,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 5,
"metadata": {
"collapsed": false
},
diff --git a/doc/pub/svm/pdf/svm-minted.pdf b/doc/pub/svm/pdf/svm-minted.pdf
index 9cea03dbb..b57ad3883 100644
Binary files a/doc/pub/svm/pdf/svm-minted.pdf and b/doc/pub/svm/pdf/svm-minted.pdf differ
diff --git a/doc/src/SupportVMachines/svm.do.txt b/doc/src/SupportVMachines/svm.do.txt
index da75a7df4..55bf6099f 100644
--- a/doc/src/SupportVMachines/svm.do.txt
+++ b/doc/src/SupportVMachines/svm.do.txt
@@ -37,12 +37,91 @@ The theory behind support vector machines (SVM hereafter) is based on
the mathematical description of so-called hyperplanes. Let us start
with a two-dimensional case. This will also allow us to introduce our
first SVM examples. These will be tailored to the case of two specific
-classes, as displayed in the figure here.
+classes, as displayed in the figure here based on the usage of the petal data.
We assume here that our data set can be well separated into two
domains, where a straight line does the job in the separating the two
-classes. Here the two classes are represented by either crosses or
+classes. Here the two classes are represented by either squares or
circles.
+!bc pycod
+from sklearn import datasets
+from sklearn.svm import SVC, LinearSVC
+from sklearn.linear_model import SGDClassifier
+from sklearn.preprocessing import StandardScaler
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+iris = datasets.load_iris()
+X = iris["data"][:, (2, 3)] # petal length, petal width
+y = iris["target"]
+
+setosa_or_versicolor = (y == 0) | (y == 1)
+X = X[setosa_or_versicolor]
+y = y[setosa_or_versicolor]
+
+
+
+C = 5
+alpha = 1 / (C * len(X))
+
+lin_clf = LinearSVC(loss="hinge", C=C, random_state=42)
+svm_clf = SVC(kernel="linear", C=C)
+sgd_clf = SGDClassifier(loss="hinge", learning_rate="constant", eta0=0.001, alpha=alpha,
+ max_iter=100000, random_state=42)
+
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+
+lin_clf.fit(X_scaled, y)
+svm_clf.fit(X_scaled, y)
+sgd_clf.fit(X_scaled, y)
+
+print("LinearSVC: ", lin_clf.intercept_, lin_clf.coef_)
+print("SVC: ", svm_clf.intercept_, svm_clf.coef_)
+print("SGDClassifier(alpha={:.5f}):".format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
+
+# Compute the slope and bias of each decision boundary
+w1 = -lin_clf.coef_[0, 0]/lin_clf.coef_[0, 1]
+b1 = -lin_clf.intercept_[0]/lin_clf.coef_[0, 1]
+w2 = -svm_clf.coef_[0, 0]/svm_clf.coef_[0, 1]
+b2 = -svm_clf.intercept_[0]/svm_clf.coef_[0, 1]
+w3 = -sgd_clf.coef_[0, 0]/sgd_clf.coef_[0, 1]
+b3 = -sgd_clf.intercept_[0]/sgd_clf.coef_[0, 1]
+
+# Transform the decision boundary lines back to the original scale
+line1 = scaler.inverse_transform([[-10, -10 * w1 + b1], [10, 10 * w1 + b1]])
+line2 = scaler.inverse_transform([[-10, -10 * w2 + b2], [10, 10 * w2 + b2]])
+line3 = scaler.inverse_transform([[-10, -10 * w3 + b3], [10, 10 * w3 + b3]])
+
+# Plot all three decision boundaries
+plt.figure(figsize=(11, 4))
+plt.plot(line1[:, 0], line1[:, 1], "k:", label="LinearSVC")
+plt.plot(line2[:, 0], line2[:, 1], "b--", linewidth=2, label="SVC")
+plt.plot(line3[:, 0], line3[:, 1], "r-", label="SGDClassifier")
+plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs") # label="Iris-Versicolor"
+plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo") # label="Iris-Setosa"
+plt.xlabel("Petal length", fontsize=14)
+plt.ylabel("Petal width", fontsize=14)
+plt.legend(loc="upper center", fontsize=14)
+plt.axis([0, 5.5, 0, 2])
+
+plt.show()
+
+
+
+
+
+
+
+
+!ec
+
+
+
!split
===== What is a hyperplane? =====
@@ -553,6 +632,58 @@ obtain a separation between the classes which is almost linear.
The change of basis, from $x\rightarrow z=\phi(x)$ leads to the same type of equations to be solved, except that
we need to introduce for example a polynomial transformation to a two-dimensional training set.
+!bc pycod
+import numpy as np
+import os
+
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+X1D = np.linspace(-4, 4, 9).reshape(-1, 1)
+X2D = np.c_[X1D, X1D**2]
+y = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.plot(X1D[:, 0][y==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][y==1], np.zeros(5), "g^")
+plt.gca().get_yaxis().set_ticks([])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.axis([-4.5, 4.5, -0.2, 0.2])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(X2D[:, 0][y==0], X2D[:, 1][y==0], "bs")
+plt.plot(X2D[:, 0][y==1], X2D[:, 1][y==1], "g^")
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+plt.gca().get_yaxis().set_ticks([0, 4, 8, 12, 16])
+plt.plot([-4.5, 4.5], [6.5, 6.5], "r--", linewidth=3)
+plt.axis([-4.5, 4.5, -1, 17])
+plt.subplots_adjust(right=1)
+plt.show()
+
+!ec
+
+
+
!split
===== The equations =====
@@ -658,6 +789,202 @@ Note that some frequently used kernels (such as the Sigmoid kernel) don’t resp
well in practice.
+!split
+===== The moons example =====
+!bc pycod
+from __future__ import division, print_function, unicode_literals
+
+import numpy as np
+np.random.seed(42)
+
+import matplotlib
+import matplotlib.pyplot as plt
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+
+
+
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import LinearSVC
+
+
+from sklearn.datasets import make_moons
+X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
+
+def plot_dataset(X, y, axes):
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
+ plt.axis(axes)
+ plt.grid(True, which='both')
+ plt.xlabel(r"$x_1$", fontsize=20)
+ plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
+
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.show()
+
+from sklearn.datasets import make_moons
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+
+polynomial_svm_clf = Pipeline([
+ ("poly_features", PolynomialFeatures(degree=3)),
+ ("scaler", StandardScaler()),
+ ("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
+ ])
+
+polynomial_svm_clf.fit(X, y)
+
+def plot_predictions(clf, axes):
+ x0s = np.linspace(axes[0], axes[1], 100)
+ x1s = np.linspace(axes[2], axes[3], 100)
+ x0, x1 = np.meshgrid(x0s, x1s)
+ X = np.c_[x0.ravel(), x1.ravel()]
+ y_pred = clf.predict(X).reshape(x0.shape)
+ y_decision = clf.decision_function(X).reshape(x0.shape)
+ plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
+ plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
+
+plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+
+plt.show()
+
+
+from sklearn.svm import SVC
+
+poly_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=3, coef0=1, C=5))
+ ])
+poly_kernel_svm_clf.fit(X, y)
+
+poly100_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="poly", degree=10, coef0=100, C=5))
+ ])
+poly100_kernel_svm_clf.fit(X, y)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plot_predictions(poly_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=3, r=1, C=5$", fontsize=18)
+
+plt.subplot(122)
+plot_predictions(poly100_kernel_svm_clf, [-1.5, 2.5, -1, 1.5])
+plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+plt.title(r"$d=10, r=100, C=5$", fontsize=18)
+
+plt.show()
+
+def gaussian_rbf(x, landmark, gamma):
+ return np.exp(-gamma * np.linalg.norm(x - landmark, axis=1)**2)
+
+gamma = 0.3
+
+x1s = np.linspace(-4.5, 4.5, 200).reshape(-1, 1)
+x2s = gaussian_rbf(x1s, -2, gamma)
+x3s = gaussian_rbf(x1s, 1, gamma)
+
+XK = np.c_[gaussian_rbf(X1D, -2, gamma), gaussian_rbf(X1D, 1, gamma)]
+yk = np.array([0, 0, 1, 1, 1, 1, 1, 0, 0])
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.scatter(x=[-2, 1], y=[0, 0], s=150, alpha=0.5, c="red")
+plt.plot(X1D[:, 0][yk==0], np.zeros(4), "bs")
+plt.plot(X1D[:, 0][yk==1], np.zeros(5), "g^")
+plt.plot(x1s, x2s, "g--")
+plt.plot(x1s, x3s, "b:")
+plt.gca().get_yaxis().set_ticks([0, 0.25, 0.5, 0.75, 1])
+plt.xlabel(r"$x_1$", fontsize=20)
+plt.ylabel(r"Similarity", fontsize=14)
+plt.annotate(r'$\mathbf{x}$',
+ xy=(X1D[3, 0], 0),
+ xytext=(-0.5, 0.20),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.text(-2, 0.9, "$x_2$", ha="center", fontsize=20)
+plt.text(1, 0.9, "$x_3$", ha="center", fontsize=20)
+plt.axis([-4.5, 4.5, -0.1, 1.1])
+
+plt.subplot(122)
+plt.grid(True, which='both')
+plt.axhline(y=0, color='k')
+plt.axvline(x=0, color='k')
+plt.plot(XK[:, 0][yk==0], XK[:, 1][yk==0], "bs")
+plt.plot(XK[:, 0][yk==1], XK[:, 1][yk==1], "g^")
+plt.xlabel(r"$x_2$", fontsize=20)
+plt.ylabel(r"$x_3$ ", fontsize=20, rotation=0)
+plt.annotate(r'$\phi\left(\mathbf{x}\right)$',
+ xy=(XK[3, 0], XK[3, 1]),
+ xytext=(0.65, 0.50),
+ ha="center",
+ arrowprops=dict(facecolor='black', shrink=0.1),
+ fontsize=18,
+ )
+plt.plot([-0.1, 1.1], [0.57, -0.1], "r--", linewidth=3)
+plt.axis([-0.1, 1.1, -0.1, 1.1])
+
+plt.subplots_adjust(right=1)
+
+plt.show()
+
+
+x1_example = X1D[3, 0]
+for landmark in (-2, 1):
+ k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
+ print("Phi({}, {}) = {}".format(x1_example, landmark, k))
+
+rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=5, C=0.001))
+ ])
+rbf_kernel_svm_clf.fit(X, y)
+
+
+from sklearn.svm import SVC
+
+gamma1, gamma2 = 0.1, 5
+C1, C2 = 0.001, 1000
+hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
+
+svm_clfs = []
+for gamma, C in hyperparams:
+ rbf_kernel_svm_clf = Pipeline([
+ ("scaler", StandardScaler()),
+ ("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
+ ])
+ rbf_kernel_svm_clf.fit(X, y)
+ svm_clfs.append(rbf_kernel_svm_clf)
+
+plt.figure(figsize=(11, 7))
+
+for i, svm_clf in enumerate(svm_clfs):
+ plt.subplot(221 + i)
+ plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
+ plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
+ gamma, C = hyperparams[i]
+ plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
+
+plt.show()
+
+!ec
+
+
+
!split
===== Mathematical optimization of convex functions =====