Morten Hjorth-Jensen
1458 lines
79 KiB
HTML
1458 lines
79 KiB
HTML
<!--
|
||
HTML file automatically generated from DocOnce source
|
||
(https://github.com/doconce/doconce/)
|
||
doconce format html week46.do.txt --pygments_html_style=perldoc --html_style=solarized3 --html_links_in_new_window --html_output=week46-solarized --no_mako
|
||
-->
|
||
<html>
|
||
<head>
|
||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<meta name="description" content="Week 46: Support Vector Machines and Project 3">
|
||
<title>Week 46: Support Vector Machines and Project 3</title>
|
||
<link href="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
||
<script src="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
|
||
<script>hljs.initHighlightingOnLoad();</script>
|
||
<link href="https://thomasf.github.io/solarized-css/solarized-light.min.css" rel="stylesheet">
|
||
<style type="text/css">
|
||
h1 {color: #b58900;} /* yellow */
|
||
/* h1 {color: #cb4b16;} orange */
|
||
/* h1 {color: #d33682;} magenta, the original choice of thomasf */
|
||
code { padding: 0px; background-color: inherit; }
|
||
pre {
|
||
border: 0pt solid #93a1a1;
|
||
box-shadow: none;
|
||
}
|
||
.alert-text-small { font-size: 80%; }
|
||
.alert-text-large { font-size: 130%; }
|
||
.alert-text-normal { font-size: 90%; }
|
||
.alert {
|
||
padding:8px 35px 8px 14px; margin-bottom:18px;
|
||
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
||
border:1px solid #93a1a1;
|
||
border-radius: 4px;
|
||
-webkit-border-radius: 4px;
|
||
-moz-border-radius: 4px;
|
||
color: #555;
|
||
background-color: #eee8d5;
|
||
background-position: 10px 5px;
|
||
background-repeat: no-repeat;
|
||
background-size: 38px;
|
||
padding-left: 55px;
|
||
width: 75%;
|
||
}
|
||
.alert-block {padding-top:14px; padding-bottom:14px}
|
||
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
||
.alert li {margin-top: 1em}
|
||
.alert-block p+p {margin-top:5px}
|
||
.alert-notice { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_notice.png); }
|
||
.alert-summary { background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_summary.png); }
|
||
.alert-warning { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_warning.png); }
|
||
.alert-question {background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_yellow_question.png); }
|
||
div { text-align: justify; text-justify: inter-word; }
|
||
.tab {
|
||
padding-left: 1.5em;
|
||
}
|
||
div.toc p,a {
|
||
line-height: 1.3;
|
||
margin-top: 1.1;
|
||
margin-bottom: 1.1;
|
||
}
|
||
</style>
|
||
</head>
|
||
|
||
<!-- tocinfo
|
||
{'highest level': 2,
|
||
'sections': [('Overview of week 46', 2, None, 'overview-of-week-46'),
|
||
('Friday', 2, None, 'friday'),
|
||
('Workshop plan Friday November 19 and the rest of the lecture',
|
||
2,
|
||
None,
|
||
'workshop-plan-friday-november-19-and-the-rest-of-the-lecture'),
|
||
('Support Vector Machines, overarching aims',
|
||
2,
|
||
None,
|
||
'support-vector-machines-overarching-aims'),
|
||
('Hyperplanes and all that', 2, None, 'hyperplanes-and-all-that'),
|
||
('What is a hyperplane?', 2, None, 'what-is-a-hyperplane'),
|
||
('A $p$-dimensional space of features',
|
||
2,
|
||
None,
|
||
'a-p-dimensional-space-of-features'),
|
||
('The two-dimensional case', 2, None, 'the-two-dimensional-case'),
|
||
('Getting into the details', 2, None, 'getting-into-the-details'),
|
||
('First attempt at a minimization approach',
|
||
2,
|
||
None,
|
||
'first-attempt-at-a-minimization-approach'),
|
||
('Solving the equations', 2, None, 'solving-the-equations'),
|
||
('Code Example', 2, None, 'code-example'),
|
||
('Problems with the Simpler Approach',
|
||
2,
|
||
None,
|
||
'problems-with-the-simpler-approach'),
|
||
('A better approach', 2, None, 'a-better-approach'),
|
||
('A quick Reminder on Lagrangian Multipliers',
|
||
2,
|
||
None,
|
||
'a-quick-reminder-on-lagrangian-multipliers'),
|
||
('Adding the Multiplier', 2, None, 'adding-the-multiplier'),
|
||
('Setting up the Problem', 2, None, 'setting-up-the-problem'),
|
||
('The problem to solve', 2, None, 'the-problem-to-solve'),
|
||
('The last steps', 2, None, 'the-last-steps'),
|
||
('A soft classifier', 2, None, 'a-soft-classifier'),
|
||
('Soft optmization problem', 2, None, 'soft-optmization-problem'),
|
||
('Kernels and non-linearity',
|
||
2,
|
||
None,
|
||
'kernels-and-non-linearity'),
|
||
('The equations', 2, None, 'the-equations'),
|
||
('The problem to solve', 2, None, 'the-problem-to-solve'),
|
||
("Different kernels and Mercer's theorem",
|
||
2,
|
||
None,
|
||
'different-kernels-and-mercer-s-theorem'),
|
||
('The moons example', 2, None, 'the-moons-example'),
|
||
('Mathematical optimization of convex functions',
|
||
2,
|
||
None,
|
||
'mathematical-optimization-of-convex-functions'),
|
||
('How do we solve these problems?',
|
||
2,
|
||
None,
|
||
'how-do-we-solve-these-problems'),
|
||
('A simple example', 2, None, 'a-simple-example'),
|
||
('Back to the more realistic cases',
|
||
2,
|
||
None,
|
||
'back-to-the-more-realistic-cases')]}
|
||
end of tocinfo -->
|
||
|
||
<body>
|
||
|
||
|
||
|
||
<script type="text/x-mathjax-config">
|
||
MathJax.Hub.Config({
|
||
TeX: {
|
||
equationNumbers: { autoNumber: "AMS" },
|
||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||
}
|
||
});
|
||
</script>
|
||
<script type="text/javascript" async
|
||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||
</script>
|
||
|
||
|
||
<!-- ------------------- main content ---------------------- -->
|
||
<center>
|
||
<h1>Week 46: Support Vector Machines and Project 3</h1>
|
||
</center> <!-- document title -->
|
||
|
||
<!-- author(s): Morten Hjorth-Jensen -->
|
||
<center>
|
||
<b>Morten Hjorth-Jensen</b> [1, 2]
|
||
</center>
|
||
<!-- institution(s) -->
|
||
<center>
|
||
[1] <b>Department of Physics, University of Oslo</b>
|
||
</center>
|
||
<center>
|
||
[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b>
|
||
</center>
|
||
<br>
|
||
<center>
|
||
<h4>Nov 19, 2021</h4>
|
||
</center> <!-- date -->
|
||
<br>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="overview-of-week-46">Overview of week 46 </h2>
|
||
|
||
<ul>
|
||
<li> <b>Thursday</b>: Summary of Gradient Boosting. Start support Vector Machines</li>
|
||
<li> <b>Friday</b>: Eventual Workshop on project 3. Discussion of project 3.</li>
|
||
</ul>
|
||
<b>Note that on Thursday we will have only one lecture, from 1215pm to 1pm. There is no session 115pm-2pm.</b>
|
||
|
||
<div class="alert alert-block alert-block alert-text-normal">
|
||
<b>Reading</b>
|
||
<p>
|
||
<p>Reading recommendations:</p>
|
||
<ol>
|
||
<li> See lecture notes for week 46 at <a href="https://compphysics.github.io/MachineLearning/doc/web/course.html." target="_blank"><tt>https://compphysics.github.io/MachineLearning/doc/web/course.html.</tt></a></li>
|
||
<li> Hastie et al chapter 12</li>
|
||
<li> Bishop chapter 7.1 and 7.2</li>
|
||
</ol>
|
||
</div>
|
||
|
||
|
||
<div class="alert alert-block alert-block alert-text-normal">
|
||
<b>Videos</b>
|
||
<p>
|
||
<ol>
|
||
<li> <a href="https://www.youtube.com/watch?v=efR1C6CvhmE&ab_channel=StatQuestwithJoshStarmer" target="_blank">Overview video on Support Vector Machines</a></li>
|
||
<li> See also <a href="https://www.youtube.com/watch?v=N1vOgolbjSc&ab_channel=AliceZhao" target="_blank">this video</a>.</li>
|
||
<li> <a href="https://www.youtube.com/watch?v=3CC4N4z3GJc&ab_channel=StatQuestwithJoshStarmer" target="_blank">Excellent videos on Gradient Boosting</a></li>
|
||
</ol>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="friday">Friday </h2>
|
||
|
||
<p>Last year we had a very interesting workshop with many presentations. These were (program 2020)</p>
|
||
|
||
<ul>
|
||
<li> Maria Emine Nylund: <b>Lego Bricks Classifier</b></li>
|
||
<li> Fabio Rodrigues Pereira: <b>Financial Machine Learning</b></li>
|
||
<li> Markus Borud Pettersen: <b>Machine Learning and Brain Grid Cells</b></li>
|
||
<li> Jing Sun and Endrias Getachew Asgedom: <b>Machine learning-based approaches to denoising microseismic data</b></li>
|
||
<li> Felicia Jacobsen: <b>Analysis of Breast Cancer Data</b></li>
|
||
<li> Simon Elias Schrader: <b>Predicting atomization energies of molecules</b></li>
|
||
<li> Varvara Bazilova and Sergio Andres Diaz Mesa: <b>Glacier Mapping and Machine Learning</b></li>
|
||
<li> Gert Werner Kluge, Hanna Alida Fossen Hardersen and Sushma Sharma Adhikari: <b>Gamma ray signals stemming from dark matter in the galactic center</b></li>
|
||
</ul>
|
||
<p>We wish to organize something similar this coming Friday. The presentation last typically 5-10 minutes (some 3-5 slides) with time for questions afterwards.
|
||
Feel free to suggest topics.
|
||
</p>
|
||
|
||
<p>The program will be available asap. It depends on input from you!</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="workshop-plan-friday-november-19-and-the-rest-of-the-lecture">Workshop plan Friday November 19 and the rest of the lecture </h2>
|
||
|
||
<ol>
|
||
<li> <b>1215-1225pm</b>: Are Frode Helvig Kvanum, Gard Høivang, and David Andreas Bordvik, <em>Next-day forecasts on spot prices for electricity</em></li>
|
||
<li> <b>1225-1235pm</b>: Lidia Luque, <em>Voxel-wise multi-label brain tumor classification</em></li>
|
||
<li> <b>1235-1245pm</b>: Marcus Berget et al, <em>Locating suspicious brain activity using neural networks</em></li>
|
||
<li> <b>1245-1255pm</b>: William Ho and Tom-Ruben Traavik Kvalvaag, <em>Comparing semi-supervised learning and supervised learning for image classification</em></li>
|
||
</ol>
|
||
<p>We will use the second part of the lecture for further discussions of projects 2 and 3 and a summary on boosting methods from last week. Feel free to bring your laptops.</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="support-vector-machines-overarching-aims">Support Vector Machines, overarching aims </h2>
|
||
|
||
<p>A Support Vector Machine (SVM) is a very powerful and versatile
|
||
Machine Learning method, capable of performing linear or nonlinear
|
||
classification, regression, and even outlier detection. It is one of
|
||
the most popular models in Machine Learning, and anyone interested in
|
||
Machine Learning should have it in their toolbox. SVMs are
|
||
particularly well suited for classification of complex but small-sized or
|
||
medium-sized datasets.
|
||
</p>
|
||
|
||
<p>The case with two well-separated classes only can be understood in an
|
||
intuitive way in terms of lines in a two-dimensional space separating
|
||
the two classes (see figure below).
|
||
</p>
|
||
|
||
<p>The basic mathematics behind the SVM is however less familiar to most of us.
|
||
It relies on the definition of hyperplanes and the
|
||
definition of a <b>margin</b> which separates classes (in case of
|
||
classification problems) of variables. It is also used for regression
|
||
problems.
|
||
</p>
|
||
|
||
<p>With SVMs we distinguish between hard margin and soft margins. The
|
||
latter introduces a so-called softening parameter to be discussed
|
||
below. We distinguish also between linear and non-linear
|
||
approaches. The latter are the most frequent ones since it is rather
|
||
unlikely that we can separate classes easily by say straight lines.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="hyperplanes-and-all-that">Hyperplanes and all that </h2>
|
||
|
||
<p>The theory behind support vector machines (SVM hereafter) is based on
|
||
the mathematical description of so-called hyperplanes. Let us start
|
||
with a two-dimensional case. This will also allow us to introduce our
|
||
first SVM examples. These will be tailored to the case of two specific
|
||
classes, as displayed in the figure here based on the usage of the petal data.
|
||
</p>
|
||
|
||
<p>We assume here that our data set can be well separated into two
|
||
domains, where a straight line does the job in the separating the two
|
||
classes. Here the two classes are represented by either squares or
|
||
circles.
|
||
</p>
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC, LinearSVC
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDClassifier
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||
plt.rcParams[<span style="color: #CD5555">'axes.labelsize'</span>] = <span style="color: #B452CD">14</span>
|
||
plt.rcParams[<span style="color: #CD5555">'xtick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
plt.rcParams[<span style="color: #CD5555">'ytick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
|
||
|
||
iris = datasets.load_iris()
|
||
X = iris[<span style="color: #CD5555">"data"</span>][:, (<span style="color: #B452CD">2</span>, <span style="color: #B452CD">3</span>)] <span style="color: #228B22"># petal length, petal width</span>
|
||
y = iris[<span style="color: #CD5555">"target"</span>]
|
||
|
||
setosa_or_versicolor = (y == <span style="color: #B452CD">0</span>) | (y == <span style="color: #B452CD">1</span>)
|
||
X = X[setosa_or_versicolor]
|
||
y = y[setosa_or_versicolor]
|
||
|
||
|
||
|
||
C = <span style="color: #B452CD">5</span>
|
||
alpha = <span style="color: #B452CD">1</span> / (C * <span style="color: #658b00">len</span>(X))
|
||
|
||
lin_clf = LinearSVC(loss=<span style="color: #CD5555">"hinge"</span>, C=C, random_state=<span style="color: #B452CD">42</span>)
|
||
svm_clf = SVC(kernel=<span style="color: #CD5555">"linear"</span>, C=C)
|
||
sgd_clf = SGDClassifier(loss=<span style="color: #CD5555">"hinge"</span>, learning_rate=<span style="color: #CD5555">"constant"</span>, eta0=<span style="color: #B452CD">0.001</span>, alpha=alpha,
|
||
max_iter=<span style="color: #B452CD">100000</span>, random_state=<span style="color: #B452CD">42</span>)
|
||
|
||
scaler = StandardScaler()
|
||
X_scaled = scaler.fit_transform(X)
|
||
|
||
lin_clf.fit(X_scaled, y)
|
||
svm_clf.fit(X_scaled, y)
|
||
sgd_clf.fit(X_scaled, y)
|
||
|
||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"LinearSVC: "</span>, lin_clf.intercept_, lin_clf.coef_)
|
||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"SVC: "</span>, svm_clf.intercept_, svm_clf.coef_)
|
||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"SGDClassifier(alpha={:.5f}):"</span>.format(sgd_clf.alpha), sgd_clf.intercept_, sgd_clf.coef_)
|
||
|
||
<span style="color: #228B22"># Compute the slope and bias of each decision boundary</span>
|
||
w1 = -lin_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>]/lin_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
b1 = -lin_clf.intercept_[<span style="color: #B452CD">0</span>]/lin_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
w2 = -svm_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>]/svm_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
b2 = -svm_clf.intercept_[<span style="color: #B452CD">0</span>]/svm_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
w3 = -sgd_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>]/sgd_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
b3 = -sgd_clf.intercept_[<span style="color: #B452CD">0</span>]/sgd_clf.coef_[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>]
|
||
|
||
<span style="color: #228B22"># Transform the decision boundary lines back to the original scale</span>
|
||
line1 = scaler.inverse_transform([[-<span style="color: #B452CD">10</span>, -<span style="color: #B452CD">10</span> * w1 + b1], [<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span> * w1 + b1]])
|
||
line2 = scaler.inverse_transform([[-<span style="color: #B452CD">10</span>, -<span style="color: #B452CD">10</span> * w2 + b2], [<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span> * w2 + b2]])
|
||
line3 = scaler.inverse_transform([[-<span style="color: #B452CD">10</span>, -<span style="color: #B452CD">10</span> * w3 + b3], [<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span> * w3 + b3]])
|
||
|
||
<span style="color: #228B22"># Plot all three decision boundaries</span>
|
||
plt.figure(figsize=(<span style="color: #B452CD">11</span>, <span style="color: #B452CD">4</span>))
|
||
plt.plot(line1[:, <span style="color: #B452CD">0</span>], line1[:, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"k:"</span>, label=<span style="color: #CD5555">"LinearSVC"</span>)
|
||
plt.plot(line2[:, <span style="color: #B452CD">0</span>], line2[:, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"b--"</span>, linewidth=<span style="color: #B452CD">2</span>, label=<span style="color: #CD5555">"SVC"</span>)
|
||
plt.plot(line3[:, <span style="color: #B452CD">0</span>], line3[:, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"r-"</span>, label=<span style="color: #CD5555">"SGDClassifier"</span>)
|
||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">1</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">1</span>], <span style="color: #CD5555">"bs"</span>) <span style="color: #228B22"># label="Iris-Versicolor"</span>
|
||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">0</span>], <span style="color: #CD5555">"yo"</span>) <span style="color: #228B22"># label="Iris-Setosa"</span>
|
||
plt.xlabel(<span style="color: #CD5555">"Petal length"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||
plt.ylabel(<span style="color: #CD5555">"Petal width"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||
plt.legend(loc=<span style="color: #CD5555">"upper center"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||
plt.axis([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">5.5</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">2</span>])
|
||
|
||
plt.show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="what-is-a-hyperplane">What is a hyperplane? </h2>
|
||
|
||
<p>The aim of the SVM algorithm is to find a hyperplane in a
|
||
\( p \)-dimensional space, where \( p \) is the number of features that
|
||
distinctly classifies the data points.
|
||
</p>
|
||
|
||
<p>In a \( p \)-dimensional space, a hyperplane is what we call an affine subspace of dimension of \( p-1 \).
|
||
As an example, in two dimension, a hyperplane is simply as straight line while in three dimensions it is
|
||
a two-dimensional subspace, or stated simply, a plane.
|
||
</p>
|
||
|
||
<p>In two dimensions, with the variables \( x_1 \) and \( x_2 \), the hyperplane is defined as</p>
|
||
$$
|
||
b+w_1x_1+w_2x_2=0,
|
||
$$
|
||
|
||
<p>where \( b \) is the intercept and \( w_1 \) and \( w_2 \) define the elements of a vector orthogonal to the line
|
||
\( b+w_1x_1+w_2x_2=0 \).
|
||
In two dimensions we define the vectors \( \boldsymbol{x} =[x1,x2] \) and \( \boldsymbol{w}=[w1,w2] \).
|
||
We can then rewrite the above equation as
|
||
</p>
|
||
|
||
$$
|
||
\boldsymbol{x}^T\boldsymbol{w}+b=0.
|
||
$$
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="a-p-dimensional-space-of-features">A \( p \)-dimensional space of features </h2>
|
||
|
||
<p>We limit ourselves to two classes of outputs \( y_i \) and assign these classes the values \( y_i = \pm 1 \).
|
||
In a \( p \)-dimensional space of say \( p \) features we have a hyperplane defines as
|
||
</p>
|
||
$$
|
||
b+wx_1+w_2x_2+\dots +w_px_p=0.
|
||
$$
|
||
|
||
<p>If we define a
|
||
matrix \( \boldsymbol{X}=\left[\boldsymbol{x}_1,\boldsymbol{x}_2,\dots, \boldsymbol{x}_p\right] \)
|
||
of dimension \( n\times p \), where \( n \) represents the observations for each feature and each vector \( x_i \) is a column vector of the matrix \( \boldsymbol{X} \),
|
||
</p>
|
||
$$
|
||
\boldsymbol{x}_i = \begin{bmatrix} x_{i1} \\ x_{i2} \\ \dots \\ \dots \\ x_{ip} \end{bmatrix}.
|
||
$$
|
||
|
||
<p>If the above condition is not met for a given vector \( \boldsymbol{x}_i \) we have </p>
|
||
$$
|
||
b+w_1x_{i1}+w_2x_{i2}+\dots +w_px_{ip} >0,
|
||
$$
|
||
|
||
<p>if our output \( y_i=1 \).
|
||
In this case we say that \( \boldsymbol{x}_i \) lies on one of the sides of the hyperplane and if
|
||
</p>
|
||
$$
|
||
b+w_1x_{i1}+w_2x_{i2}+\dots +w_px_{ip} < 0,
|
||
$$
|
||
|
||
<p>for the class of observations \( y_i=-1 \),
|
||
then \( \boldsymbol{x}_i \) lies on the other side.
|
||
</p>
|
||
|
||
<p>Equivalently, for the two classes of observations we have </p>
|
||
$$
|
||
y_i\left(b+w_1x_{i1}+w_2x_{i2}+\dots +w_px_{ip}\right) > 0.
|
||
$$
|
||
|
||
<p>When we try to separate hyperplanes, if it exists, we can use it to construct a natural classifier: a test observation is assigned a given class depending on which side of the hyperplane it is located.</p>
|
||
|
||
<!-- !split -->
|
||
<h2 id="the-two-dimensional-case">The two-dimensional case </h2>
|
||
|
||
<p>Let us try to develop our intuition about SVMs by limiting ourselves to a two-dimensional
|
||
plane. To separate the two classes of data points, there are many
|
||
possible lines (hyperplanes if you prefer a more strict naming)
|
||
that could be chosen. Our objective is to find a
|
||
plane that has the maximum margin, i.e the maximum distance between
|
||
data points of both classes. Maximizing the margin distance provides
|
||
some reinforcement so that future data points can be classified with
|
||
more confidence.
|
||
</p>
|
||
|
||
<p>What a linear classifier attempts to accomplish is to split the
|
||
feature space into two half spaces by placing a hyperplane between the
|
||
data points. This hyperplane will be our decision boundary. All
|
||
points on one side of the plane will belong to class one and all points
|
||
on the other side of the plane will belong to the second class two.
|
||
</p>
|
||
|
||
<p>Unfortunately there are many ways in which we can place a hyperplane
|
||
to divide the data. Below is an example of two candidate hyperplanes
|
||
for our data sample.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="getting-into-the-details">Getting into the details </h2>
|
||
|
||
<p>Let us define the function</p>
|
||
$$
|
||
f(x) = \boldsymbol{w}^T\boldsymbol{x}+b = 0,
|
||
$$
|
||
|
||
<p>as the function that determines the line \( L \) that separates two classes (our two features), see the figure here. </p>
|
||
|
||
<p>Any point defined by \( \boldsymbol{x}_i \) and \( \boldsymbol{x}_2 \) on the line \( L \) will satisfy \( \boldsymbol{w}^T(\boldsymbol{x}_1-\boldsymbol{x}_2)=0 \). </p>
|
||
|
||
<p>The signed distance \( \delta \) from any point defined by a vector \( \boldsymbol{x} \) and a point \( \boldsymbol{x}_0 \) on the line \( L \) is then</p>
|
||
$$
|
||
\delta = \frac{1}{\vert\vert \boldsymbol{w}\vert\vert}(\boldsymbol{w}^T\boldsymbol{x}+b).
|
||
$$
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="first-attempt-at-a-minimization-approach">First attempt at a minimization approach </h2>
|
||
|
||
<p>How do we find the parameter \( b \) and the vector \( \boldsymbol{w} \)? What we could
|
||
do is to define a cost function which now contains the set of all
|
||
misclassified points \( M \) and attempt to minimize this function
|
||
</p>
|
||
|
||
$$
|
||
C(\boldsymbol{w},b) = -\sum_{i\in M} y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b).
|
||
$$
|
||
|
||
<p>We could now for example define all values \( y_i =1 \) as misclassified in case we have \( \boldsymbol{w}^T\boldsymbol{x}_i+b < 0 \) and the opposite if we have \( y_i=-1 \). Taking the derivatives gives us</p>
|
||
$$
|
||
\frac{\partial C}{\partial b} = -\sum_{i\in M} y_i,
|
||
$$
|
||
|
||
<p>and </p>
|
||
$$
|
||
\frac{\partial C}{\partial \boldsymbol{w}} = -\sum_{i\in M} y_ix_i.
|
||
$$
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="solving-the-equations">Solving the equations </h2>
|
||
|
||
<p>We can now use the Newton-Raphson method or different variants of the gradient descent family (from plain gradient descent to various stochastic gradient descent approaches) to solve the equations</p>
|
||
$$
|
||
b \leftarrow b +\eta \frac{\partial C}{\partial b},
|
||
$$
|
||
|
||
<p>and</p>
|
||
$$
|
||
\boldsymbol{w} \leftarrow \boldsymbol{w} +\eta \frac{\partial C}{\partial \boldsymbol{w}},
|
||
$$
|
||
|
||
<p>where \( \eta \) is our by now well-known learning rate. </p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="code-example">Code Example </h2>
|
||
|
||
<p>The equations we discussed above can be coded rather easily (the
|
||
framework is similar to what we developed for logistic
|
||
regression). We are going to set up a simple case with two classes only and we want to find a line which separates them the best possible way.
|
||
</p>
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;">
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="problems-with-the-simpler-approach">Problems with the Simpler Approach </h2>
|
||
|
||
<p>There are however problems with this approach, although it looks
|
||
pretty straightforward to implement. When running the above code, we see that we can easily end up with many diffeent lines which separate the two classes.
|
||
</p>
|
||
|
||
<p>For small
|
||
gaps between the entries, we may also end up needing many iterations
|
||
before the solutions converge and if the data cannot be separated
|
||
properly into two distinct classes, we may not experience a converge
|
||
at all.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="a-better-approach">A better approach </h2>
|
||
|
||
<p>A better approach is rather to try to define a large margin between
|
||
the two classes (if they are well separated from the beginning).
|
||
</p>
|
||
|
||
<p>Thus, we wish to find a margin \( M \) with \( \boldsymbol{w} \) normalized to
|
||
\( \vert\vert \boldsymbol{w}\vert\vert =1 \) subject to the condition
|
||
</p>
|
||
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, p.
|
||
$$
|
||
|
||
<p>All points are thus at a signed distance from the decision boundary defined by the line \( L \). The parameters \( b \) and \( w_1 \) and \( w_2 \) define this line. </p>
|
||
|
||
<p>We seek thus the largest value \( M \) defined by</p>
|
||
$$
|
||
\frac{1}{\vert \vert \boldsymbol{w}\vert\vert}y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M \hspace{0.1cm}\forall i=1,2,\dots, n,
|
||
$$
|
||
|
||
<p>or just </p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq M\vert \vert \boldsymbol{w}\vert\vert \hspace{0.1cm}\forall i.
|
||
$$
|
||
|
||
<p>If we scale the equation so that \( \vert \vert \boldsymbol{w}\vert\vert = 1/M \), we have to find the minimum of
|
||
\( \boldsymbol{w}^T\boldsymbol{w}=\vert \vert \boldsymbol{w}\vert\vert \) (the norm) subject to the condition
|
||
</p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) \geq 1 \hspace{0.1cm}\forall i.
|
||
$$
|
||
|
||
<p>We have thus defined our margin as the invers of the norm of
|
||
\( \boldsymbol{w} \). We want to minimize the norm in order to have a as large as
|
||
possible margin \( M \). Before we proceed, we need to remind ourselves
|
||
about Lagrangian multipliers.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="a-quick-reminder-on-lagrangian-multipliers">A quick Reminder on Lagrangian Multipliers </h2>
|
||
|
||
<p>Consider a function of three independent variables \( f(x,y,z) \) . For the function \( f \) to be an
|
||
extreme we have
|
||
</p>
|
||
$$
|
||
df=0.
|
||
$$
|
||
|
||
<p>A necessary and sufficient condition is</p>
|
||
$$
|
||
\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
|
||
$$
|
||
|
||
<p>due to</p>
|
||
$$
|
||
df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz.
|
||
$$
|
||
|
||
<p>In many problems the variables \( x,y,z \) are often subject to constraints (such as those above for the margin)
|
||
so that they are no longer all independent. It is possible at least in principle to use each
|
||
constraint to eliminate one variable
|
||
and to proceed with a new and smaller set of independent varables.
|
||
</p>
|
||
|
||
<p>The use of so-called Lagrangian multipliers is an alternative technique when the elimination
|
||
of variables is incovenient or undesirable. Assume that we have an equation of constraint on
|
||
the variables \( x,y,z \)
|
||
</p>
|
||
$$
|
||
\phi(x,y,z) = 0,
|
||
$$
|
||
|
||
<p> resulting in</p>
|
||
$$
|
||
d\phi = \frac{\partial \phi}{\partial x}dx+\frac{\partial \phi}{\partial y}dy+\frac{\partial \phi}{\partial z}dz =0.
|
||
$$
|
||
|
||
<p>Now we cannot set anymore</p>
|
||
$$
|
||
\frac{\partial f}{\partial x} =\frac{\partial f}{\partial y}=\frac{\partial f}{\partial z}=0,
|
||
$$
|
||
|
||
<p>if \( df=0 \) is wanted
|
||
because there are now only two independent variables! Assume \( x \) and \( y \) are the independent
|
||
variables.
|
||
Then \( dz \) is no longer arbitrary.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="adding-the-multiplier">Adding the Multiplier </h2>
|
||
|
||
<p>However, we can add to</p>
|
||
$$
|
||
df = \frac{\partial f}{\partial x}dx+\frac{\partial f}{\partial y}dy+\frac{\partial f}{\partial z}dz,
|
||
$$
|
||
|
||
<p>a multiplum of \( d\phi \), viz. \( \lambda d\phi \), resulting in</p>
|
||
$$
|
||
df+\lambda d\phi = (\frac{\partial f}{\partial z}+\lambda
|
||
\frac{\partial \phi}{\partial x})dx+(\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y})dy+
|
||
(\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z})dz =0.
|
||
$$
|
||
|
||
<p>Our multiplier is chosen so that</p>
|
||
$$
|
||
\frac{\partial f}{\partial z}+\lambda\frac{\partial \phi}{\partial z} =0.
|
||
$$
|
||
|
||
<p>We need to remember that we took \( dx \) and \( dy \) to be arbitrary and thus we must have</p>
|
||
$$
|
||
\frac{\partial f}{\partial x}+\lambda\frac{\partial \phi}{\partial x} =0,
|
||
$$
|
||
|
||
<p>and</p>
|
||
$$
|
||
\frac{\partial f}{\partial y}+\lambda\frac{\partial \phi}{\partial y} =0.
|
||
$$
|
||
|
||
<p>When all these equations are satisfied, \( df=0 \). We have four unknowns, \( x,y,z \) and
|
||
\( \lambda \). Actually we want only \( x,y,z \), \( \lambda \) needs not to be determined,
|
||
it is therefore often called
|
||
Lagrange's undetermined multiplier.
|
||
If we have a set of constraints \( \phi_k \) we have the equations
|
||
</p>
|
||
$$
|
||
\frac{\partial f}{\partial x_i}+\sum_k\lambda_k\frac{\partial \phi_k}{\partial x_i} =0.
|
||
$$
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="setting-up-the-problem">Setting up the Problem </h2>
|
||
<p>In order to solve the above problem, we define the following Lagrangian function to be minimized </p>
|
||
$$
|
||
{\cal L}(\lambda,b,\boldsymbol{w})=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-1\right],
|
||
$$
|
||
|
||
<p>where \( \lambda_i \) is a so-called Lagrange multiplier subject to the condition \( \lambda_i \geq 0 \).</p>
|
||
|
||
<p>Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain </p>
|
||
$$
|
||
\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
|
||
$$
|
||
|
||
<p>and </p>
|
||
$$
|
||
\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
|
||
$$
|
||
|
||
<p>Inserting these constraints into the equation for \( {\cal L} \) we obtain</p>
|
||
$$
|
||
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
|
||
$$
|
||
|
||
<p>subject to the constraints \( \lambda_i\geq 0 \) and \( \sum_i\lambda_iy_i=0 \).
|
||
We must in addition satisfy the <a href="https://en.wikipedia.org/wiki/Karush%E2%80%93Kuhn%E2%80%93Tucker_conditions" target="_blank">Karush-Kuhn-Tucker</a> (KKT) condition
|
||
</p>
|
||
$$
|
||
\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -1\right] \hspace{0.1cm}\forall i.
|
||
$$
|
||
|
||
<ol>
|
||
<li> If \( \lambda_i > 0 \), then \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) and we say that \( x_i \) is on the boundary.</li>
|
||
<li> If \( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)> 1 \), we say \( x_i \) is not on the boundary and we set \( \lambda_i=0 \).</li>
|
||
</ol>
|
||
<p>When \( \lambda_i > 0 \), the vectors \( \boldsymbol{x}_i \) are called support vectors. They are the vectors closest to the line (or hyperplane) and define the margin \( M \). </p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="the-problem-to-solve">The problem to solve </h2>
|
||
|
||
<p>We can rewrite </p>
|
||
$$
|
||
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
|
||
$$
|
||
|
||
<p>and its constraints in terms of a matrix-vector problem where we minimize w.r.t. \( \lambda \) the following problem</p>
|
||
$$
|
||
\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1\boldsymbol{x}_1^T\boldsymbol{x}_1 & y_1y_2\boldsymbol{x}_1^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_1^T\boldsymbol{x}_n \\
|
||
y_2y_1\boldsymbol{x}_2^T\boldsymbol{x}_1 & y_2y_2\boldsymbol{x}_2^T\boldsymbol{x}_2 & \dots & \dots & y_1y_n\boldsymbol{x}_2^T\boldsymbol{x}_n \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
y_ny_1\boldsymbol{x}_n^T\boldsymbol{x}_1 & y_ny_2\boldsymbol{x}_n^T\boldsymbol{x}_2 & \dots & \dots & y_ny_n\boldsymbol{x}_n^T\boldsymbol{x}_n \\
|
||
\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
|
||
$$
|
||
|
||
<p>subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
|
||
\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="the-last-steps">The last steps </h2>
|
||
|
||
<p>Solving the above problem, yields the values of \( \lambda_i \).
|
||
To find the coefficients of your hyperplane we need simply to compute
|
||
</p>
|
||
$$
|
||
\boldsymbol{w}=\sum_{i} \lambda_iy_i\boldsymbol{x}_i.
|
||
$$
|
||
|
||
<p>With our vector \( \boldsymbol{w} \) we can in turn find the value of the intercept \( b \) (here in two dimensions) via </p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1,
|
||
$$
|
||
|
||
<p>resulting in</p>
|
||
$$
|
||
b = \frac{1}{y_i}-\boldsymbol{w}^T\boldsymbol{x}_i,
|
||
$$
|
||
|
||
<p>or if we write it out in terms of the support vectors only, with \( N_s \) being their number, we have</p>
|
||
$$
|
||
b = \frac{1}{N_s}\sum_{j\in N_s}\left(y_j-\sum_{i=1}^n\lambda_iy_i\boldsymbol{x}_i^T\boldsymbol{x}_j\right).
|
||
$$
|
||
|
||
<p>With our hyperplane coefficients we can use our classifier to assign any observation by simply using </p>
|
||
$$
|
||
y_i = \mathrm{sign}(\boldsymbol{w}^T\boldsymbol{x}_i+b).
|
||
$$
|
||
|
||
<p>Below we discuss how to find the optimal values of \( \lambda_i \). Before we proceed however, we discuss now the so-called soft classifier. </p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="a-soft-classifier">A soft classifier </h2>
|
||
|
||
<p>Till now, the margin is strictly defined by the support vectors. This defines what is called a hard classifier, that is the margins are well defined.</p>
|
||
|
||
<p>Suppose now that classes overlap in feature space, as shown in the
|
||
figure here. One way to deal with this problem before we define the
|
||
so-called <b>kernel approach</b>, is to allow a kind of slack in the sense
|
||
that we allow some points to be on the wrong side of the margin.
|
||
</p>
|
||
|
||
<p>We introduce thus the so-called <b>slack</b> variables \( \boldsymbol{\xi} =[\xi_1,x_2,\dots,x_n] \) and
|
||
modify our previous equation
|
||
</p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1,
|
||
$$
|
||
|
||
<p>to </p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i,
|
||
$$
|
||
|
||
<p>with the requirement \( \xi_i\geq 0 \). The total violation is now \( \sum_i\xi \).
|
||
The value \( \xi_i \) in the constraint the last constraint corresponds to the amount by which the prediction
|
||
\( y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1 \) is on the wrong side of its margin. Hence by bounding the sum \( \sum_i \xi_i \),
|
||
we bound the total amount by which predictions fall on the wrong side of their margins.
|
||
</p>
|
||
|
||
<p>Misclassifications occur when \( \xi_i > 1 \). Thus bounding the total sum by some value \( C \) bounds in turn the total number of
|
||
misclassifications.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="soft-optmization-problem">Soft optmization problem </h2>
|
||
|
||
<p>This has in turn the consequences that we change our optmization problem to finding the minimum of </p>
|
||
$$
|
||
{\cal L}=\frac{1}{2}\boldsymbol{w}^T\boldsymbol{w}-\sum_{i=1}^n\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)-(1-\xi_)\right]+C\sum_{i=1}^n\xi_i-\sum_{i=1}^n\gamma_i\xi_i,
|
||
$$
|
||
|
||
<p>subject to </p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b)=1-\xi_i \hspace{0.1cm}\forall i,
|
||
$$
|
||
|
||
<p>with the requirement \( \xi_i\geq 0 \).</p>
|
||
|
||
<p>Taking the derivatives with respect to \( b \) and \( \boldsymbol{w} \) we obtain </p>
|
||
$$
|
||
\frac{\partial {\cal L}}{\partial b} = -\sum_{i} \lambda_iy_i=0,
|
||
$$
|
||
|
||
<p>and </p>
|
||
$$
|
||
\frac{\partial {\cal L}}{\partial \boldsymbol{w}} = 0 = \boldsymbol{w}-\sum_{i} \lambda_iy_i\boldsymbol{x}_i,
|
||
$$
|
||
|
||
<p>and</p>
|
||
$$
|
||
\lambda_i = C-\gamma_i \hspace{0.1cm}\forall i.
|
||
$$
|
||
|
||
<p>Inserting these constraints into the equation for \( {\cal L} \) we obtain the same equation as before</p>
|
||
$$
|
||
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{x}_j,
|
||
$$
|
||
|
||
<p>but now subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) and \( 0\leq\lambda_i \leq C \).
|
||
We must in addition satisfy the Karush-Kuhn-Tucker condition which now reads
|
||
</p>
|
||
$$
|
||
\lambda_i\left[y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_)\right]=0 \hspace{0.1cm}\forall i,
|
||
$$
|
||
|
||
$$
|
||
\gamma_i\xi_i = 0,
|
||
$$
|
||
|
||
<p>and </p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{x}_i+b) -(1-\xi_) \geq 0 \hspace{0.1cm}\forall i.
|
||
$$
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="kernels-and-non-linearity">Kernels and non-linearity </h2>
|
||
|
||
<p>The cases we have studied till now, were all characterized by two classes
|
||
with a close to linear separability. The classifiers we have described
|
||
so far find linear boundaries in our input feature space. It is
|
||
possible to make our procedure more flexible by exploring the feature
|
||
space using other basis expansions such as higher-order polynomials,
|
||
wavelets, splines etc.
|
||
</p>
|
||
|
||
<p>If our feature space is not easy to separate, as shown in the figure
|
||
here, we can achieve a better separation by introducing more complex
|
||
basis functions. The ideal would be, as shown in the next figure, to, via a specific transformation to
|
||
obtain a separation between the classes which is almost linear.
|
||
</p>
|
||
|
||
<p>The change of basis, from \( x\rightarrow z=\phi(x) \) leads to the same type of equations to be solved, except that
|
||
we need to introduce for example a polynomial transformation to a two-dimensional training set.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||
|
||
np.random.seed(<span style="color: #B452CD">42</span>)
|
||
|
||
<span style="color: #228B22"># To plot pretty figures</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||
plt.rcParams[<span style="color: #CD5555">'axes.labelsize'</span>] = <span style="color: #B452CD">14</span>
|
||
plt.rcParams[<span style="color: #CD5555">'xtick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
plt.rcParams[<span style="color: #CD5555">'ytick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
||
|
||
|
||
|
||
X1D = np.linspace(-<span style="color: #B452CD">4</span>, <span style="color: #B452CD">4</span>, <span style="color: #B452CD">9</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||
X2D = np.c_[X1D, X1D**<span style="color: #B452CD">2</span>]
|
||
y = np.array([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>])
|
||
|
||
plt.figure(figsize=(<span style="color: #B452CD">11</span>, <span style="color: #B452CD">4</span>))
|
||
|
||
plt.subplot(<span style="color: #B452CD">121</span>)
|
||
plt.grid(<span style="color: #8B008B; font-weight: bold">True</span>, which=<span style="color: #CD5555">'both'</span>)
|
||
plt.axhline(y=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.plot(X1D[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">0</span>], np.zeros(<span style="color: #B452CD">4</span>), <span style="color: #CD5555">"bs"</span>)
|
||
plt.plot(X1D[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">1</span>], np.zeros(<span style="color: #B452CD">5</span>), <span style="color: #CD5555">"g^"</span>)
|
||
plt.gca().get_yaxis().set_ticks([])
|
||
plt.xlabel(<span style="color: #CD5555">r"$x_1$"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.axis([-<span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">4.5</span>, -<span style="color: #B452CD">0.2</span>, <span style="color: #B452CD">0.2</span>])
|
||
|
||
plt.subplot(<span style="color: #B452CD">122</span>)
|
||
plt.grid(<span style="color: #8B008B; font-weight: bold">True</span>, which=<span style="color: #CD5555">'both'</span>)
|
||
plt.axhline(y=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.axvline(x=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.plot(X2D[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">0</span>], X2D[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">0</span>], <span style="color: #CD5555">"bs"</span>)
|
||
plt.plot(X2D[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">1</span>], X2D[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">1</span>], <span style="color: #CD5555">"g^"</span>)
|
||
plt.xlabel(<span style="color: #CD5555">r"$x_1$"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.ylabel(<span style="color: #CD5555">r"$x_2$"</span>, fontsize=<span style="color: #B452CD">20</span>, rotation=<span style="color: #B452CD">0</span>)
|
||
plt.gca().get_yaxis().set_ticks([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">4</span>, <span style="color: #B452CD">8</span>, <span style="color: #B452CD">12</span>, <span style="color: #B452CD">16</span>])
|
||
plt.plot([-<span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">4.5</span>], [<span style="color: #B452CD">6.5</span>, <span style="color: #B452CD">6.5</span>], <span style="color: #CD5555">"r--"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||
plt.axis([-<span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">4.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">17</span>])
|
||
plt.subplots_adjust(right=<span style="color: #B452CD">1</span>)
|
||
plt.show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="the-equations">The equations </h2>
|
||
|
||
<p>Suppose we define a polynomial transformation of degree two only (we continue to live in a plane with \( x_i \) and \( y_i \) as variables)</p>
|
||
$$
|
||
z = \phi(x_i) =\left(x_i^2, y_i^2, \sqrt{2}x_iy_i\right).
|
||
$$
|
||
|
||
<p>With our new basis, the equations we solved earlier are basically the same, that is we have now (without the slack option for simplicity)</p>
|
||
$$
|
||
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{z}_i^T\boldsymbol{z}_j,
|
||
$$
|
||
|
||
<p>subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \), and for the support vectors</p>
|
||
$$
|
||
y_i(\boldsymbol{w}^T\boldsymbol{z}_i+b)= 1 \hspace{0.1cm}\forall i,
|
||
$$
|
||
|
||
<p>from which we also find \( b \).
|
||
To compute \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we define the kernel \( K(\boldsymbol{x}_i,\boldsymbol{x}_j) \) as
|
||
</p>
|
||
$$
|
||
K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\boldsymbol{z}_i^T\boldsymbol{z}_j= \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
|
||
$$
|
||
|
||
<p>For the above example, the kernel reads</p>
|
||
$$
|
||
K(\boldsymbol{x}_i,\boldsymbol{x}_j)=[x_i^2, y_i^2, \sqrt{2}x_iy_i]^T\begin{bmatrix} x_j^2 \\ y_j^2 \\ \sqrt{2}x_jy_j \end{bmatrix}=x_i^2x_j^2+2x_ix_jy_iy_j+y_i^2y_j^2.
|
||
$$
|
||
|
||
<p>We note that this is nothing but the dot product of the two original
|
||
vectors \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \). Instead of thus computing the
|
||
product in the Lagrangian of \( \boldsymbol{z}_i^T\boldsymbol{z}_j \) we simply compute
|
||
the dot product \( (\boldsymbol{x}_i^T\boldsymbol{x}_j)^2 \).
|
||
</p>
|
||
|
||
<p>This leads to the so-called
|
||
kernel trick and the result leads to the same as if we went through
|
||
the trouble of performing the transformation
|
||
\( \phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j) \) during the SVM calculations.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="the-problem-to-solve">The problem to solve </h2>
|
||
<p>Using our definition of the kernel We can rewrite again the Lagrangian</p>
|
||
$$
|
||
{\cal L}=\sum_i\lambda_i-\frac{1}{2}\sum_{ij}^n\lambda_i\lambda_jy_iy_j\boldsymbol{x}_i^T\boldsymbol{z}_j,
|
||
$$
|
||
|
||
<p>subject to the constraints \( \lambda_i\geq 0 \), \( \sum_i\lambda_iy_i=0 \) in terms of a convex optimization problem</p>
|
||
$$
|
||
\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\
|
||
y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\
|
||
\end{bmatrix}\boldsymbol{\lambda}-\mathbb{1}\boldsymbol{\lambda},
|
||
$$
|
||
|
||
<p>subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
|
||
\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
|
||
If we add the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \).
|
||
</p>
|
||
|
||
<p>We can rewrite this (see the solutions below) in terms of a convex optimization problem of the type</p>
|
||
$$
|
||
\begin{align*}
|
||
&\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
|
||
&\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \hspace{0.2cm} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
|
||
\end{align*}
|
||
$$
|
||
|
||
<p>Below we discuss how to solve these equations. Here we note that the matrix \( \boldsymbol{P} \) has matrix elements \( p_{ij}=y_iy_jK(\boldsymbol{x}_i,\boldsymbol{x}_j) \).
|
||
Given a kernel \( K \) and the targets \( y_i \) this matrix is easy to set up. The constraint \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \) leads to \( f=0 \) and \( \boldsymbol{A}=\boldsymbol{y} \). How to set up the matrix \( \boldsymbol{G} \) is discussed later. Here note that the inequalities \( 0\leq \lambda_i \leq C \) can be split up into
|
||
\( 0\leq \lambda_i \) and \( \lambda_i \leq C \). These two inequalities define then the matrix \( \boldsymbol{G} \) and the vector \( \boldsymbol{h} \).
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="different-kernels-and-mercer-s-theorem">Different kernels and Mercer's theorem </h2>
|
||
|
||
<p>There are several popular kernels being used. These are</p>
|
||
<ol>
|
||
<li> Linear: \( K(\boldsymbol{x},\boldsymbol{y})=\boldsymbol{x}^T\boldsymbol{y} \),</li>
|
||
<li> Polynomial: \( K(\boldsymbol{x},\boldsymbol{y})=(\boldsymbol{x}^T\boldsymbol{y}+\gamma)^d \),</li>
|
||
<li> Gaussian Radial Basis Function: \( K(\boldsymbol{x},\boldsymbol{y})=\exp{\left(-\gamma\vert\vert\boldsymbol{x}-\boldsymbol{y}\vert\vert^2\right)} \),</li>
|
||
<li> Tanh: \( K(\boldsymbol{x},\boldsymbol{y})=\tanh{(\boldsymbol{x}^T\boldsymbol{y}+\gamma)} \),</li>
|
||
</ol>
|
||
<p>and many other ones.</p>
|
||
|
||
<p>An important theorem for us is <a href="https://en.wikipedia.org/wiki/Mercer%27s_theorem" target="_blank">Mercer's
|
||
theorem</a>. The
|
||
theorem states that if a kernel function \( K \) is symmetric, continuous
|
||
and leads to a positive semi-definite matrix \( \boldsymbol{P} \) then there
|
||
exists a function \( \phi \) that maps \( \boldsymbol{x}_i \) and \( \boldsymbol{x}_j \) into
|
||
another space (possibly with much higher dimensions) such that
|
||
</p>
|
||
|
||
$$
|
||
K(\boldsymbol{x}_i,\boldsymbol{x}_j)=\phi(\boldsymbol{x}_i)^T\phi(\boldsymbol{x}_j).
|
||
$$
|
||
|
||
<p>So you can use \( K \) as a kernel since you know \( \phi \) exists, even if
|
||
you don’t know what \( \phi \) is.
|
||
</p>
|
||
|
||
<p>Note that some frequently used kernels (such as the Sigmoid kernel)
|
||
don’t respect all of Mercer’s conditions, yet they generally work well
|
||
in practice.
|
||
</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="the-moons-example">The moons example </h2>
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">__future__</span> <span style="color: #8B008B; font-weight: bold">import</span> division, print_function, unicode_literals
|
||
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||
np.random.seed(<span style="color: #B452CD">42</span>)
|
||
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||
plt.rcParams[<span style="color: #CD5555">'axes.labelsize'</span>] = <span style="color: #B452CD">14</span>
|
||
plt.rcParams[<span style="color: #CD5555">'xtick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
plt.rcParams[<span style="color: #CD5555">'ytick.labelsize'</span>] = <span style="color: #B452CD">12</span>
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
||
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> LinearSVC
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||
X, y = make_moons(n_samples=<span style="color: #B452CD">100</span>, noise=<span style="color: #B452CD">0.15</span>, random_state=<span style="color: #B452CD">42</span>)
|
||
|
||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_dataset</span>(X, y, axes):
|
||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">0</span>], <span style="color: #CD5555">"bs"</span>)
|
||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">1</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">1</span>], <span style="color: #CD5555">"g^"</span>)
|
||
plt.axis(axes)
|
||
plt.grid(<span style="color: #8B008B; font-weight: bold">True</span>, which=<span style="color: #CD5555">'both'</span>)
|
||
plt.xlabel(<span style="color: #CD5555">r"$x_1$"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.ylabel(<span style="color: #CD5555">r"$x_2$"</span>, fontsize=<span style="color: #B452CD">20</span>, rotation=<span style="color: #B452CD">0</span>)
|
||
|
||
plot_dataset(X, y, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plt.show()
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.pipeline</span> <span style="color: #8B008B; font-weight: bold">import</span> Pipeline
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> PolynomialFeatures
|
||
|
||
polynomial_svm_clf = Pipeline([
|
||
(<span style="color: #CD5555">"poly_features"</span>, PolynomialFeatures(degree=<span style="color: #B452CD">3</span>)),
|
||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||
(<span style="color: #CD5555">"svm_clf"</span>, LinearSVC(C=<span style="color: #B452CD">10</span>, loss=<span style="color: #CD5555">"hinge"</span>, random_state=<span style="color: #B452CD">42</span>))
|
||
])
|
||
|
||
polynomial_svm_clf.fit(X, y)
|
||
|
||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_predictions</span>(clf, axes):
|
||
x0s = np.linspace(axes[<span style="color: #B452CD">0</span>], axes[<span style="color: #B452CD">1</span>], <span style="color: #B452CD">100</span>)
|
||
x1s = np.linspace(axes[<span style="color: #B452CD">2</span>], axes[<span style="color: #B452CD">3</span>], <span style="color: #B452CD">100</span>)
|
||
x0, x1 = np.meshgrid(x0s, x1s)
|
||
X = np.c_[x0.ravel(), x1.ravel()]
|
||
y_pred = clf.predict(X).reshape(x0.shape)
|
||
y_decision = clf.decision_function(X).reshape(x0.shape)
|
||
plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=<span style="color: #B452CD">0.2</span>)
|
||
plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=<span style="color: #B452CD">0.1</span>)
|
||
|
||
plot_predictions(polynomial_svm_clf, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plot_dataset(X, y, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
|
||
plt.show()
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||
|
||
poly_kernel_svm_clf = Pipeline([
|
||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"poly"</span>, degree=<span style="color: #B452CD">3</span>, coef0=<span style="color: #B452CD">1</span>, C=<span style="color: #B452CD">5</span>))
|
||
])
|
||
poly_kernel_svm_clf.fit(X, y)
|
||
|
||
poly100_kernel_svm_clf = Pipeline([
|
||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"poly"</span>, degree=<span style="color: #B452CD">10</span>, coef0=<span style="color: #B452CD">100</span>, C=<span style="color: #B452CD">5</span>))
|
||
])
|
||
poly100_kernel_svm_clf.fit(X, y)
|
||
|
||
plt.figure(figsize=(<span style="color: #B452CD">11</span>, <span style="color: #B452CD">4</span>))
|
||
|
||
plt.subplot(<span style="color: #B452CD">121</span>)
|
||
plot_predictions(poly_kernel_svm_clf, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plot_dataset(X, y, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plt.title(<span style="color: #CD5555">r"$d=3, r=1, C=5$"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||
|
||
plt.subplot(<span style="color: #B452CD">122</span>)
|
||
plot_predictions(poly100_kernel_svm_clf, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plot_dataset(X, y, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plt.title(<span style="color: #CD5555">r"$d=10, r=100, C=5$"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||
|
||
plt.show()
|
||
|
||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gaussian_rbf</span>(x, landmark, gamma):
|
||
<span style="color: #8B008B; font-weight: bold">return</span> np.exp(-gamma * np.linalg.norm(x - landmark, axis=<span style="color: #B452CD">1</span>)**<span style="color: #B452CD">2</span>)
|
||
|
||
gamma = <span style="color: #B452CD">0.3</span>
|
||
|
||
x1s = np.linspace(-<span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">200</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||
x2s = gaussian_rbf(x1s, -<span style="color: #B452CD">2</span>, gamma)
|
||
x3s = gaussian_rbf(x1s, <span style="color: #B452CD">1</span>, gamma)
|
||
|
||
XK = np.c_[gaussian_rbf(X1D, -<span style="color: #B452CD">2</span>, gamma), gaussian_rbf(X1D, <span style="color: #B452CD">1</span>, gamma)]
|
||
yk = np.array([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>])
|
||
|
||
plt.figure(figsize=(<span style="color: #B452CD">11</span>, <span style="color: #B452CD">4</span>))
|
||
|
||
plt.subplot(<span style="color: #B452CD">121</span>)
|
||
plt.grid(<span style="color: #8B008B; font-weight: bold">True</span>, which=<span style="color: #CD5555">'both'</span>)
|
||
plt.axhline(y=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.scatter(x=[-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">1</span>], y=[<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], s=<span style="color: #B452CD">150</span>, alpha=<span style="color: #B452CD">0.5</span>, c=<span style="color: #CD5555">"red"</span>)
|
||
plt.plot(X1D[:, <span style="color: #B452CD">0</span>][yk==<span style="color: #B452CD">0</span>], np.zeros(<span style="color: #B452CD">4</span>), <span style="color: #CD5555">"bs"</span>)
|
||
plt.plot(X1D[:, <span style="color: #B452CD">0</span>][yk==<span style="color: #B452CD">1</span>], np.zeros(<span style="color: #B452CD">5</span>), <span style="color: #CD5555">"g^"</span>)
|
||
plt.plot(x1s, x2s, <span style="color: #CD5555">"g--"</span>)
|
||
plt.plot(x1s, x3s, <span style="color: #CD5555">"b:"</span>)
|
||
plt.gca().get_yaxis().set_ticks([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">0.25</span>, <span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">0.75</span>, <span style="color: #B452CD">1</span>])
|
||
plt.xlabel(<span style="color: #CD5555">r"$x_1$"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.ylabel(<span style="color: #CD5555">r"Similarity"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||
plt.annotate(<span style="color: #CD5555">r'$\mathbf{x}$'</span>,
|
||
xy=(X1D[<span style="color: #B452CD">3</span>, <span style="color: #B452CD">0</span>], <span style="color: #B452CD">0</span>),
|
||
xytext=(-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">0.20</span>),
|
||
ha=<span style="color: #CD5555">"center"</span>,
|
||
arrowprops=<span style="color: #658b00">dict</span>(facecolor=<span style="color: #CD5555">'black'</span>, shrink=<span style="color: #B452CD">0.1</span>),
|
||
fontsize=<span style="color: #B452CD">18</span>,
|
||
)
|
||
plt.text(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">0.9</span>, <span style="color: #CD5555">"$x_2$"</span>, ha=<span style="color: #CD5555">"center"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.text(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.9</span>, <span style="color: #CD5555">"$x_3$"</span>, ha=<span style="color: #CD5555">"center"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.axis([-<span style="color: #B452CD">4.5</span>, <span style="color: #B452CD">4.5</span>, -<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
|
||
|
||
plt.subplot(<span style="color: #B452CD">122</span>)
|
||
plt.grid(<span style="color: #8B008B; font-weight: bold">True</span>, which=<span style="color: #CD5555">'both'</span>)
|
||
plt.axhline(y=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.axvline(x=<span style="color: #B452CD">0</span>, color=<span style="color: #CD5555">'k'</span>)
|
||
plt.plot(XK[:, <span style="color: #B452CD">0</span>][yk==<span style="color: #B452CD">0</span>], XK[:, <span style="color: #B452CD">1</span>][yk==<span style="color: #B452CD">0</span>], <span style="color: #CD5555">"bs"</span>)
|
||
plt.plot(XK[:, <span style="color: #B452CD">0</span>][yk==<span style="color: #B452CD">1</span>], XK[:, <span style="color: #B452CD">1</span>][yk==<span style="color: #B452CD">1</span>], <span style="color: #CD5555">"g^"</span>)
|
||
plt.xlabel(<span style="color: #CD5555">r"$x_2$"</span>, fontsize=<span style="color: #B452CD">20</span>)
|
||
plt.ylabel(<span style="color: #CD5555">r"$x_3$ "</span>, fontsize=<span style="color: #B452CD">20</span>, rotation=<span style="color: #B452CD">0</span>)
|
||
plt.annotate(<span style="color: #CD5555">r'$\phi\left(\mathbf{x}\right)$'</span>,
|
||
xy=(XK[<span style="color: #B452CD">3</span>, <span style="color: #B452CD">0</span>], XK[<span style="color: #B452CD">3</span>, <span style="color: #B452CD">1</span>]),
|
||
xytext=(<span style="color: #B452CD">0.65</span>, <span style="color: #B452CD">0.50</span>),
|
||
ha=<span style="color: #CD5555">"center"</span>,
|
||
arrowprops=<span style="color: #658b00">dict</span>(facecolor=<span style="color: #CD5555">'black'</span>, shrink=<span style="color: #B452CD">0.1</span>),
|
||
fontsize=<span style="color: #B452CD">18</span>,
|
||
)
|
||
plt.plot([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>], [<span style="color: #B452CD">0.57</span>, -<span style="color: #B452CD">0.1</span>], <span style="color: #CD5555">"r--"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||
plt.axis([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>, -<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
|
||
|
||
plt.subplots_adjust(right=<span style="color: #B452CD">1</span>)
|
||
|
||
plt.show()
|
||
|
||
|
||
x1_example = X1D[<span style="color: #B452CD">3</span>, <span style="color: #B452CD">0</span>]
|
||
<span style="color: #8B008B; font-weight: bold">for</span> landmark <span style="color: #8B008B">in</span> (-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">1</span>):
|
||
k = gaussian_rbf(np.array([[x1_example]]), np.array([[landmark]]), gamma)
|
||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Phi({}, {}) = {}"</span>.format(x1_example, landmark, k))
|
||
|
||
rbf_kernel_svm_clf = Pipeline([
|
||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"rbf"</span>, gamma=<span style="color: #B452CD">5</span>, C=<span style="color: #B452CD">0.001</span>))
|
||
])
|
||
rbf_kernel_svm_clf.fit(X, y)
|
||
|
||
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||
|
||
gamma1, gamma2 = <span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">5</span>
|
||
C1, C2 = <span style="color: #B452CD">0.001</span>, <span style="color: #B452CD">1000</span>
|
||
hyperparams = (gamma1, C1), (gamma1, C2), (gamma2, C1), (gamma2, C2)
|
||
|
||
svm_clfs = []
|
||
<span style="color: #8B008B; font-weight: bold">for</span> gamma, C <span style="color: #8B008B">in</span> hyperparams:
|
||
rbf_kernel_svm_clf = Pipeline([
|
||
(<span style="color: #CD5555">"scaler"</span>, StandardScaler()),
|
||
(<span style="color: #CD5555">"svm_clf"</span>, SVC(kernel=<span style="color: #CD5555">"rbf"</span>, gamma=gamma, C=C))
|
||
])
|
||
rbf_kernel_svm_clf.fit(X, y)
|
||
svm_clfs.append(rbf_kernel_svm_clf)
|
||
|
||
plt.figure(figsize=(<span style="color: #B452CD">11</span>, <span style="color: #B452CD">7</span>))
|
||
|
||
<span style="color: #8B008B; font-weight: bold">for</span> i, svm_clf <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(svm_clfs):
|
||
plt.subplot(<span style="color: #B452CD">221</span> + i)
|
||
plot_predictions(svm_clf, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
plot_dataset(X, y, [-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>])
|
||
gamma, C = hyperparams[i]
|
||
plt.title(<span style="color: #CD5555">r"$\gamma = {}, C = {}$"</span>.format(gamma, C), fontsize=<span style="color: #B452CD">16</span>)
|
||
|
||
plt.show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="mathematical-optimization-of-convex-functions">Mathematical optimization of convex functions </h2>
|
||
|
||
<p>A mathematical (quadratic) optimization problem, or just optimization problem, has the form</p>
|
||
$$
|
||
\begin{align*}
|
||
&\mathrm{min}_{\lambda}\hspace{0.2cm} \frac{1}{2}\boldsymbol{\lambda}^T\boldsymbol{P}\boldsymbol{\lambda}+\boldsymbol{q}^T\boldsymbol{\lambda},\\ \nonumber
|
||
&\mathrm{subject\hspace{0.1cm}to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{\lambda} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{\lambda}=f.
|
||
\end{align*}
|
||
$$
|
||
|
||
<p>subject to some constraints for say a selected set \( i=1,2,\dots, n \).
|
||
In our case we are optimizing with respect to the Lagrangian multipliers \( \lambda_i \), and the
|
||
vector \( \boldsymbol{\lambda}=[\lambda_1, \lambda_2,\dots, \lambda_n] \) is the optimization variable we are dealing with.
|
||
</p>
|
||
|
||
<p>In our case we are particularly interested in a class of optimization problems called convex optmization problems.
|
||
In our discussion on gradient descent methods we discussed at length the definition of a convex function.
|
||
</p>
|
||
|
||
<p>Convex optimization problems play a central role in applied mathematics and we recommend strongly <a href="http://web.stanford.edu/~boyd/cvxbook/" target="_blank">Boyd and Vandenberghe's text on the topics</a>.</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="how-do-we-solve-these-problems">How do we solve these problems? </h2>
|
||
|
||
<p>If we use Python as programming language and wish to venture beyond
|
||
<b>scikit-learn</b>, <b>tensorflow</b> and similar software which makes our
|
||
lives so much easier, we need to dive into the wonderful world of
|
||
quadratic programming. We can, if we wish, solve the minimization
|
||
problem using say standard gradient methods or conjugate gradient
|
||
methods. However, these methods tend to exhibit a rather slow
|
||
converge. So, welcome to the promised land of quadratic programming.
|
||
</p>
|
||
|
||
<p>The functions we need are contained in the quadratic programming package <b>CVXOPT</b> and we need to import it together with <b>numpy</b> as</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">cvxopt</span>
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>This will make our life much easier. You don't need t write your own optimizer.</p>
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="a-simple-example">A simple example </h2>
|
||
|
||
<p>We remind ourselves about the general problem we want to solve</p>
|
||
$$
|
||
\begin{align*}
|
||
&\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}\boldsymbol{x}^T\boldsymbol{P}\boldsymbol{x}+\boldsymbol{q}^T\boldsymbol{x},\\ \nonumber
|
||
&\mathrm{subject\hspace{0.1cm} to} \hspace{0.2cm} \boldsymbol{G}\boldsymbol{x} \preceq \boldsymbol{h} \wedge \boldsymbol{A}\boldsymbol{x}=f.
|
||
\end{align*}
|
||
$$
|
||
|
||
<p>Let us show how to perform the optmization using a simple case. Assume we want to optimize the following problem</p>
|
||
$$
|
||
\begin{align*}
|
||
&\mathrm{min}_{x}\hspace{0.2cm} \frac{1}{2}x^2+5x+3y \\ \nonumber
|
||
&\mathrm{subject to} \\ \nonumber
|
||
&x, y \geq 0 \\ \nonumber
|
||
&x+3y \geq 15 \\ \nonumber
|
||
&2x+5y \leq 100 \\ \nonumber
|
||
&3x+4y \leq 80. \\ \nonumber
|
||
\end{align*}
|
||
$$
|
||
|
||
<p>The minimization problem can be rewritten in terms of vectors and matrices as (with \( x \) and \( y \) being the unknowns)</p>
|
||
$$
|
||
\frac{1}{2}\begin{bmatrix} x\\ y \end{bmatrix}^T \begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} + \begin{bmatrix}3\\ 4 \end{bmatrix}^T \begin{bmatrix}x \\ y \end{bmatrix}.
|
||
$$
|
||
|
||
<p>Similarly, we can now set up the inequalities (we need to change \( \geq \) to \( \leq \) by multiplying with \( -1 \) on bot sides) as the following matrix-vector equation</p>
|
||
$$
|
||
\begin{bmatrix} -1 & 0 \\ 0 & -1 \\ -1 & -3 \\ 2 & 5 \\ 3 & 4\end{bmatrix}\begin{bmatrix} x \\ y\end{bmatrix} \preceq \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
|
||
$$
|
||
|
||
<p>We have collapsed all the inequalities into a single matrix \( \boldsymbol{G} \). We see also that our matrix </p>
|
||
$$
|
||
\boldsymbol{P} =\begin{bmatrix} 1 & 0\\ 0 & 0 \end{bmatrix}
|
||
$$
|
||
|
||
<p>is clearly positive semi-definite (all eigenvalues larger or equal zero).
|
||
Finally, the vector \( \boldsymbol{h} \) is defined as
|
||
</p>
|
||
$$
|
||
\boldsymbol{h} = \begin{bmatrix}0 \\ 0\\ -15 \\ 100 \\ 80\end{bmatrix}.
|
||
$$
|
||
|
||
<p>Since we don't have any equalities the matrix \( \boldsymbol{A} \) is set to zero
|
||
The following code solves the equations for us
|
||
</p>
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #eeeedd">
|
||
<pre style="line-height: 125%;"><span style="color: #228B22"># Import the necessary packages</span>
|
||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">cvxopt</span> <span style="color: #8B008B; font-weight: bold">import</span> matrix
|
||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">cvxopt</span> <span style="color: #8B008B; font-weight: bold">import</span> solvers
|
||
P = matrix(numpy.diag([<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>]), tc=<span style="color: #a61717; background-color: #e3d2d2">’</span>d<span style="color: #a61717; background-color: #e3d2d2">’</span>)
|
||
q = matrix(numpy.array([<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>]), tc=<span style="color: #a61717; background-color: #e3d2d2">’</span>d<span style="color: #a61717; background-color: #e3d2d2">’</span>)
|
||
G = matrix(numpy.array([[-<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">0</span>,-<span style="color: #B452CD">1</span>],[-<span style="color: #B452CD">1</span>,-<span style="color: #B452CD">3</span>],[<span style="color: #B452CD">2</span>,<span style="color: #B452CD">5</span>],[<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>]]), tc=<span style="color: #a61717; background-color: #e3d2d2">’</span>d<span style="color: #a61717; background-color: #e3d2d2">’</span>)
|
||
h = matrix(numpy.array([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,-<span style="color: #B452CD">15</span>,<span style="color: #B452CD">100</span>,<span style="color: #B452CD">80</span>]), tc=<span style="color: #a61717; background-color: #e3d2d2">’</span>d<span style="color: #a61717; background-color: #e3d2d2">’</span>)
|
||
<span style="color: #228B22"># Construct the QP, invoke solver</span>
|
||
sol = solvers.qp(P,q,G,h)
|
||
<span style="color: #228B22"># Extract optimal value and solution</span>
|
||
sol[<span style="color: #a61717; background-color: #e3d2d2">’</span>x<span style="color: #a61717; background-color: #e3d2d2">’</span>]
|
||
sol[<span style="color: #a61717; background-color: #e3d2d2">’</span>primal objective<span style="color: #a61717; background-color: #e3d2d2">’</span>]
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||
<h2 id="back-to-the-more-realistic-cases">Back to the more realistic cases </h2>
|
||
|
||
<p>We are now ready to return to our setup of the optmization problem for a more realistic case. Introducing the <b>slack</b> parameter \( C \) we have</p>
|
||
$$
|
||
\frac{1}{2} \boldsymbol{\lambda}^T\begin{bmatrix} y_1y_1K(\boldsymbol{x}_1,\boldsymbol{x}_1) & y_1y_2K(\boldsymbol{x}_1,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_1,\boldsymbol{x}_n) \\
|
||
y_2y_1K(\boldsymbol{x}_2,\boldsymbol{x}_1) & y_2y_2K(\boldsymbol{x}_2,\boldsymbol{x}_2) & \dots & \dots & y_1y_nK(\boldsymbol{x}_2,\boldsymbol{x}_n) \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots \\
|
||
y_ny_1K(\boldsymbol{x}_n,\boldsymbol{x}_1) & y_ny_2K(\boldsymbol{x}_n\boldsymbol{x}_2) & \dots & \dots & y_ny_nK(\boldsymbol{x}_n,\boldsymbol{x}_n) \\
|
||
\end{bmatrix}\boldsymbol{\lambda}-\mathbb{I}\boldsymbol{\lambda},
|
||
$$
|
||
|
||
<p>subject to \( \boldsymbol{y}^T\boldsymbol{\lambda}=0 \). Here we defined the vectors \( \boldsymbol{\lambda} =[\lambda_1,\lambda_2,\dots,\lambda_n] \) and
|
||
\( \boldsymbol{y}=[y_1,y_2,\dots,y_n] \).
|
||
With the slack constants this leads to the additional constraint \( 0\leq \lambda_i \leq C \).
|
||
</p>
|
||
|
||
<b>code will be added</b>
|
||
|
||
<!-- ------------------- end of main content --------------- -->
|
||
<center style="font-size:80%">
|
||
<!-- copyright --> © 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
||
</center>
|
||
</body>
|
||
</html>
|
||
|