mhjensen
697 lines
27 KiB
HTML
697 lines
27 KiB
HTML
\
|
|
<!DOCTYPE html>
|
|
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
|
|
|
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
|
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
|
|
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<!--
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
|
|
-->
|
|
|
|
<!-- For syntax highlighting -->
|
|
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
|
|
|
|
<!-- Printing and PDF exports -->
|
|
<script>
|
|
var link = document.createElement( 'link' );
|
|
link.rel = 'stylesheet';
|
|
link.type = 'text/css';
|
|
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
|
|
document.getElementsByTagName( 'head' )[0].appendChild( link );
|
|
</script>
|
|
|
|
<style type="text/css">
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
.reveal .alert-text-small { font-size: 80%; }
|
|
.reveal .alert-text-large { font-size: 130%; }
|
|
.reveal .alert-text-normal { font-size: 90%; }
|
|
.reveal .alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:5px solid #bababa;
|
|
-webkit-border-radius: 14px; -moz-border-radius: 14px;
|
|
border-radius:14px;
|
|
background-position: 10px 10px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 30px; /* 55px; if icon */
|
|
}
|
|
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
|
|
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
/*.reveal .alert li {margin-top: 1em}*/
|
|
.reveal .alert-block p+p {margin-top:5px}
|
|
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
|
|
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
|
|
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
|
|
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
|
|
|
|
</style>
|
|
|
|
|
|
|
|
<!-- Styles for table layout of slides -->
|
|
<style type="text/css">
|
|
td.padding {
|
|
padding-top:20px;
|
|
padding-bottom:20px;
|
|
padding-right:50px;
|
|
padding-left:50px;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
<body>
|
|
<div class="reveal">
|
|
|
|
<!-- Any section element inside the <div class="slides"> container
|
|
is displayed as a slide -->
|
|
|
|
<div class="slides">
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Logistic Regression</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p> <br>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p> <br>
|
|
<center><h4>Sep 26, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec0">Logistic Regression </h2>
|
|
|
|
<p>
|
|
In linear regression our main interest was centered on learning the
|
|
coefficients of a functional fit (say a polynomial) in order to be
|
|
able to predict the response of a continuous variable on some unseen
|
|
data. The fit to the continuous variable \( y_i \) is based on some
|
|
independent variables \( \hat{x}_i \). Linear regression resulted in
|
|
analytical expressions (in terms of matrices to invert) for several
|
|
quantities, ranging from the variance and thereby the confidence
|
|
intervals of the parameters \( \hat{\beta} \) to the mean squared
|
|
error. If we can invert the product of the design matrices, linear
|
|
regression gives then a simple recipe for fitting our data.
|
|
|
|
<p>
|
|
Classification problems, however, are concerned with outcomes taking
|
|
the form of discrete variables (i.e. categories). We may for example,
|
|
on the basis of DNA sequencing for a number of patients, like to find
|
|
out which mutations are important for a certain disease; or based on
|
|
scans of various patients' brains, figure out if there is a tumor or
|
|
not; or given a specific physical system, we'd like to identify its
|
|
state, say whether it is an ordered or disordered system (typical
|
|
situation in solid state physics); or classify the status of a
|
|
patient, whether she/he has a stroke or not and many other similar
|
|
situations.
|
|
|
|
<p>
|
|
The most common situation we encounter when we apply logistic
|
|
regression is that of two possible outcomes, normally denoted as a
|
|
binary outcome, true or false, positive or negative, success or
|
|
failure etc.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec1">Optimization and Deep learning </h2>
|
|
|
|
<p>
|
|
Logistic regression will also serve as our stepping stone towards neural
|
|
network algorithms and supervised deep learning. For logistic
|
|
learning, the minimization of the cost function leads to a non-linear
|
|
equation in the parameters \( \hat{\beta} \). The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms.
|
|
|
|
<p>
|
|
We note also that many of the topics discussed here
|
|
regression are also commonly used in modern supervised Deep Learning
|
|
models, as we will see later.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec2">Basics </h2>
|
|
|
|
<p>
|
|
We consider the case where the dependent variables, also called the
|
|
responses or the outcomes, \( y_i \) are discrete and only take values
|
|
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
|
|
|
<p>
|
|
The goal is to predict the
|
|
output classes from the design matrix \( \hat{X}\in\mathbb{R}^{n\times p} \)
|
|
made of \( n \) samples, each of which carries \( p \) features or predictors. The
|
|
primary goal is to identify the classes to which new unseen samples
|
|
belong.
|
|
|
|
<p>
|
|
Let us specialize to the case of two classes only, with outputs \( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a credit card user who could default or not on her/his credit card debt. That is
|
|
<p> <br>
|
|
$$
|
|
y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}.
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec3">Linear classifier </h2>
|
|
|
|
<p>
|
|
Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if \( y_i > 0.5 \) and the no default case \( y_i \leq 0.5 \).
|
|
|
|
<p>
|
|
We would then have our
|
|
weighted linear combination, namely
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
\hat{y} = \hat{X}^T\hat{\beta} + \hat{\epsilon},
|
|
\tag{1}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
|
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators/predictors.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec4">Some selected properties </h2>
|
|
|
|
<p>
|
|
The main problem with our function is that it
|
|
takes values on the entire real axis. In the case of
|
|
logistic regression, however, the labels \( y_i \) are discrete
|
|
variables.
|
|
|
|
<p>
|
|
One simple way to get a discrete output is to have sign
|
|
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
|
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
|
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
|
literature. This model is extremely simple. However, in many cases it is more
|
|
favorable to use a ``soft" classifier that outputs
|
|
the probability of a given category. This leads us to the logistic function.
|
|
|
|
<p>
|
|
The code for plotting the perceptron can be seen here. This si nothing but the standard <a href="https://en.wikipedia.org/wiki/Heaviside_step_function" target="_blank">Heaviside step function</a>.
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec5">The logistic function </h2>
|
|
|
|
<p>
|
|
The perceptron is an example of a ``hard classification" model. We
|
|
will encounter this model when we discuss neural networks as
|
|
well. Each datapoint is deterministically assigned to a category (i.e
|
|
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
|
classifier that outputs the probability of a given category rather
|
|
than a single value. For example, given \( x_i \), the classifier
|
|
outputs the probability of being in a category \( k \). Logistic regression
|
|
is the most common example of a so-called soft classifier. In logistic
|
|
regression, the probability that a data point \( x_i \)
|
|
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event,
|
|
<p> <br>
|
|
$$
|
|
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
|
$$
|
|
<p> <br>
|
|
|
|
Note that \( 1-p(t)= p(-t) \).
|
|
The following code plots the logistic function.
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span>
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec6">Two parameters </h2>
|
|
|
|
<p>
|
|
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
|
<p> <br>
|
|
$$
|
|
\begin{align*}
|
|
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
|
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
|
\end{align*}
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
|
|
|
<p>
|
|
Note that we used
|
|
<p> <br>
|
|
$$
|
|
p(y_i=0\vert x_i, \hat{\beta}) = 1-p(y_i=1\vert x_i, \hat{\beta}).
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec7">Maximum likelihood </h2>
|
|
|
|
<p>
|
|
In order to define the total likelihood for all possible outcomes from a
|
|
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
|
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called <a href="https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" target="_blank">Maximum Likelihood Estimation</a> (MLE) principle.
|
|
We aim thus at maximizing
|
|
the probability of seeing the observed data. We can then approximate the
|
|
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
|
<p> <br>
|
|
$$
|
|
\begin{align*}
|
|
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
|
\end{align*}
|
|
$$
|
|
<p> <br>
|
|
|
|
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
|
<p> <br>
|
|
$$
|
|
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right]\right).
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec8">The cost function rewritten </h2>
|
|
|
|
<p>
|
|
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
|
<p> <br>
|
|
$$
|
|
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
|
$$
|
|
<p> <br>
|
|
|
|
<p>
|
|
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
|
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
|
<p> <br>
|
|
$$
|
|
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right).
|
|
$$
|
|
<p> <br>
|
|
|
|
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
|
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec9">Minimizing the cross entropy </h2>
|
|
|
|
<p>
|
|
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
|
therefore, any local minimizer is a global minimizer.
|
|
|
|
<p>
|
|
Minimizing this
|
|
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
|
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
|
$$
|
|
<p> <br>
|
|
|
|
and
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec10">A more compact expression </h2>
|
|
|
|
<p>
|
|
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
|
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
|
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
|
derivative of cost function as
|
|
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
|
$$
|
|
<p> <br>
|
|
|
|
<p>
|
|
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
|
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
|
|
|
<p> <br>
|
|
$$
|
|
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec11">Extending to more predictors </h2>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec12">Including more classes </h2>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec13">Optimizing the cost function </h2>
|
|
|
|
<p>
|
|
Newton's method and gradient descent methods
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec14">A <b>scikit-learn</b> example </h2>
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
|
|
iris = datasets.load_iris()
|
|
<span style="color: #658b00">list</span>(iris.keys())
|
|
[<span style="color: #CD5555">'data'</span>, <span style="color: #CD5555">'target_names'</span>, <span style="color: #CD5555">'feature_names'</span>, <span style="color: #CD5555">'target'</span>, <span style="color: #CD5555">'DESCR'</span>]
|
|
X = iris[<span style="color: #CD5555">"data"</span>][:, <span style="color: #B452CD">3</span>:] <span style="color: #228B22"># petal width</span>
|
|
y = (iris[<span style="color: #CD5555">"target"</span>] == <span style="color: #B452CD">2</span>).astype(np.int) <span style="color: #228B22"># 1 if Iris-Virginica, else 0</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
|
log_reg = LogisticRegression()
|
|
log_reg.fit(X, y)
|
|
|
|
X_new = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">3</span>, <span style="color: #B452CD">1000</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
|
y_proba = log_reg.predict_proba(X_new)
|
|
plt.plot(X_new, y_proba[:, <span style="color: #B452CD">1</span>], <span style="color: #CD5555">"g-"</span>, label=<span style="color: #CD5555">"Iris-Virginica"</span>)
|
|
plt.plot(X_new, y_proba[:, <span style="color: #B452CD">0</span>], <span style="color: #CD5555">"b--"</span>, label=<span style="color: #CD5555">"Not Iris-Virginica"</span>)
|
|
plt.show()
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec15">A simple classification problem </h2>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, linear_model
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_data</span>():
|
|
np.random.seed(<span style="color: #B452CD">0</span>)
|
|
X, y = datasets.make_moons(<span style="color: #B452CD">200</span>, noise=<span style="color: #B452CD">0.20</span>)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> X, y
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">visualize</span>(X, y, clf):
|
|
<span style="color: #228B22"># plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)</span>
|
|
<span style="color: #228B22"># plt.show()</span>
|
|
plot_decision_boundary(<span style="color: #8B008B; font-weight: bold">lambda</span> x: clf.predict(x), X, y)
|
|
plt.title(<span style="color: #CD5555">"Logistic Regression"</span>)
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_decision_boundary</span>(pred_func, X, y):
|
|
<span style="color: #228B22"># Set min and max values and give it some padding</span>
|
|
x_min, x_max = X[:, <span style="color: #B452CD">0</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">0</span>].max() + .<span style="color: #B452CD">5</span>
|
|
y_min, y_max = X[:, <span style="color: #B452CD">1</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">1</span>].max() + .<span style="color: #B452CD">5</span>
|
|
h = <span style="color: #B452CD">0.01</span>
|
|
<span style="color: #228B22"># Generate a grid of points with distance h between them</span>
|
|
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
|
<span style="color: #228B22"># Predict the function value for the whole gid</span>
|
|
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
|
Z = Z.reshape(xx.shape)
|
|
<span style="color: #228B22"># Plot the contour and training examples</span>
|
|
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
|
plt.scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], c=y, cmap=plt.cm.Spectral)
|
|
plt.show()
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">classify</span>(X, y):
|
|
clf = linear_model.LogisticRegressionCV()
|
|
clf.fit(X, y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> clf
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">main</span>():
|
|
X, y = generate_data()
|
|
<span style="color: #228B22"># visualize(X, y)</span>
|
|
clf = classify(X, y)
|
|
visualize(X, y, clf)
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #00688B">__name__</span> == <span style="color: #CD5555">"__main__"</span>:
|
|
main()
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
|
|
</div> <!-- class="slides" -->
|
|
</div> <!-- class="reveal" -->
|
|
|
|
<script src="reveal.js/lib/js/head.min.js"></script>
|
|
<script src="reveal.js/js/reveal.js"></script>
|
|
|
|
<script>
|
|
// Full list of configuration options available here:
|
|
// https://github.com/hakimel/reveal.js#configuration
|
|
Reveal.initialize({
|
|
|
|
// Display navigation controls in the bottom right corner
|
|
controls: true,
|
|
|
|
// Display progress bar (below the horiz. slider)
|
|
progress: true,
|
|
|
|
// Display the page number of the current slide
|
|
slideNumber: true,
|
|
|
|
// Push each slide change to the browser history
|
|
history: false,
|
|
|
|
// Enable keyboard shortcuts for navigation
|
|
keyboard: true,
|
|
|
|
// Enable the slide overview mode
|
|
overview: true,
|
|
|
|
// Vertical centering of slides
|
|
//center: true,
|
|
center: false,
|
|
|
|
// Enables touch navigation on devices with touch input
|
|
touch: true,
|
|
|
|
// Loop the presentation
|
|
loop: false,
|
|
|
|
// Change the presentation direction to be RTL
|
|
rtl: false,
|
|
|
|
// Turns fragments on and off globally
|
|
fragments: true,
|
|
|
|
// Flags if the presentation is running in an embedded mode,
|
|
// i.e. contained within a limited portion of the screen
|
|
embedded: false,
|
|
|
|
// Number of milliseconds between automatically proceeding to the
|
|
// next slide, disabled when set to 0, this value can be overwritten
|
|
// by using a data-autoslide attribute on your slides
|
|
autoSlide: 0,
|
|
|
|
// Stop auto-sliding after user input
|
|
autoSlideStoppable: true,
|
|
|
|
// Enable slide navigation via mouse wheel
|
|
mouseWheel: false,
|
|
|
|
// Hides the address bar on mobile devices
|
|
hideAddressBar: true,
|
|
|
|
// Opens links in an iframe preview overlay
|
|
previewLinks: false,
|
|
|
|
// Transition style
|
|
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
|
|
|
|
// Transition speed
|
|
transitionSpeed: 'default', // default/fast/slow
|
|
|
|
// Transition style for full page slide backgrounds
|
|
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
|
|
|
|
// Number of slides away from the current that are visible
|
|
viewDistance: 3,
|
|
|
|
// Parallax background image
|
|
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
|
|
|
|
// Parallax background size
|
|
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
|
|
|
|
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
|
|
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
|
|
|
|
});
|
|
|
|
Reveal.initialize({
|
|
dependencies: [
|
|
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
|
|
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
|
|
|
|
// Interpret Markdown in <section> elements
|
|
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
|
|
// Syntax highlight for <code> elements
|
|
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
|
|
|
|
// Zoom in and out with Alt+click
|
|
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Speaker notes
|
|
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Remote control your reveal.js presentation using a touch device
|
|
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// MathJax
|
|
//{ src: 'reveal.js/plugin/math/math.js', async: true }
|
|
]
|
|
});
|
|
|
|
Reveal.initialize({
|
|
|
|
// The "normal" size of the presentation, aspect ratio will be preserved
|
|
// when the presentation is scaled to fit different resolutions. Can be
|
|
// specified using percentage units.
|
|
width: 1170, // original: 960,
|
|
height: 700,
|
|
|
|
// Factor of the display size that should remain empty around the content
|
|
margin: 0.1,
|
|
|
|
// Bounds for smallest/largest possible scale to apply to content
|
|
minScale: 0.2,
|
|
maxScale: 1.0
|
|
|
|
});
|
|
</script>
|
|
|
|
<!-- begin footer logo
|
|
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
|
|
<img src="somelogo.png">
|
|
</div>
|
|
end footer logo -->
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|