mhjensen
503 lines
17 KiB
HTML
503 lines
17 KiB
HTML
\
|
|
<!DOCTYPE html>
|
|
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
|
|
|
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
|
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
|
|
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<!--
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
|
|
-->
|
|
|
|
<!-- For syntax highlighting -->
|
|
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
|
|
|
|
<!-- Printing and PDF exports -->
|
|
<script>
|
|
var link = document.createElement( 'link' );
|
|
link.rel = 'stylesheet';
|
|
link.type = 'text/css';
|
|
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
|
|
document.getElementsByTagName( 'head' )[0].appendChild( link );
|
|
</script>
|
|
|
|
<style type="text/css">
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
.reveal .alert-text-small { font-size: 80%; }
|
|
.reveal .alert-text-large { font-size: 130%; }
|
|
.reveal .alert-text-normal { font-size: 90%; }
|
|
.reveal .alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:5px solid #bababa;
|
|
-webkit-border-radius: 14px; -moz-border-radius: 14px;
|
|
border-radius:14px;
|
|
background-position: 10px 10px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 30px; /* 55px; if icon */
|
|
}
|
|
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
|
|
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
/*.reveal .alert li {margin-top: 1em}*/
|
|
.reveal .alert-block p+p {margin-top:5px}
|
|
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
|
|
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
|
|
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
|
|
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
|
|
|
|
</style>
|
|
|
|
|
|
|
|
<!-- Styles for table layout of slides -->
|
|
<style type="text/css">
|
|
td.padding {
|
|
padding-top:20px;
|
|
padding-bottom:20px;
|
|
padding-right:50px;
|
|
padding-left:50px;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
<body>
|
|
<div class="reveal">
|
|
|
|
<!-- Any section element inside the <div class="slides"> container
|
|
is displayed as a slide -->
|
|
|
|
<div class="slides">
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Logistic Regression</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p> <br>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p> <br>
|
|
<center><h4>Sep 17, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec0">Logistic Regression </h2>
|
|
|
|
<p>
|
|
So far we have focused on learning from datasets for which there is a
|
|
<b>continuous</b> output. In linear regression we have been
|
|
concerned with learning the coefficients of a polynomial to predict
|
|
the response of a continuous variable \( y_i \) on unseen data based on
|
|
its independent variables \( {\bf x}_i \).
|
|
|
|
<p>
|
|
Classification problems,
|
|
however, are concerned with outcomes taking the form of discrete
|
|
variables (i.e. categories). For example, we may want to detect if
|
|
there's a cat or a dog in an image. Or given a specific system,
|
|
we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).
|
|
(e.g. ordered/disordered).
|
|
|
|
<p>
|
|
<b>Logistic regression deals with binary, dichotomous outcomes (e.g. True or
|
|
False, Success or Failure, etc.). It is worth noting that logistic
|
|
regression is also commonly used in modern supervised Deep Learning
|
|
models</b>, as we will see later.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec1">Basics </h2>
|
|
|
|
<p>
|
|
We consider the case where the dependent variables \( y_i\in\mathbb{Z} \)
|
|
are discrete and only take values from \( m=0,\dots,M-1 \) (i.e. \( M \)
|
|
classes).
|
|
|
|
<p>
|
|
The goal is to predict the
|
|
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
|
made of \( n \) samples, each of which bears \( p \) features. The
|
|
primary goal is to identify the classes to which new unseen samples
|
|
belong.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec2">Linear classifier </h2>
|
|
|
|
<p>
|
|
Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
s_i = \boldsymbol{x}_i^T\boldsymbol{w} + b_0 \equiv \mathbf{x}_i^T\mathbf{w},
|
|
\tag{1}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
where we use the short-hand notation
|
|
\( \mathbf{x}_i = (1,\boldsymbol{x}_i) \) and \( \mathbf{w}_i = (b_0,\boldsymbol{w}_i) \).
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec3">Some selected properties </h2>
|
|
|
|
<p>
|
|
This function takes values on the entire real axis. In the case of
|
|
logistic regression, however, the labels \( y_i \) are discrete
|
|
variables. One simple way to get a discrete output is to have sign
|
|
functions that map the output of a linear regressor to \( \{0,1\} \),
|
|
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise. Indeed,
|
|
this is commonly known as the "perceptron" in the machine learning
|
|
literature. This model is extremely simple, and it is favorable in
|
|
many cases (e.g. noisy data) to have a ``soft" classifier that outputs
|
|
the probability of a given category. For example, given
|
|
\( \mathbf{x}_i \), the classifier outputs the probability of being in
|
|
category \( m \). One such function is the logistic (or sigmoid) function:
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
f(s) = \frac{1}{1+\mathrm e^{-s}}.
|
|
\tag{2}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec4">The cross-entropy as a cost function for logistic regression </h2>
|
|
|
|
<p>
|
|
The perceptron is an example of a ``hard classification": each datapoint is deterministically assigned to a category (i.e \( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft" classifier that outputs the probability of a given category rather than a single value. For example, given \( \mathbf{x}_i \), the classifier outputs the probability of being in category \( m \).
|
|
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point \( \boldsymbol{x}_i \) belongs to a category \( y_i=\{0,1\} \) is is given by
|
|
<p> <br>
|
|
$$
|
|
\begin{eqnarray}
|
|
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
|
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
|
\end{eqnarray}
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( \boldsymbol{\theta}=\mathbf{w} \) are the weights we wish to learn from the data.
|
|
|
|
<p>
|
|
Notice that in terms of the logistic function, we can write
|
|
<p> <br>
|
|
$$
|
|
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec5">Maximum likelihood </h2>
|
|
|
|
<p>
|
|
We now define the cost function for logistic regression using Maximum
|
|
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
|
to maximize the probability of seeing the observed data. Consider a
|
|
dataset \( \mathcal{D}=\{(y_i,\boldsymbol{x}_i)\} \) with binary labels
|
|
\( y_i\in\{0,1\} \) where the data points are drawn independently. The
|
|
likelihood of the seeing the data under our model is just:
|
|
<p> <br>
|
|
$$
|
|
\begin{align}
|
|
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
|
\tag{3}
|
|
\end{align}
|
|
$$
|
|
<p> <br>
|
|
|
|
from which we can readily compute the log-likelihood:
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
|
\tag{4}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \)
|
|
<p> <br>
|
|
$$
|
|
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
|
$$
|
|
<p> <br>
|
|
|
|
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
|
<p> <br>
|
|
$$
|
|
\begin{eqnarray}
|
|
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
|
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
|
\end{eqnarray}
|
|
$$
|
|
<p> <br>
|
|
|
|
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
|
in practice we usually supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec6">Minimizing the cross entropy </h2>
|
|
|
|
<p>
|
|
The cross entropy is a convex function of the weights \( \mathbf{w} \) and,
|
|
therefore, any local minimizer is a global minimizer. Minimizing this
|
|
cost function leads to the following equation
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
|
\tag{5}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
<p>
|
|
where we made use of the logistic function identity \( \partial_z f(z) =
|
|
f(z)[1-f(z)] \). This equation defines a transcendental equation for
|
|
\( \mathbf{w} \), the solution of which, unlike linear regression, cannot
|
|
be written in a closed form.
|
|
Here we need gradient descent methods!
|
|
</section>
|
|
|
|
|
|
|
|
</div> <!-- class="slides" -->
|
|
</div> <!-- class="reveal" -->
|
|
|
|
<script src="reveal.js/lib/js/head.min.js"></script>
|
|
<script src="reveal.js/js/reveal.js"></script>
|
|
|
|
<script>
|
|
// Full list of configuration options available here:
|
|
// https://github.com/hakimel/reveal.js#configuration
|
|
Reveal.initialize({
|
|
|
|
// Display navigation controls in the bottom right corner
|
|
controls: true,
|
|
|
|
// Display progress bar (below the horiz. slider)
|
|
progress: true,
|
|
|
|
// Display the page number of the current slide
|
|
slideNumber: true,
|
|
|
|
// Push each slide change to the browser history
|
|
history: false,
|
|
|
|
// Enable keyboard shortcuts for navigation
|
|
keyboard: true,
|
|
|
|
// Enable the slide overview mode
|
|
overview: true,
|
|
|
|
// Vertical centering of slides
|
|
//center: true,
|
|
center: false,
|
|
|
|
// Enables touch navigation on devices with touch input
|
|
touch: true,
|
|
|
|
// Loop the presentation
|
|
loop: false,
|
|
|
|
// Change the presentation direction to be RTL
|
|
rtl: false,
|
|
|
|
// Turns fragments on and off globally
|
|
fragments: true,
|
|
|
|
// Flags if the presentation is running in an embedded mode,
|
|
// i.e. contained within a limited portion of the screen
|
|
embedded: false,
|
|
|
|
// Number of milliseconds between automatically proceeding to the
|
|
// next slide, disabled when set to 0, this value can be overwritten
|
|
// by using a data-autoslide attribute on your slides
|
|
autoSlide: 0,
|
|
|
|
// Stop auto-sliding after user input
|
|
autoSlideStoppable: true,
|
|
|
|
// Enable slide navigation via mouse wheel
|
|
mouseWheel: false,
|
|
|
|
// Hides the address bar on mobile devices
|
|
hideAddressBar: true,
|
|
|
|
// Opens links in an iframe preview overlay
|
|
previewLinks: false,
|
|
|
|
// Transition style
|
|
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
|
|
|
|
// Transition speed
|
|
transitionSpeed: 'default', // default/fast/slow
|
|
|
|
// Transition style for full page slide backgrounds
|
|
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
|
|
|
|
// Number of slides away from the current that are visible
|
|
viewDistance: 3,
|
|
|
|
// Parallax background image
|
|
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
|
|
|
|
// Parallax background size
|
|
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
|
|
|
|
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
|
|
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
|
|
|
|
});
|
|
|
|
Reveal.initialize({
|
|
dependencies: [
|
|
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
|
|
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
|
|
|
|
// Interpret Markdown in <section> elements
|
|
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
|
|
// Syntax highlight for <code> elements
|
|
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
|
|
|
|
// Zoom in and out with Alt+click
|
|
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Speaker notes
|
|
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Remote control your reveal.js presentation using a touch device
|
|
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// MathJax
|
|
//{ src: 'reveal.js/plugin/math/math.js', async: true }
|
|
]
|
|
});
|
|
|
|
Reveal.initialize({
|
|
|
|
// The "normal" size of the presentation, aspect ratio will be preserved
|
|
// when the presentation is scaled to fit different resolutions. Can be
|
|
// specified using percentage units.
|
|
width: 1170, // original: 960,
|
|
height: 700,
|
|
|
|
// Factor of the display size that should remain empty around the content
|
|
margin: 0.1,
|
|
|
|
// Bounds for smallest/largest possible scale to apply to content
|
|
minScale: 0.2,
|
|
maxScale: 1.0
|
|
|
|
});
|
|
</script>
|
|
|
|
<!-- begin footer logo
|
|
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
|
|
<img src="somelogo.png">
|
|
</div>
|
|
end footer logo -->
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|