updated log reg
This commit is contained in:
mhjensen
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -130,7 +139,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 20, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Sep 25, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<p>
|
||||
@@ -146,6 +155,8 @@ MathJax.Hub.Config({
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -117,25 +126,34 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0" class="anchor">Logistic Regression </h2>
|
||||
|
||||
<p>
|
||||
So far we have focused on learning from datasets for which there is a
|
||||
<b>continuous</b> output. In linear regression we have been
|
||||
concerned with learning the coefficients of a polynomial to predict
|
||||
the response of a continuous variable \( y_i \) on unseen data based on
|
||||
its independent variables \( {\bf x}_i \).
|
||||
In linear regression our main interest was centered on learning the
|
||||
coefficients of a functional fit (say a polynomial) in order to be
|
||||
able to predict the response of a continuous variable on some unseen
|
||||
data. The fit to the continuous variable \( y_i \) is based on some
|
||||
independent variables \( \hat{x}_i \). Linear regression resulted in
|
||||
analytical expressions (in terms of matrices to invert) for several
|
||||
quantities, ranging from the variance and thereby the confidence
|
||||
intervals of the parameters \( \hat{\beta} \) to the mean squared
|
||||
error. If we can invert the product of the design matrices, linear
|
||||
regression gives then a simple recipe for fitting our data.
|
||||
|
||||
<p>
|
||||
Classification problems,
|
||||
however, are concerned with outcomes taking the form of discrete
|
||||
variables (i.e. categories). For example, we may want to detect if
|
||||
there's a cat or a dog in an image. Or given a specific system,
|
||||
we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).
|
||||
(e.g. ordered/disordered).
|
||||
Classification problems, however, are concerned with outcomes taking
|
||||
the form of discrete variables (i.e. categories). We may for example,
|
||||
on the basis of DNA sequencing for a number of patients, like to find
|
||||
out which mutations are important for a certain disease; or based on
|
||||
scans of various patients' brains, figure out if there is a tumor or
|
||||
not; or given a specific physical system, we'd like to identify its
|
||||
state, say whether it is an ordered or disordered system (typical
|
||||
situation in solid state physics); or classify the status of a
|
||||
patient, whether she/he has a stroke or not and many other similar
|
||||
situations.
|
||||
|
||||
<p>
|
||||
<b>Logistic regression deals with binary, dichotomous outcomes (e.g. True or
|
||||
False, Success or Failure, etc.). It is worth noting that logistic
|
||||
regression is also commonly used in modern supervised Deep Learning
|
||||
models</b>, as we will see later.
|
||||
The most common situation we encounter when we apply logistic
|
||||
regression is that of two possible outcomes, normally denoted as a
|
||||
binary outcome, true or false, positive or negative, success or
|
||||
failure etc.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -152,6 +170,9 @@ models</b>, as we will see later.
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -109,21 +118,20 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0002"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec1" class="anchor">Basics </h2>
|
||||
<h2 id="___sec1" class="anchor">Optimization and Deep learning </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables \( y_i\in\mathbb{Z} \)
|
||||
are discrete and only take values from \( m=0,\dots,M-1 \) (i.e. \( M \)
|
||||
classes).
|
||||
Logistic regression will also serve as our stepping towards neural
|
||||
network algorithms and supervised deep learning. For logistic
|
||||
learning, the minimization of the cost function leads to a non-linear
|
||||
equation in the parameters \( \hat{\beta} \). The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms.
|
||||
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which bears \( p \) features. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
We note also that many of the topics discussed here
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models, as we will see later.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -140,6 +148,10 @@ belong.
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -109,21 +118,27 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0003"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec2" class="anchor">Linear classifier </h2>
|
||||
<h2 id="___sec2" class="anchor">Basics </h2>
|
||||
|
||||
<p>
|
||||
Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset
|
||||
$$
|
||||
\begin{equation}
|
||||
s_i = \boldsymbol{x}_i^T\boldsymbol{w} + b_0 \equiv \mathbf{x}_i^T\mathbf{w},
|
||||
\tag{1}
|
||||
\end{equation}
|
||||
$$
|
||||
We consider the case where the dependent variables, also called the
|
||||
responses or the outcomes) \( y_i \) are discrete and only take values
|
||||
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
||||
|
||||
where we use the short-hand notation
|
||||
\( \mathbf{x}_i = (1,\boldsymbol{x}_i) \) and \( \mathbf{w}_i = (b_0,\boldsymbol{w}_i) \).
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which carries \( p \) features. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
|
||||
<p>
|
||||
Let us specialize to the case of two classes only, with outputs \( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a credit card use who could default or not on her/his debt. That is
|
||||
$$
|
||||
y_i = \begin{bmatrix} 0 & \mathrm{no default}\\ 1 & \mathrm{default} \end{bmatrix}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -140,6 +155,11 @@ where we use the short-hand notation
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -111,29 +120,23 @@ MathJax.Hub.Config({
|
||||
<a name="part0004"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec3" class="anchor">Some selected properties </h2>
|
||||
<h2 id="___sec3" class="anchor">Linear classifier </h2>
|
||||
|
||||
<p>
|
||||
This function takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels \( y_i \) are discrete
|
||||
variables. One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise. Indeed,
|
||||
this is commonly known as the "perceptron" in the machine learning
|
||||
literature. This model is extremely simple, and it is favorable in
|
||||
many cases (e.g. noisy data) to have a ``soft" classifier that outputs
|
||||
the probability of a given category. For example, given
|
||||
\( \mathbf{x}_i \), the classifier outputs the probability of being in
|
||||
category \( m \). One such function is the logistic (or sigmoid) function:
|
||||
Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if \( y_i > 0.5 \) and the no default case \( y_i \leq 0.5 \).
|
||||
|
||||
<p>
|
||||
We would then have our
|
||||
weighted linear combination, namely
|
||||
$$
|
||||
\begin{equation}
|
||||
f(s) = \frac{1}{1+\mathrm e^{-s}}.
|
||||
\tag{2}
|
||||
\end{equation}
|
||||
\hat{y} = \hat{X}_i^T\hat{\beta} + \hat{\epsilon},
|
||||
\tag{1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
||||
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
||||
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -150,6 +153,12 @@ Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -111,25 +120,22 @@ MathJax.Hub.Config({
|
||||
<a name="part0005"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec4" class="anchor">The cross-entropy as a cost function for logistic regression </h2>
|
||||
<h2 id="___sec4" class="anchor">Some selected properties </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification": each datapoint is deterministically assigned to a category (i.e \( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft" classifier that outputs the probability of a given category rather than a single value. For example, given \( \mathbf{x}_i \), the classifier outputs the probability of being in category \( m \).
|
||||
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point \( \boldsymbol{x}_i \) belongs to a category \( y_i=\{0,1\} \) is is given by
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
||||
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
||||
\end{eqnarray}
|
||||
$$
|
||||
|
||||
where \( \boldsymbol{\theta}=\mathbf{w} \) are the weights we wish to learn from the data.
|
||||
The main problem with our function is that it
|
||||
takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels \( y_i \) are discrete
|
||||
variables.
|
||||
|
||||
<p>
|
||||
Notice that in terms of the logistic function, we can write
|
||||
$$
|
||||
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
||||
$$
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
||||
literature. This model is extremely simple. However, in many cases it is more
|
||||
favorable to use a ``soft" classifier that outputs
|
||||
the probability of a given category. This leads us to the logistic function.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -146,6 +152,13 @@ $$
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -109,31 +118,17 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0006"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec5" class="anchor">Maximum likelihood </h2>
|
||||
<h2 id="___sec5" class="anchor">The logistic function </h2>
|
||||
|
||||
<p>
|
||||
We now define the cost function for logistic regression using Maximum
|
||||
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
||||
to maximize the probability of seeing the observed data. Consider a
|
||||
dataset \( \mathcal{D}=\{(y_i,\boldsymbol{x}_i)\} \) with binary labels
|
||||
\( y_i\in\{0,1\} \) where the data points are drawn independently. The
|
||||
likelihood of the seeing the data under our model is just:
|
||||
A widely used model is the so-called logistic (or sigmoid) function which ranges from \( 0 \) to \( 1 \),
|
||||
$$
|
||||
\begin{align}
|
||||
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
||||
\tag{3}
|
||||
\end{align}
|
||||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
$$
|
||||
|
||||
from which we can readily compute the log-likelihood:
|
||||
$$
|
||||
\begin{equation}
|
||||
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\tag{4}
|
||||
\end{equation}
|
||||
$$
|
||||
Note that \( 1-p(t)= p(-t) \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -150,6 +145,12 @@ $$
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -110,21 +119,20 @@ MathJax.Hub.Config({
|
||||
|
||||
<a name="part0007"></a>
|
||||
<!-- !split -->
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \)
|
||||
$$
|
||||
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
$$
|
||||
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
||||
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
||||
\end{eqnarray}
|
||||
$$
|
||||
<h2 id="___sec6" class="anchor">Probabilistic classifiers </h2>
|
||||
|
||||
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
||||
in practice we usually supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification" model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given \( x_i \), the classifier
|
||||
outputs the probability of being in a category \( k \). Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point \( x_i \)
|
||||
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid)
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -141,6 +149,12 @@ in practice we usually supplement the cross-entropy with additional regularizati
|
||||
<li class="active"><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -111,26 +120,24 @@ MathJax.Hub.Config({
|
||||
<a name="part0008"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec6" class="anchor">Minimizing the cross entropy </h2>
|
||||
<h2 id="___sec7" class="anchor">Two parameters </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \mathbf{w} \) and,
|
||||
therefore, any local minimizer is a global minimizer. Minimizing this
|
||||
cost function leads to the following equation
|
||||
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
||||
$$
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
||||
\tag{5}
|
||||
\end{equation}
|
||||
$$
|
||||
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
||||
|
||||
<p>
|
||||
where we made use of the logistic function identity \( \partial_z f(z) =
|
||||
f(z)[1-f(z)] \). This equation defines a transcendental equation for
|
||||
\( \mathbf{w} \), the solution of which, unlike linear regression, cannot
|
||||
be written in a closed form.
|
||||
Here we need gradient descent methods!
|
||||
Note that we used
|
||||
$$
|
||||
p(y_i=1) = 1-p(y_i=0).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -147,6 +154,12 @@ Here we need gradient descent methods!
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li class="active"><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -109,34 +118,29 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0009"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec7" class="anchor">A <b>scikit-learn</b> example </h2>
|
||||
<h2 id="___sec8" class="anchor">Maximum likelihood </h2>
|
||||
|
||||
<p>
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
||||
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
||||
$$
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
iris <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_iris()
|
||||
<span style="color: #008000">list</span>(iris<span style="color: #666666">.</span>keys())
|
||||
[<span style="color: #BA2121">'data'</span>, <span style="color: #BA2121">'target_names'</span>, <span style="color: #BA2121">'feature_names'</span>, <span style="color: #BA2121">'target'</span>, <span style="color: #BA2121">'DESCR'</span>]
|
||||
X <span style="color: #666666">=</span> iris[<span style="color: #BA2121">"data"</span>][:, <span style="color: #666666">3</span>:] <span style="color: #408080; font-style: italic"># petal width</span>
|
||||
y <span style="color: #666666">=</span> (iris[<span style="color: #BA2121">"target"</span>] <span style="color: #666666">==</span> <span style="color: #666666">2</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int) <span style="color: #408080; font-style: italic"># 1 if Iris-Virginica, else 0</span>
|
||||
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right].
|
||||
$$
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
log_reg <span style="color: #666666">=</span> LogisticRegression()
|
||||
log_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1000</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_proba <span style="color: #666666">=</span> log_reg<span style="color: #666666">.</span>predict_proba(X_new)
|
||||
plt<span style="color: #666666">.</span>plot(X_new, y_proba[:, <span style="color: #666666">1</span>], <span style="color: #BA2121">"g-"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_new, y_proba[:, <span style="color: #666666">0</span>], <span style="color: #BA2121">"b--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Not Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -151,6 +155,13 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li class="active"><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0010"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec9" class="anchor">The cost function rewritten </h2>
|
||||
|
||||
<p>
|
||||
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs009.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs002.html">3</a></li>
|
||||
<li><a href="._LogReg-bs003.html">4</a></li>
|
||||
<li><a href="._LogReg-bs004.html">5</a></li>
|
||||
<li><a href="._LogReg-bs005.html">6</a></li>
|
||||
<li><a href="._LogReg-bs006.html">7</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li class="active"><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0011"></a>
|
||||
<!-- !split -->
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs010.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs003.html">4</a></li>
|
||||
<li><a href="._LogReg-bs004.html">5</a></li>
|
||||
<li><a href="._LogReg-bs005.html">6</a></li>
|
||||
<li><a href="._LogReg-bs006.html">7</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li class="active"><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0012"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec10" class="anchor">Minimizing the cross entropy </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
<p>
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs011.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs004.html">5</a></li>
|
||||
<li><a href="._LogReg-bs005.html">6</a></li>
|
||||
<li><a href="._LogReg-bs006.html">7</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li class="active"><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0013"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec11" class="anchor">A more compact expression </h2>
|
||||
|
||||
<p>
|
||||
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
||||
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
||||
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
||||
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
||||
|
||||
$$
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs012.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs005.html">6</a></li>
|
||||
<li><a href="._LogReg-bs006.html">7</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li class="active"><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0014"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec12" class="anchor">A <b>scikit-learn</b> example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
iris <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_iris()
|
||||
<span style="color: #008000">list</span>(iris<span style="color: #666666">.</span>keys())
|
||||
[<span style="color: #BA2121">'data'</span>, <span style="color: #BA2121">'target_names'</span>, <span style="color: #BA2121">'feature_names'</span>, <span style="color: #BA2121">'target'</span>, <span style="color: #BA2121">'DESCR'</span>]
|
||||
X <span style="color: #666666">=</span> iris[<span style="color: #BA2121">"data"</span>][:, <span style="color: #666666">3</span>:] <span style="color: #408080; font-style: italic"># petal width</span>
|
||||
y <span style="color: #666666">=</span> (iris[<span style="color: #BA2121">"target"</span>] <span style="color: #666666">==</span> <span style="color: #666666">2</span>)<span style="color: #666666">.</span>astype(np<span style="color: #666666">.</span>int) <span style="color: #408080; font-style: italic"># 1 if Iris-Virginica, else 0</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
||||
log_reg <span style="color: #666666">=</span> LogisticRegression()
|
||||
log_reg<span style="color: #666666">.</span>fit(X, y)
|
||||
|
||||
X_new <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1000</span>)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
||||
y_proba <span style="color: #666666">=</span> log_reg<span style="color: #666666">.</span>predict_proba(X_new)
|
||||
plt<span style="color: #666666">.</span>plot(X_new, y_proba[:, <span style="color: #666666">1</span>], <span style="color: #BA2121">"g-"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(X_new, y_proba[:, <span style="color: #666666">0</span>], <span style="color: #BA2121">"b--"</span>, label<span style="color: #666666">=</span><span style="color: #BA2121">"Not Iris-Virginica"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs013.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs006.html">7</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li class="active"><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Logistic Regression">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Logistic Regression</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="LogReg-bs.html">Data Analysis and Machine Learning: Logistic Regression</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0015"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec13" class="anchor">A simple classification problem </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets, linear_model
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_data</span>():
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
X, y <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>make_moons(<span style="color: #666666">200</span>, noise<span style="color: #666666">=0.20</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> X, y
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">visualize</span>(X, y, clf):
|
||||
<span style="color: #408080; font-style: italic"># plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)</span>
|
||||
<span style="color: #408080; font-style: italic"># plt.show()</span>
|
||||
plot_decision_boundary(<span style="color: #008000; font-weight: bold">lambda</span> x: clf<span style="color: #666666">.</span>predict(x), X, y)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Logistic Regression"</span>)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(pred_func, X, y):
|
||||
<span style="color: #408080; font-style: italic"># Set min and max values and give it some padding</span>
|
||||
x_min, x_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
y_min, y_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
h <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
|
||||
<span style="color: #408080; font-style: italic"># Generate a grid of points with distance h between them</span>
|
||||
xx, yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(np<span style="color: #666666">.</span>arange(x_min, x_max, h), np<span style="color: #666666">.</span>arange(y_min, y_max, h))
|
||||
<span style="color: #408080; font-style: italic"># Predict the function value for the whole gid</span>
|
||||
Z <span style="color: #666666">=</span> pred_func(np<span style="color: #666666">.</span>c_[xx<span style="color: #666666">.</span>ravel(), yy<span style="color: #666666">.</span>ravel()])
|
||||
Z <span style="color: #666666">=</span> Z<span style="color: #666666">.</span>reshape(xx<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Plot the contour and training examples</span>
|
||||
plt<span style="color: #666666">.</span>contourf(xx, yy, Z, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], c<span style="color: #666666">=</span>y, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">classify</span>(X, y):
|
||||
clf <span style="color: #666666">=</span> linear_model<span style="color: #666666">.</span>LogisticRegressionCV()
|
||||
clf<span style="color: #666666">.</span>fit(X, y)
|
||||
<span style="color: #008000; font-weight: bold">return</span> clf
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">main</span>():
|
||||
X, y <span style="color: #666666">=</span> generate_data()
|
||||
<span style="color: #408080; font-style: italic"># visualize(X, y)</span>
|
||||
clf <span style="color: #666666">=</span> classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"__main__"</span>:
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._LogReg-bs014.html">«</a></li>
|
||||
<li><a href="._LogReg-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="._LogReg-bs010.html">11</a></li>
|
||||
<li><a href="._LogReg-bs011.html">12</a></li>
|
||||
<li><a href="._LogReg-bs012.html">13</a></li>
|
||||
<li><a href="._LogReg-bs013.html">14</a></li>
|
||||
<li><a href="._LogReg-bs014.html">15</a></li>
|
||||
<li class="active"><a href="._LogReg-bs015.html">16</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -41,16 +41,19 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -89,13 +92,19 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs001.html#___sec0" style="font-size: 80%;">Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">The cross-entropy as a cost function for logistic regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec6" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec7" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs002.html#___sec1" style="font-size: 80%;">Optimization and Deep learning</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs003.html#___sec2" style="font-size: 80%;">Basics</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs004.html#___sec3" style="font-size: 80%;">Linear classifier</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs005.html#___sec4" style="font-size: 80%;">Some selected properties</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs006.html#___sec5" style="font-size: 80%;">The logistic function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs007.html#___sec6" style="font-size: 80%;">Probabilistic classifiers</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs008.html#___sec7" style="font-size: 80%;">Two parameters</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs009.html#___sec8" style="font-size: 80%;">Maximum likelihood</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs010.html#___sec9" style="font-size: 80%;">The cost function rewritten</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs012.html#___sec10" style="font-size: 80%;">Minimizing the cross entropy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs013.html#___sec11" style="font-size: 80%;">A more compact expression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs014.html#___sec12" style="font-size: 80%;">A <b>scikit-learn</b> example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._LogReg-bs015.html#___sec13" style="font-size: 80%;">A simple classification problem</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -130,7 +139,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 20, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Sep 25, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<p>
|
||||
@@ -146,6 +155,8 @@ MathJax.Hub.Config({
|
||||
<li><a href="._LogReg-bs007.html">8</a></li>
|
||||
<li><a href="._LogReg-bs008.html">9</a></li>
|
||||
<li><a href="._LogReg-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._LogReg-bs015.html">16</a></li>
|
||||
<li><a href="._LogReg-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Sep 20, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Sep 25, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -162,202 +162,288 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Logistic Regression </h2>
|
||||
|
||||
<p>
|
||||
So far we have focused on learning from datasets for which there is a
|
||||
<b>continuous</b> output. In linear regression we have been
|
||||
concerned with learning the coefficients of a polynomial to predict
|
||||
the response of a continuous variable \( y_i \) on unseen data based on
|
||||
its independent variables \( {\bf x}_i \).
|
||||
In linear regression our main interest was centered on learning the
|
||||
coefficients of a functional fit (say a polynomial) in order to be
|
||||
able to predict the response of a continuous variable on some unseen
|
||||
data. The fit to the continuous variable \( y_i \) is based on some
|
||||
independent variables \( \hat{x}_i \). Linear regression resulted in
|
||||
analytical expressions (in terms of matrices to invert) for several
|
||||
quantities, ranging from the variance and thereby the confidence
|
||||
intervals of the parameters \( \hat{\beta} \) to the mean squared
|
||||
error. If we can invert the product of the design matrices, linear
|
||||
regression gives then a simple recipe for fitting our data.
|
||||
|
||||
<p>
|
||||
Classification problems,
|
||||
however, are concerned with outcomes taking the form of discrete
|
||||
variables (i.e. categories). For example, we may want to detect if
|
||||
there's a cat or a dog in an image. Or given a specific system,
|
||||
we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).
|
||||
(e.g. ordered/disordered).
|
||||
Classification problems, however, are concerned with outcomes taking
|
||||
the form of discrete variables (i.e. categories). We may for example,
|
||||
on the basis of DNA sequencing for a number of patients, like to find
|
||||
out which mutations are important for a certain disease; or based on
|
||||
scans of various patients' brains, figure out if there is a tumor or
|
||||
not; or given a specific physical system, we'd like to identify its
|
||||
state, say whether it is an ordered or disordered system (typical
|
||||
situation in solid state physics); or classify the status of a
|
||||
patient, whether she/he has a stroke or not and many other similar
|
||||
situations.
|
||||
|
||||
<p>
|
||||
<b>Logistic regression deals with binary, dichotomous outcomes (e.g. True or
|
||||
False, Success or Failure, etc.). It is worth noting that logistic
|
||||
regression is also commonly used in modern supervised Deep Learning
|
||||
models</b>, as we will see later.
|
||||
The most common situation we encounter when we apply logistic
|
||||
regression is that of two possible outcomes, normally denoted as a
|
||||
binary outcome, true or false, positive or negative, success or
|
||||
failure etc.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec1">Basics </h2>
|
||||
<h2 id="___sec1">Optimization and Deep learning </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables \( y_i\in\mathbb{Z} \)
|
||||
are discrete and only take values from \( m=0,\dots,M-1 \) (i.e. \( M \)
|
||||
classes).
|
||||
Logistic regression will also serve as our stepping towards neural
|
||||
network algorithms and supervised deep learning. For logistic
|
||||
learning, the minimization of the cost function leads to a non-linear
|
||||
equation in the parameters \( \hat{\beta} \). The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms.
|
||||
|
||||
<p>
|
||||
We note also that many of the topics discussed here
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models, as we will see later.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec2">Basics </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables, also called the
|
||||
responses or the outcomes) \( y_i \) are discrete and only take values
|
||||
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
||||
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which bears \( p \) features. The
|
||||
made of \( n \) samples, each of which carries \( p \) features. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
|
||||
<p>
|
||||
Let us specialize to the case of two classes only, with outputs \( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a credit card use who could default or not on her/his debt. That is
|
||||
<p> <br>
|
||||
$$
|
||||
y_i = \begin{bmatrix} 0 & \mathrm{no default}\\ 1 & \mathrm{default} \end{bmatrix}.
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec2">Linear classifier </h2>
|
||||
<h2 id="___sec3">Linear classifier </h2>
|
||||
|
||||
<p>
|
||||
Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset
|
||||
Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if \( y_i > 0.5 \) and the no default case \( y_i \leq 0.5 \).
|
||||
|
||||
<p>
|
||||
We would then have our
|
||||
weighted linear combination, namely
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
s_i = \boldsymbol{x}_i^T\boldsymbol{w} + b_0 \equiv \mathbf{x}_i^T\mathbf{w},
|
||||
\hat{y} = \hat{X}_i^T\hat{\beta} + \hat{\epsilon},
|
||||
\tag{1}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
where we use the short-hand notation
|
||||
\( \mathbf{x}_i = (1,\boldsymbol{x}_i) \) and \( \mathbf{w}_i = (b_0,\boldsymbol{w}_i) \).
|
||||
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
||||
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec3">Some selected properties </h2>
|
||||
<h2 id="___sec4">Some selected properties </h2>
|
||||
|
||||
<p>
|
||||
This function takes values on the entire real axis. In the case of
|
||||
The main problem with our function is that it
|
||||
takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels \( y_i \) are discrete
|
||||
variables. One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise. Indeed,
|
||||
this is commonly known as the "perceptron" in the machine learning
|
||||
literature. This model is extremely simple, and it is favorable in
|
||||
many cases (e.g. noisy data) to have a ``soft" classifier that outputs
|
||||
the probability of a given category. For example, given
|
||||
\( \mathbf{x}_i \), the classifier outputs the probability of being in
|
||||
category \( m \). One such function is the logistic (or sigmoid) function:
|
||||
variables.
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
f(s) = \frac{1}{1+\mathrm e^{-s}}.
|
||||
\tag{2}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
||||
<p>
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
||||
literature. This model is extremely simple. However, in many cases it is more
|
||||
favorable to use a ``soft" classifier that outputs
|
||||
the probability of a given category. This leads us to the logistic function.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec4">The cross-entropy as a cost function for logistic regression </h2>
|
||||
<h2 id="___sec5">The logistic function </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification": each datapoint is deterministically assigned to a category (i.e \( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft" classifier that outputs the probability of a given category rather than a single value. For example, given \( \mathbf{x}_i \), the classifier outputs the probability of being in category \( m \).
|
||||
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point \( \boldsymbol{x}_i \) belongs to a category \( y_i=\{0,1\} \) is is given by
|
||||
A widely used model is the so-called logistic (or sigmoid) function which ranges from \( 0 \) to \( 1 \),
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
||||
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
||||
\end{eqnarray}
|
||||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
where \( \boldsymbol{\theta}=\mathbf{w} \) are the weights we wish to learn from the data.
|
||||
Note that \( 1-p(t)= p(-t) \).
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec6">Probabilistic classifiers </h2>
|
||||
|
||||
<p>
|
||||
Notice that in terms of the logistic function, we can write
|
||||
The perceptron is an example of a ``hard classification" model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given \( x_i \), the classifier
|
||||
outputs the probability of being in a category \( k \). Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point \( x_i \)
|
||||
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid)
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec7">Two parameters </h2>
|
||||
|
||||
<p>
|
||||
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
||||
<p> <br>
|
||||
$$
|
||||
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
||||
|
||||
<p>
|
||||
Note that we used
|
||||
<p> <br>
|
||||
$$
|
||||
p(y_i=1) = 1-p(y_i=0).
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec5">Maximum likelihood </h2>
|
||||
<h2 id="___sec8">Maximum likelihood </h2>
|
||||
|
||||
<p>
|
||||
We now define the cost function for logistic regression using Maximum
|
||||
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
||||
to maximize the probability of seeing the observed data. Consider a
|
||||
dataset \( \mathcal{D}=\{(y_i,\boldsymbol{x}_i)\} \) with binary labels
|
||||
\( y_i\in\{0,1\} \) where the data points are drawn independently. The
|
||||
likelihood of the seeing the data under our model is just:
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
||||
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{align}
|
||||
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
||||
\tag{3}
|
||||
\end{align}
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
from which we can readily compute the log-likelihood:
|
||||
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\tag{4}
|
||||
\end{equation}
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right].
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \)
|
||||
<p> <br>
|
||||
$$
|
||||
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
$$
|
||||
<p> <br>
|
||||
<h2 id="___sec9">The cost function rewritten </h2>
|
||||
|
||||
<p>
|
||||
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
||||
<p> <br>
|
||||
$$
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
||||
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
||||
\end{eqnarray}
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
||||
in practice we usually supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec6">Minimizing the cross entropy </h2>
|
||||
<h2 id="___sec10">Minimizing the cross entropy </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \mathbf{w} \) and,
|
||||
therefore, any local minimizer is a global minimizer. Minimizing this
|
||||
cost function leads to the following equation
|
||||
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
<p>
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
||||
\tag{5}
|
||||
\end{equation}
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
where we made use of the logistic function identity \( \partial_z f(z) =
|
||||
f(z)[1-f(z)] \). This equation defines a transcendental equation for
|
||||
\( \mathbf{w} \), the solution of which, unlike linear regression, cannot
|
||||
be written in a closed form.
|
||||
Here we need gradient descent methods!
|
||||
and
|
||||
<p> <br>
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec7">A <b>scikit-learn</b> example </h2>
|
||||
<h2 id="___sec11">A more compact expression </h2>
|
||||
|
||||
<p>
|
||||
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
||||
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
||||
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
||||
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec12">A <b>scikit-learn</b> example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -384,6 +470,64 @@ plt.show()
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec13">A simple classification problem </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, linear_model
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_data</span>():
|
||||
np.random.seed(<span style="color: #B452CD">0</span>)
|
||||
X, y = datasets.make_moons(<span style="color: #B452CD">200</span>, noise=<span style="color: #B452CD">0.20</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> X, y
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">visualize</span>(X, y, clf):
|
||||
<span style="color: #228B22"># plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)</span>
|
||||
<span style="color: #228B22"># plt.show()</span>
|
||||
plot_decision_boundary(<span style="color: #8B008B; font-weight: bold">lambda</span> x: clf.predict(x), X, y)
|
||||
plt.title(<span style="color: #CD5555">"Logistic Regression"</span>)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_decision_boundary</span>(pred_func, X, y):
|
||||
<span style="color: #228B22"># Set min and max values and give it some padding</span>
|
||||
x_min, x_max = X[:, <span style="color: #B452CD">0</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">0</span>].max() + .<span style="color: #B452CD">5</span>
|
||||
y_min, y_max = X[:, <span style="color: #B452CD">1</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">1</span>].max() + .<span style="color: #B452CD">5</span>
|
||||
h = <span style="color: #B452CD">0.01</span>
|
||||
<span style="color: #228B22"># Generate a grid of points with distance h between them</span>
|
||||
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
||||
<span style="color: #228B22"># Predict the function value for the whole gid</span>
|
||||
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
||||
Z = Z.reshape(xx.shape)
|
||||
<span style="color: #228B22"># Plot the contour and training examples</span>
|
||||
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
||||
plt.scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], c=y, cmap=plt.cm.Spectral)
|
||||
plt.show()
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">classify</span>(X, y):
|
||||
clf = linear_model.LogisticRegressionCV()
|
||||
clf.fit(X, y)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> clf
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">main</span>():
|
||||
X, y = generate_data()
|
||||
<span style="color: #228B22"># visualize(X, y)</span>
|
||||
clf = classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #00688B">__name__</span> == <span style="color: #CD5555">"__main__"</span>:
|
||||
main()
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
</div> <!-- class="slides" -->
|
||||
</div> <!-- class="reveal" -->
|
||||
|
||||
@@ -35,16 +35,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -86,7 +89,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 20, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Sep 25, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -94,183 +97,260 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Logistic Regression </h2>
|
||||
|
||||
<p>
|
||||
So far we have focused on learning from datasets for which there is a
|
||||
<b>continuous</b> output. In linear regression we have been
|
||||
concerned with learning the coefficients of a polynomial to predict
|
||||
the response of a continuous variable \( y_i \) on unseen data based on
|
||||
its independent variables \( {\bf x}_i \).
|
||||
In linear regression our main interest was centered on learning the
|
||||
coefficients of a functional fit (say a polynomial) in order to be
|
||||
able to predict the response of a continuous variable on some unseen
|
||||
data. The fit to the continuous variable \( y_i \) is based on some
|
||||
independent variables \( \hat{x}_i \). Linear regression resulted in
|
||||
analytical expressions (in terms of matrices to invert) for several
|
||||
quantities, ranging from the variance and thereby the confidence
|
||||
intervals of the parameters \( \hat{\beta} \) to the mean squared
|
||||
error. If we can invert the product of the design matrices, linear
|
||||
regression gives then a simple recipe for fitting our data.
|
||||
|
||||
<p>
|
||||
Classification problems,
|
||||
however, are concerned with outcomes taking the form of discrete
|
||||
variables (i.e. categories). For example, we may want to detect if
|
||||
there's a cat or a dog in an image. Or given a specific system,
|
||||
we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).
|
||||
(e.g. ordered/disordered).
|
||||
Classification problems, however, are concerned with outcomes taking
|
||||
the form of discrete variables (i.e. categories). We may for example,
|
||||
on the basis of DNA sequencing for a number of patients, like to find
|
||||
out which mutations are important for a certain disease; or based on
|
||||
scans of various patients' brains, figure out if there is a tumor or
|
||||
not; or given a specific physical system, we'd like to identify its
|
||||
state, say whether it is an ordered or disordered system (typical
|
||||
situation in solid state physics); or classify the status of a
|
||||
patient, whether she/he has a stroke or not and many other similar
|
||||
situations.
|
||||
|
||||
<p>
|
||||
<b>Logistic regression deals with binary, dichotomous outcomes (e.g. True or
|
||||
False, Success or Failure, etc.). It is worth noting that logistic
|
||||
regression is also commonly used in modern supervised Deep Learning
|
||||
models</b>, as we will see later.
|
||||
The most common situation we encounter when we apply logistic
|
||||
regression is that of two possible outcomes, normally denoted as a
|
||||
binary outcome, true or false, positive or negative, success or
|
||||
failure etc.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec1">Optimization and Deep learning </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression will also serve as our stepping towards neural
|
||||
network algorithms and supervised deep learning. For logistic
|
||||
learning, the minimization of the cost function leads to a non-linear
|
||||
equation in the parameters \( \hat{\beta} \). The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms.
|
||||
|
||||
<p>
|
||||
We note also that many of the topics discussed here
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models, as we will see later.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec1">Basics </h2>
|
||||
<h2 id="___sec2">Basics </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables \( y_i\in\mathbb{Z} \)
|
||||
are discrete and only take values from \( m=0,\dots,M-1 \) (i.e. \( M \)
|
||||
classes).
|
||||
We consider the case where the dependent variables, also called the
|
||||
responses or the outcomes) \( y_i \) are discrete and only take values
|
||||
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
||||
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which bears \( p \) features. The
|
||||
made of \( n \) samples, each of which carries \( p \) features. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Linear classifier </h2>
|
||||
Let us specialize to the case of two classes only, with outputs \( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a credit card use who could default or not on her/his debt. That is
|
||||
$$
|
||||
y_i = \begin{bmatrix} 0 & \mathrm{no default}\\ 1 & \mathrm{default} \end{bmatrix}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Linear classifier </h2>
|
||||
|
||||
<p>
|
||||
Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if \( y_i > 0.5 \) and the no default case \( y_i \leq 0.5 \).
|
||||
|
||||
<p>
|
||||
We would then have our
|
||||
weighted linear combination, namely
|
||||
$$
|
||||
\begin{equation}
|
||||
s_i = \boldsymbol{x}_i^T\boldsymbol{w} + b_0 \equiv \mathbf{x}_i^T\mathbf{w},
|
||||
\hat{y} = \hat{X}_i^T\hat{\beta} + \hat{\epsilon},
|
||||
\label{_auto1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
where we use the short-hand notation
|
||||
\( \mathbf{x}_i = (1,\boldsymbol{x}_i) \) and \( \mathbf{w}_i = (b_0,\boldsymbol{w}_i) \).
|
||||
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
||||
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Some selected properties </h2>
|
||||
<h2 id="___sec4">Some selected properties </h2>
|
||||
|
||||
<p>
|
||||
This function takes values on the entire real axis. In the case of
|
||||
The main problem with our function is that it
|
||||
takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels \( y_i \) are discrete
|
||||
variables. One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise. Indeed,
|
||||
this is commonly known as the "perceptron" in the machine learning
|
||||
literature. This model is extremely simple, and it is favorable in
|
||||
many cases (e.g. noisy data) to have a ``soft" classifier that outputs
|
||||
the probability of a given category. For example, given
|
||||
\( \mathbf{x}_i \), the classifier outputs the probability of being in
|
||||
category \( m \). One such function is the logistic (or sigmoid) function:
|
||||
variables.
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
f(s) = \frac{1}{1+\mathrm e^{-s}}.
|
||||
\label{eq:log_fun}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
||||
<p>
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
||||
literature. This model is extremely simple. However, in many cases it is more
|
||||
favorable to use a ``soft" classifier that outputs
|
||||
the probability of a given category. This leads us to the logistic function.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">The cross-entropy as a cost function for logistic regression </h2>
|
||||
<h2 id="___sec5">The logistic function </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification": each datapoint is deterministically assigned to a category (i.e \( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft" classifier that outputs the probability of a given category rather than a single value. For example, given \( \mathbf{x}_i \), the classifier outputs the probability of being in category \( m \).
|
||||
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point \( \boldsymbol{x}_i \) belongs to a category \( y_i=\{0,1\} \) is is given by
|
||||
A widely used model is the so-called logistic (or sigmoid) function which ranges from \( 0 \) to \( 1 \),
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
||||
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
||||
\end{eqnarray}
|
||||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
$$
|
||||
|
||||
where \( \boldsymbol{\theta}=\mathbf{w} \) are the weights we wish to learn from the data.
|
||||
Note that \( 1-p(t)= p(-t) \).
|
||||
|
||||
<p>
|
||||
Notice that in terms of the logistic function, we can write
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Probabilistic classifiers </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification" model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given \( x_i \), the classifier
|
||||
outputs the probability of being in a category \( k \). Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point \( x_i \)
|
||||
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid)
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">Two parameters </h2>
|
||||
|
||||
<p>
|
||||
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
||||
$$
|
||||
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
||||
|
||||
<p>
|
||||
Note that we used
|
||||
$$
|
||||
p(y_i=1) = 1-p(y_i=0).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec5">Maximum likelihood </h2>
|
||||
<h2 id="___sec8">Maximum likelihood </h2>
|
||||
|
||||
<p>
|
||||
We now define the cost function for logistic regression using Maximum
|
||||
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
||||
to maximize the probability of seeing the observed data. Consider a
|
||||
dataset \( \mathcal{D}=\{(y_i,\boldsymbol{x}_i)\} \) with binary labels
|
||||
\( y_i\in\{0,1\} \) where the data points are drawn independently. The
|
||||
likelihood of the seeing the data under our model is just:
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
||||
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
||||
$$
|
||||
\begin{align}
|
||||
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
||||
\label{_auto2}
|
||||
\end{align}
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
from which we can readily compute the log-likelihood:
|
||||
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
||||
$$
|
||||
\begin{equation}
|
||||
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\label{_auto3}
|
||||
\end{equation}
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right].
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \)
|
||||
|
||||
<h2 id="___sec9">The cost function rewritten </h2>
|
||||
|
||||
<p>
|
||||
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
||||
$$
|
||||
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
||||
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
||||
\end{eqnarray}
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
||||
in practice we usually supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Minimizing the cross entropy </h2>
|
||||
<h2 id="___sec10">Minimizing the cross entropy </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \mathbf{w} \) and,
|
||||
therefore, any local minimizer is a global minimizer. Minimizing this
|
||||
cost function leads to the following equation
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
||||
\label{_auto4}
|
||||
\end{equation}
|
||||
$$
|
||||
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
<p>
|
||||
where we made use of the logistic function identity \( \partial_z f(z) =
|
||||
f(z)[1-f(z)] \). This equation defines a transcendental equation for
|
||||
\( \mathbf{w} \), the solution of which, unlike linear regression, cannot
|
||||
be written in a closed form.
|
||||
Here we need gradient descent methods!
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">A <b>scikit-learn</b> example </h2>
|
||||
<h2 id="___sec11">A more compact expression </h2>
|
||||
|
||||
<p>
|
||||
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
||||
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
||||
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
||||
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
||||
|
||||
$$
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec12">A <b>scikit-learn</b> example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -295,6 +375,63 @@ plt.plot(X_new, y_proba[:, <span style="color: #B452CD">0</span>], <span style="
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec13">A simple classification problem </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, linear_model
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_data</span>():
|
||||
np.random.seed(<span style="color: #B452CD">0</span>)
|
||||
X, y = datasets.make_moons(<span style="color: #B452CD">200</span>, noise=<span style="color: #B452CD">0.20</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> X, y
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">visualize</span>(X, y, clf):
|
||||
<span style="color: #228B22"># plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)</span>
|
||||
<span style="color: #228B22"># plt.show()</span>
|
||||
plot_decision_boundary(<span style="color: #8B008B; font-weight: bold">lambda</span> x: clf.predict(x), X, y)
|
||||
plt.title(<span style="color: #CD5555">"Logistic Regression"</span>)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_decision_boundary</span>(pred_func, X, y):
|
||||
<span style="color: #228B22"># Set min and max values and give it some padding</span>
|
||||
x_min, x_max = X[:, <span style="color: #B452CD">0</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">0</span>].max() + .<span style="color: #B452CD">5</span>
|
||||
y_min, y_max = X[:, <span style="color: #B452CD">1</span>].min() - .<span style="color: #B452CD">5</span>, X[:, <span style="color: #B452CD">1</span>].max() + .<span style="color: #B452CD">5</span>
|
||||
h = <span style="color: #B452CD">0.01</span>
|
||||
<span style="color: #228B22"># Generate a grid of points with distance h between them</span>
|
||||
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
||||
<span style="color: #228B22"># Predict the function value for the whole gid</span>
|
||||
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
||||
Z = Z.reshape(xx.shape)
|
||||
<span style="color: #228B22"># Plot the contour and training examples</span>
|
||||
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
||||
plt.scatter(X[:, <span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>], c=y, cmap=plt.cm.Spectral)
|
||||
plt.show()
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">classify</span>(X, y):
|
||||
clf = linear_model.LogisticRegressionCV()
|
||||
clf.fit(X, y)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> clf
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">main</span>():
|
||||
X, y = generate_data()
|
||||
<span style="color: #228B22"># visualize(X, y)</span>
|
||||
clf = classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> <span style="color: #00688B">__name__</span> == <span style="color: #CD5555">"__main__"</span>:
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
+245
-108
@@ -40,16 +40,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Logistic Regression', 2, None, '___sec0'),
|
||||
('Basics', 2, None, '___sec1'),
|
||||
('Linear classifier', 2, None, '___sec2'),
|
||||
('Some selected properties', 2, None, '___sec3'),
|
||||
('The cross-entropy as a cost function for logistic regression',
|
||||
2,
|
||||
None,
|
||||
'___sec4'),
|
||||
('Maximum likelihood', 2, None, '___sec5'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec6'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec7')]}
|
||||
('Optimization and Deep learning', 2, None, '___sec1'),
|
||||
('Basics', 2, None, '___sec2'),
|
||||
('Linear classifier', 2, None, '___sec3'),
|
||||
('Some selected properties', 2, None, '___sec4'),
|
||||
('The logistic function', 2, None, '___sec5'),
|
||||
('Probabilistic classifiers', 2, None, '___sec6'),
|
||||
('Two parameters', 2, None, '___sec7'),
|
||||
('Maximum likelihood', 2, None, '___sec8'),
|
||||
('The cost function rewritten', 2, None, '___sec9'),
|
||||
('Minimizing the cross entropy', 2, None, '___sec10'),
|
||||
('A more compact expression', 2, None, '___sec11'),
|
||||
('A _scikit-learn_ example', 2, None, '___sec12'),
|
||||
('A simple classification problem', 2, None, '___sec13')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -91,7 +94,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 20, 2018</h4></center> <!-- date -->
|
||||
<center><h4>Sep 25, 2018</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split -->
|
||||
@@ -99,183 +102,260 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Logistic Regression </h2>
|
||||
|
||||
<p>
|
||||
So far we have focused on learning from datasets for which there is a
|
||||
<b>continuous</b> output. In linear regression we have been
|
||||
concerned with learning the coefficients of a polynomial to predict
|
||||
the response of a continuous variable \( y_i \) on unseen data based on
|
||||
its independent variables \( {\bf x}_i \).
|
||||
In linear regression our main interest was centered on learning the
|
||||
coefficients of a functional fit (say a polynomial) in order to be
|
||||
able to predict the response of a continuous variable on some unseen
|
||||
data. The fit to the continuous variable \( y_i \) is based on some
|
||||
independent variables \( \hat{x}_i \). Linear regression resulted in
|
||||
analytical expressions (in terms of matrices to invert) for several
|
||||
quantities, ranging from the variance and thereby the confidence
|
||||
intervals of the parameters \( \hat{\beta} \) to the mean squared
|
||||
error. If we can invert the product of the design matrices, linear
|
||||
regression gives then a simple recipe for fitting our data.
|
||||
|
||||
<p>
|
||||
Classification problems,
|
||||
however, are concerned with outcomes taking the form of discrete
|
||||
variables (i.e. categories). For example, we may want to detect if
|
||||
there's a cat or a dog in an image. Or given a specific system,
|
||||
we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).
|
||||
(e.g. ordered/disordered).
|
||||
Classification problems, however, are concerned with outcomes taking
|
||||
the form of discrete variables (i.e. categories). We may for example,
|
||||
on the basis of DNA sequencing for a number of patients, like to find
|
||||
out which mutations are important for a certain disease; or based on
|
||||
scans of various patients' brains, figure out if there is a tumor or
|
||||
not; or given a specific physical system, we'd like to identify its
|
||||
state, say whether it is an ordered or disordered system (typical
|
||||
situation in solid state physics); or classify the status of a
|
||||
patient, whether she/he has a stroke or not and many other similar
|
||||
situations.
|
||||
|
||||
<p>
|
||||
<b>Logistic regression deals with binary, dichotomous outcomes (e.g. True or
|
||||
False, Success or Failure, etc.). It is worth noting that logistic
|
||||
regression is also commonly used in modern supervised Deep Learning
|
||||
models</b>, as we will see later.
|
||||
The most common situation we encounter when we apply logistic
|
||||
regression is that of two possible outcomes, normally denoted as a
|
||||
binary outcome, true or false, positive or negative, success or
|
||||
failure etc.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec1">Optimization and Deep learning </h2>
|
||||
|
||||
<p>
|
||||
Logistic regression will also serve as our stepping towards neural
|
||||
network algorithms and supervised deep learning. For logistic
|
||||
learning, the minimization of the cost function leads to a non-linear
|
||||
equation in the parameters \( \hat{\beta} \). The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms.
|
||||
|
||||
<p>
|
||||
We note also that many of the topics discussed here
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models, as we will see later.
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec1">Basics </h2>
|
||||
<h2 id="___sec2">Basics </h2>
|
||||
|
||||
<p>
|
||||
We consider the case where the dependent variables \( y_i\in\mathbb{Z} \)
|
||||
are discrete and only take values from \( m=0,\dots,M-1 \) (i.e. \( M \)
|
||||
classes).
|
||||
We consider the case where the dependent variables, also called the
|
||||
responses or the outcomes) \( y_i \) are discrete and only take values
|
||||
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
||||
|
||||
<p>
|
||||
The goal is to predict the
|
||||
output classes from the design matrix \( X\in\mathbb{R}^{n\times p} \)
|
||||
made of \( n \) samples, each of which bears \( p \) features. The
|
||||
made of \( n \) samples, each of which carries \( p \) features. The
|
||||
primary goal is to identify the classes to which new unseen samples
|
||||
belong.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec2">Linear classifier </h2>
|
||||
Let us specialize to the case of two classes only, with outputs \( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a credit card use who could default or not on her/his debt. That is
|
||||
$$
|
||||
y_i = \begin{bmatrix} 0 & \mathrm{no default}\\ 1 & \mathrm{default} \end{bmatrix}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Linear classifier </h2>
|
||||
|
||||
<p>
|
||||
Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if \( y_i > 0.5 \) and the no default case \( y_i \leq 0.5 \).
|
||||
|
||||
<p>
|
||||
We would then have our
|
||||
weighted linear combination, namely
|
||||
$$
|
||||
\begin{equation}
|
||||
s_i = \boldsymbol{x}_i^T\boldsymbol{w} + b_0 \equiv \mathbf{x}_i^T\mathbf{w},
|
||||
\hat{y} = \hat{X}_i^T\hat{\beta} + \hat{\epsilon},
|
||||
\label{_auto1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
where we use the short-hand notation
|
||||
\( \mathbf{x}_i = (1,\boldsymbol{x}_i) \) and \( \mathbf{w}_i = (b_0,\boldsymbol{w}_i) \).
|
||||
where \( \hat{y} \) is a vector representing the possible outcomes, \( \hat{X} \) is our
|
||||
\( n\times p \) design matrix and \( \hat{\beta} \) represents our estimators.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec3">Some selected properties </h2>
|
||||
<h2 id="___sec4">Some selected properties </h2>
|
||||
|
||||
<p>
|
||||
This function takes values on the entire real axis. In the case of
|
||||
The main problem with our function is that it
|
||||
takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels \( y_i \) are discrete
|
||||
variables. One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise. Indeed,
|
||||
this is commonly known as the "perceptron" in the machine learning
|
||||
literature. This model is extremely simple, and it is favorable in
|
||||
many cases (e.g. noisy data) to have a ``soft" classifier that outputs
|
||||
the probability of a given category. For example, given
|
||||
\( \mathbf{x}_i \), the classifier outputs the probability of being in
|
||||
category \( m \). One such function is the logistic (or sigmoid) function:
|
||||
variables.
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
f(s) = \frac{1}{1+\mathrm e^{-s}}.
|
||||
\label{eq:log_fun}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
Note that \( 1-f(s)= f(-s) \), which will be useful shortly.
|
||||
<p>
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values \( \{0,1\} \),
|
||||
\( f(s_i)=sign(s_i)=1 \) if \( s_i\ge 0 \) and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the "perceptron" model in the machine learning
|
||||
literature. This model is extremely simple. However, in many cases it is more
|
||||
favorable to use a ``soft" classifier that outputs
|
||||
the probability of a given category. This leads us to the logistic function.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec4">The cross-entropy as a cost function for logistic regression </h2>
|
||||
<h2 id="___sec5">The logistic function </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification": each datapoint is deterministically assigned to a category (i.e \( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft" classifier that outputs the probability of a given category rather than a single value. For example, given \( \mathbf{x}_i \), the classifier outputs the probability of being in category \( m \).
|
||||
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point \( \boldsymbol{x}_i \) belongs to a category \( y_i=\{0,1\} \) is is given by
|
||||
A widely used model is the so-called logistic (or sigmoid) function which ranges from \( 0 \) to \( 1 \),
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
||||
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
||||
\end{eqnarray}
|
||||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
$$
|
||||
|
||||
where \( \boldsymbol{\theta}=\mathbf{w} \) are the weights we wish to learn from the data.
|
||||
Note that \( 1-p(t)= p(-t) \).
|
||||
|
||||
<p>
|
||||
Notice that in terms of the logistic function, we can write
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Probabilistic classifiers </h2>
|
||||
|
||||
<p>
|
||||
The perceptron is an example of a ``hard classification" model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
\( y_i=0 \) or \( y_i=1 \)). In many cases, it is favorable to have a "soft"
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given \( x_i \), the classifier
|
||||
outputs the probability of being in a category \( k \). Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point \( x_i \)
|
||||
belongs to a category \( y_i=\{0,1\} \) is given by the so-called logit function (or Sigmoid)
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">Two parameters </h2>
|
||||
|
||||
<p>
|
||||
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
|
||||
$$
|
||||
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
where \( \hat{\beta} \) are the weights we wish to extract from data, in our case \( \beta_0 \) and \( \beta_1 \).
|
||||
|
||||
<p>
|
||||
Note that we used
|
||||
$$
|
||||
p(y_i=1) = 1-p(y_i=0).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec5">Maximum likelihood </h2>
|
||||
<h2 id="___sec8">Maximum likelihood </h2>
|
||||
|
||||
<p>
|
||||
We now define the cost function for logistic regression using Maximum
|
||||
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
||||
to maximize the probability of seeing the observed data. Consider a
|
||||
dataset \( \mathcal{D}=\{(y_i,\boldsymbol{x}_i)\} \) with binary labels
|
||||
\( y_i\in\{0,1\} \) where the data points are drawn independently. The
|
||||
likelihood of the seeing the data under our model is just:
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
||||
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
||||
$$
|
||||
\begin{align}
|
||||
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
||||
\label{_auto2}
|
||||
\end{align}
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
$$
|
||||
|
||||
from which we can readily compute the log-likelihood:
|
||||
from which we obtain the log-likelihood and our <b>cost/loss</b> function
|
||||
$$
|
||||
\begin{equation}
|
||||
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\label{_auto3}
|
||||
\end{equation}
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right].
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \)
|
||||
|
||||
<h2 id="___sec9">The cost function rewritten </h2>
|
||||
|
||||
<p>
|
||||
Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as
|
||||
$$
|
||||
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \beta \).
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
$$
|
||||
\begin{eqnarray}
|
||||
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
||||
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
||||
\end{eqnarray}
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
||||
in practice we usually supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec6">Minimizing the cross entropy </h2>
|
||||
<h2 id="___sec10">Minimizing the cross entropy </h2>
|
||||
|
||||
<p>
|
||||
The cross entropy is a convex function of the weights \( \mathbf{w} \) and,
|
||||
therefore, any local minimizer is a global minimizer. Minimizing this
|
||||
cost function leads to the following equation
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
||||
\label{_auto4}
|
||||
\end{equation}
|
||||
$$
|
||||
The cross entropy is a convex function of the weights \( \hat{\beta} \) and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
<p>
|
||||
where we made use of the logistic function identity \( \partial_z f(z) =
|
||||
f(z)[1-f(z)] \). This equation defines a transcendental equation for
|
||||
\( \mathbf{w} \), the solution of which, unlike linear regression, cannot
|
||||
be written in a closed form.
|
||||
Here we need gradient descent methods!
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters \( \beta_0 \) and \( \beta_1 \) we obtain
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec7">A <b>scikit-learn</b> example </h2>
|
||||
<h2 id="___sec11">A more compact expression </h2>
|
||||
|
||||
<p>
|
||||
Let us now define a vector \( \hat{y} \) with \( n \) elements \( y_i \), an
|
||||
\( n\times p \) matrix \( \hat{X} \) which contains the \( x_i \) values and a
|
||||
vector \( \hat{p} \) of fitted probabilities \( p(y_i\vert x_i,\hat{\beta}) \). We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
$$
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
$$
|
||||
|
||||
<p>
|
||||
If we in addition define a diagonal matrix \( \hat{W} \) with elements
|
||||
\( p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta}) \), we can obtain a compact expression of the second derivative as
|
||||
|
||||
$$
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec12">A <b>scikit-learn</b> example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -300,6 +380,63 @@ plt<span style="color: #666666">.</span>plot(X_new, y_proba[:, <span style="colo
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec13">A simple classification problem </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets, linear_model
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_data</span>():
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
X, y <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>make_moons(<span style="color: #666666">200</span>, noise<span style="color: #666666">=0.20</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> X, y
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">visualize</span>(X, y, clf):
|
||||
<span style="color: #408080; font-style: italic"># plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)</span>
|
||||
<span style="color: #408080; font-style: italic"># plt.show()</span>
|
||||
plot_decision_boundary(<span style="color: #008000; font-weight: bold">lambda</span> x: clf<span style="color: #666666">.</span>predict(x), X, y)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Logistic Regression"</span>)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_decision_boundary</span>(pred_func, X, y):
|
||||
<span style="color: #408080; font-style: italic"># Set min and max values and give it some padding</span>
|
||||
x_min, x_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">0</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
y_min, y_max <span style="color: #666666">=</span> X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>min() <span style="color: #666666">-</span> <span style="color: #666666">.5</span>, X[:, <span style="color: #666666">1</span>]<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">.5</span>
|
||||
h <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
|
||||
<span style="color: #408080; font-style: italic"># Generate a grid of points with distance h between them</span>
|
||||
xx, yy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>meshgrid(np<span style="color: #666666">.</span>arange(x_min, x_max, h), np<span style="color: #666666">.</span>arange(y_min, y_max, h))
|
||||
<span style="color: #408080; font-style: italic"># Predict the function value for the whole gid</span>
|
||||
Z <span style="color: #666666">=</span> pred_func(np<span style="color: #666666">.</span>c_[xx<span style="color: #666666">.</span>ravel(), yy<span style="color: #666666">.</span>ravel()])
|
||||
Z <span style="color: #666666">=</span> Z<span style="color: #666666">.</span>reshape(xx<span style="color: #666666">.</span>shape)
|
||||
<span style="color: #408080; font-style: italic"># Plot the contour and training examples</span>
|
||||
plt<span style="color: #666666">.</span>contourf(xx, yy, Z, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>scatter(X[:, <span style="color: #666666">0</span>], X[:, <span style="color: #666666">1</span>], c<span style="color: #666666">=</span>y, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">classify</span>(X, y):
|
||||
clf <span style="color: #666666">=</span> linear_model<span style="color: #666666">.</span>LogisticRegressionCV()
|
||||
clf<span style="color: #666666">.</span>fit(X, y)
|
||||
<span style="color: #008000; font-weight: bold">return</span> clf
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">main</span>():
|
||||
X, y <span style="color: #666666">=</span> generate_data()
|
||||
<span style="color: #408080; font-style: italic"># visualize(X, y)</span>
|
||||
clf <span style="color: #666666">=</span> classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"__main__"</span>:
|
||||
main()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
+262
-120
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Sep 20, 2018**\n",
|
||||
"Date: **Sep 25, 2018**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -20,42 +20,81 @@
|
||||
"<!-- !split -->\n",
|
||||
"## Logistic Regression\n",
|
||||
"\n",
|
||||
"So far we have focused on learning from datasets for which there is a\n",
|
||||
"**continuous** output. In linear regression we have been \n",
|
||||
"concerned with learning the coefficients of a polynomial to predict\n",
|
||||
"the response of a continuous variable $y_i$ on unseen data based on\n",
|
||||
"its independent variables ${\\bf x}_i$. \n",
|
||||
"In linear regression our main interest was centered on learning the\n",
|
||||
"coefficients of a functional fit (say a polynomial) in order to be\n",
|
||||
"able to predict the response of a continuous variable on some unseen\n",
|
||||
"data. The fit to the continuous variable $y_i$ is based on some\n",
|
||||
"independent variables $\\hat{x}_i$. Linear regression resulted in\n",
|
||||
"analytical expressions (in terms of matrices to invert) for several\n",
|
||||
"quantities, ranging from the variance and thereby the confidence\n",
|
||||
"intervals of the parameters $\\hat{\\beta}$ to the mean squared\n",
|
||||
"error. If we can invert the product of the design matrices, linear\n",
|
||||
"regression gives then a simple recipe for fitting our data.\n",
|
||||
"\n",
|
||||
"Classification problems,\n",
|
||||
"however, are concerned with outcomes taking the form of discrete\n",
|
||||
"variables (i.e. categories). For example, we may want to detect if\n",
|
||||
"there's a cat or a dog in an image. Or given a specific system,\n",
|
||||
"we'd like to identify its state, say whether it is an ordered or disordered system (typical situation in solid state physics).\n",
|
||||
"(e.g. ordered/disordered). \n",
|
||||
"\n",
|
||||
"**Logistic regression deals with binary, dichotomous outcomes (e.g. True or\n",
|
||||
"False, Success or Failure, etc.). It is worth noting that logistic\n",
|
||||
"regression is also commonly used in modern supervised Deep Learning\n",
|
||||
"models**, as we will see later.\n",
|
||||
"Classification problems, however, are concerned with outcomes taking\n",
|
||||
"the form of discrete variables (i.e. categories). We may for example,\n",
|
||||
"on the basis of DNA sequencing for a number of patients, like to find\n",
|
||||
"out which mutations are important for a certain disease; or based on\n",
|
||||
"scans of various patients' brains, figure out if there is a tumor or\n",
|
||||
"not; or given a specific physical system, we'd like to identify its\n",
|
||||
"state, say whether it is an ordered or disordered system (typical\n",
|
||||
"situation in solid state physics); or classify the status of a\n",
|
||||
"patient, whether she/he has a stroke or not and many other similar\n",
|
||||
"situations.\n",
|
||||
"\n",
|
||||
"The most common situation we encounter when we apply logistic\n",
|
||||
"regression is that of two possible outcomes, normally denoted as a\n",
|
||||
"binary outcome, true or false, positive or negative, success or\n",
|
||||
"failure etc.\n",
|
||||
"\n",
|
||||
"## Optimization and Deep learning\n",
|
||||
"\n",
|
||||
"Logistic regression will also serve as our stepping towards neural\n",
|
||||
"network algorithms and supervised deep learning. For logistic\n",
|
||||
"learning, the minimization of the cost function leads to a non-linear\n",
|
||||
"equation in the parameters $\\hat{\\beta}$. The optmization of the problem calls therefore for minimization algorithms. This forms the bottle neck of all machine learning algorithms, namely how to find reliable minima of a multi-variable function. This leads us to the family of gradient descent methods. The latter are the working horses of basically all modern machine learning algorithms. \n",
|
||||
"\n",
|
||||
"We note also that many of the topics discussed here \n",
|
||||
"regression are also commonly used in modern supervised Deep Learning\n",
|
||||
"models, as we will see later.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"<!-- !split -->\n",
|
||||
"## Basics\n",
|
||||
"\n",
|
||||
"We consider the case where the dependent variables $y_i\\in\\mathbb{Z}$\n",
|
||||
"are discrete and only take values from $m=0,\\dots,M-1$ (i.e. $M$\n",
|
||||
"classes).\n",
|
||||
"We consider the case where the dependent variables, also called the\n",
|
||||
"responses or the outcomes) $y_i$ are discrete and only take values\n",
|
||||
"from $k=0,\\dots,K-1$ (i.e. $K$ classes).\n",
|
||||
"\n",
|
||||
"The goal is to predict the\n",
|
||||
"output classes from the design matrix $X\\in\\mathbb{R}^{n\\times p}$\n",
|
||||
"made of $n$ samples, each of which bears $p$ features. The\n",
|
||||
"made of $n$ samples, each of which carries $p$ features. The\n",
|
||||
"primary goal is to identify the classes to which new unseen samples\n",
|
||||
"belong.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Let us specialize to the case of two classes only, with outputs $y_i=0$ and $y_i=1$. Our outcomes could represent the status of a credit card use who could default or not on her/his debt. That is"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"y_i = \\begin{bmatrix} 0 & \\mathrm{no default}\\\\ 1 & \\mathrm{default} \\end{bmatrix}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Linear classifier\n",
|
||||
"\n",
|
||||
"Let us start by considering a slightly simpler classifier: a linear classifier that categorizes examples using a weighted linear-combination of the features and an additive offset"
|
||||
"Before moving to the logistic model, let us try to use our linear regression model to classify these two outcomes. We could for example fit a linear model to the default case if $y_i > 0.5$ and the no default case $y_i \\leq 0.5$. \n",
|
||||
"\n",
|
||||
"We would then have our \n",
|
||||
"weighted linear combination, namely"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -67,7 +106,7 @@
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
"s_i = \\boldsymbol{x}_i^T\\boldsymbol{w} + b_0 \\equiv \\mathbf{x}_i^T\\mathbf{w},\n",
|
||||
"\\hat{y} = \\hat{X}_i^T\\hat{\\beta} + \\hat{\\epsilon},\n",
|
||||
"\\label{_auto1} \\tag{1}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
@@ -77,36 +116,35 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where we use the short-hand notation \n",
|
||||
"$\\mathbf{x}_i = (1,\\boldsymbol{x}_i)$ and $\\mathbf{w}_i = (b_0,\\boldsymbol{w}_i)$. \n",
|
||||
"where $\\hat{y}$ is a vector representing the possible outcomes, $\\hat{X}$ is our\n",
|
||||
"$n\\times p$ design matrix and $\\hat{\\beta}$ represents our estimators.\n",
|
||||
"\n",
|
||||
"## Some selected properties\n",
|
||||
"\n",
|
||||
"This function takes values on the entire real axis. In the case of\n",
|
||||
"The main problem with our function is that it \n",
|
||||
"takes values on the entire real axis. In the case of\n",
|
||||
"logistic regression, however, the labels $y_i$ are discrete\n",
|
||||
"variables. One simple way to get a discrete output is to have sign\n",
|
||||
"functions that map the output of a linear regressor to $\\{0,1\\}$,\n",
|
||||
"$f(s_i)=sign(s_i)=1$ if $s_i\\ge 0$ and 0 if otherwise. Indeed,\n",
|
||||
"this is commonly known as the \"perceptron\" in the machine learning\n",
|
||||
"literature. This model is extremely simple, and it is favorable in\n",
|
||||
"many cases (e.g. noisy data) to have a ``soft\" classifier that outputs\n",
|
||||
"the probability of a given category. For example, given\n",
|
||||
"$\\mathbf{x}_i$, the classifier outputs the probability of being in\n",
|
||||
"category $m$. One such function is the logistic (or sigmoid) function:"
|
||||
"variables. \n",
|
||||
"\n",
|
||||
"One simple way to get a discrete output is to have sign\n",
|
||||
"functions that map the output of a linear regressor to values $\\{0,1\\}$,\n",
|
||||
"$f(s_i)=sign(s_i)=1$ if $s_i\\ge 0$ and 0 if otherwise. \n",
|
||||
"We will encounter this model in our first demonstration of neural networks. Historically it is called the \"perceptron\" model in the machine learning\n",
|
||||
"literature. This model is extremely simple. However, in many cases it is more\n",
|
||||
"favorable to use a ``soft\" classifier that outputs\n",
|
||||
"the probability of a given category. This leads us to the logistic function.\n",
|
||||
"\n",
|
||||
"## The logistic function\n",
|
||||
"\n",
|
||||
"A widely used model is the so-called logistic (or sigmoid) function which ranges from $0$ to $1$,"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"eq:log_fun\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
"f(s) = \\frac{1}{1+\\mathrm e^{-s}}.\n",
|
||||
"\\label{eq:log_fun} \\tag{2}\n",
|
||||
"\\end{equation}\n",
|
||||
"p(t) = \\frac{1}{1+\\mathrm \\exp{-t}}=\\frac{\\exp{t}}{1+\\mathrm \\exp{t}}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -114,12 +152,25 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that $1-f(s)= f(-s)$, which will be useful shortly. \n",
|
||||
"Note that $1-p(t)= p(-t)$.\n",
|
||||
"\n",
|
||||
"## The cross-entropy as a cost function for logistic regression\n",
|
||||
"## Probabilistic classifiers\n",
|
||||
"\n",
|
||||
"The perceptron is an example of a ``hard classification\": each datapoint is deterministically assigned to a category (i.e $y_i=0$ or $y_i=1$). In many cases, it is favorable to have a \"soft\" classifier that outputs the probability of a given category rather than a single value. For example, given $\\mathbf{x}_i$, the classifier outputs the probability of being in category $m$. \n",
|
||||
"Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point $\\boldsymbol{x}_i$ belongs to a category $y_i=\\{0,1\\}$ is is given by"
|
||||
"The perceptron is an example of a ``hard classification\" model. We\n",
|
||||
"will encounter this model when we discuss neural networks as\n",
|
||||
"well. Each datapoint is deterministically assigned to a category (i.e\n",
|
||||
"$y_i=0$ or $y_i=1$). In many cases, it is favorable to have a \"soft\"\n",
|
||||
"classifier that outputs the probability of a given category rather\n",
|
||||
"than a single value. For example, given $x_i$, the classifier\n",
|
||||
"outputs the probability of being in a category $k$. Logistic regression\n",
|
||||
"is the most common example of a so-called soft classifier. In logistic\n",
|
||||
"regression, the probability that a data point $x_i$\n",
|
||||
"belongs to a category $y_i=\\{0,1\\}$ is given by the so-called logit function (or Sigmoid)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Two parameters\n",
|
||||
"\n",
|
||||
"We assume now that we have two classes with $y_i$ either $0$ or $1$. Furthermore we assume also that we have only two parameters $\\beta$ in our fitting of the Sigmoid function, that is we define probabilities"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -127,10 +178,10 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\begin{eqnarray}\n",
|
||||
"P(y_i=1|\\boldsymbol{x}_i,\\boldsymbol{\\theta)} &=& \\frac{1}{1+\\mathrm{e}^{-\\mathbf{x}^T_i\\mathbf{w}}},\\nonumber\\\\\n",
|
||||
"P(y_i=0|\\boldsymbol{x}_i,\\boldsymbol{\\theta)} &=& 1 - P(y_i=1|\\boldsymbol{x}_i,\\boldsymbol{\\theta)},\n",
|
||||
"\\end{eqnarray}\n",
|
||||
"\\begin{align*}\n",
|
||||
"p(y_i=1|x_i,\\hat{\\beta}) &= \\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}},\\nonumber\\\\\n",
|
||||
"p(y_i=0|x_i,\\hat{\\beta}) &= 1 - p(y_i=1|x_i,\\hat{\\beta}),\n",
|
||||
"\\end{align*}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -138,10 +189,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $\\boldsymbol{\\theta}=\\mathbf{w}$ are the weights we wish to learn from the data. \n",
|
||||
"where $\\hat{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Notice that in terms of the logistic function, we can write"
|
||||
"Note that we used"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -149,7 +199,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"P(y_i=1) =f(\\mathbf{x}_i^T\\mathbf{w})=1-P(y_i=0).\n",
|
||||
"p(y_i=1) = 1-p(y_i=0).\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -160,12 +210,12 @@
|
||||
"<!-- !split -->\n",
|
||||
"## Maximum likelihood\n",
|
||||
"\n",
|
||||
"We now define the cost function for logistic regression using Maximum\n",
|
||||
"Likelihood Estimation (MLE). Recall, that in MLE we choose parameters\n",
|
||||
"to maximize the probability of seeing the observed data. Consider a\n",
|
||||
"dataset $\\mathcal{D}=\\{(y_i,\\boldsymbol{x}_i)\\}$ with binary labels\n",
|
||||
"$y_i\\in\\{0,1\\}$ where the data points are drawn independently. The\n",
|
||||
"likelihood of the seeing the data under our model is just:"
|
||||
"In order to define the total likelihood for all possible outcomes from a \n",
|
||||
"dataset $\\mathcal{D}=\\{(y_i,x_i)\\}$, with the binary labels\n",
|
||||
"$y_i\\in\\{0,1\\}$ and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle. \n",
|
||||
"We aim thus at maximizing \n",
|
||||
"the probability of seeing the observed data. We can then approximate the \n",
|
||||
"likelihood in terms of the product of the individual probabilities of a specific outcome $y_i$, that is"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -173,7 +223,9 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"P(\\mathcal{D}|\\mathbf{w}) = \\prod_{i=1}^n \\left[f(\\mathbf{x}_i^T\\mathbf{w})\\right]^{y_i}\\left[1-f(\\mathbf{x}_i^T\\mathbf{w})\\right]^{1-y_i}\\nonumber\n",
|
||||
"\\begin{align*}\n",
|
||||
"P(\\mathcal{D}|\\hat{\\beta})& = \\prod_{i=1}^n \\left[p(y_i=1|x_i,\\hat{\\beta})\\right]^{y_i}\\left[1-p(y_i=1|x_i,\\hat{\\beta}))\\right]^{1-y_i}\\nonumber \\\\\n",
|
||||
"\\end{align*}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -181,43 +233,25 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto2\"></div>\n",
|
||||
"from which we obtain the log-likelihood and our **cost/loss** function"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\mathcal{C}(\\hat{\\beta}) = \\sum_{i=1}^n y_i\\log{p(y_i=1|x_i,\\hat{\\beta})} + (1-y_i)\\log\\left[1-p(y_i=1|x_i,\\hat{\\beta}))\\right].\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The cost function rewritten\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation} \n",
|
||||
"\\label{_auto2} \\tag{3}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"from which we can readily compute the log-likelihood:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto3\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
"l(\\mathbf{w}) = \\sum_{i=1}^n y_i\\log f(\\mathbf{x}_i^T\\mathbf{w}) + (1-y_i)\\log\\left[1-f(\\mathbf{x}_i^T\\mathbf{w})\\right].\n",
|
||||
"\\label{_auto3} \\tag{4}\n",
|
||||
"\\end{equation}\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\\theta$"
|
||||
"Reordering the logarithms, we can rewrite the **cost/loss** function as"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -225,7 +259,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\hat{\\mathbf{w}} = \\sum_{i=1}^n y_i\\log f(\\mathbf{x}_i^T\\mathbf{w}) + (1-y_i)\\log\\left[1-f(\\mathbf{x}_i^T\\mathbf{w})\\right].\n",
|
||||
"\\mathcal{C}(\\hat{\\beta}) = \\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right).\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -233,6 +267,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\\beta$.\n",
|
||||
"Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that"
|
||||
]
|
||||
},
|
||||
@@ -241,10 +276,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\begin{eqnarray}\n",
|
||||
"\\mathcal{C}(\\mathbf{w}) &=& - l(\\mathbf{w}) \\\\\n",
|
||||
"&=& \\sum_{i=1}^n -y_i\\log f(\\mathbf{x}_i^T\\mathbf{w}) - (1-y_i)\\log\\left[1-f(\\mathbf{x}_i^T\\mathbf{w})\\right].\\nonumber\n",
|
||||
"\\end{eqnarray}\n",
|
||||
"\\mathcal{C}(\\hat{\\beta})=-\\sum_{i=1}^n \\left(y_i(\\beta_0+\\beta_1x_i) -\\log{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right).\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -252,28 +284,25 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This equation is known in statistics as the \\emph{cross entropy}. Finally, we note that just as in linear regression, \n",
|
||||
"in practice we usually supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression.\n",
|
||||
"This equation is known in statistics as the **cross entropy**. Finally, we note that just as in linear regression, \n",
|
||||
"in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression.\n",
|
||||
"\n",
|
||||
"## Minimizing the cross entropy\n",
|
||||
"\n",
|
||||
"The cross entropy is a convex function of the weights $\\mathbf{w}$ and,\n",
|
||||
"therefore, any local minimizer is a global minimizer. Minimizing this\n",
|
||||
"cost function leads to the following equation"
|
||||
"The cross entropy is a convex function of the weights $\\hat{\\beta}$ and,\n",
|
||||
"therefore, any local minimizer is a global minimizer. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Minimizing this\n",
|
||||
"cost function with respect to the two parameters $\\beta_0$ and $\\beta_1$ we obtain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<!-- Equation labels as ordinary links -->\n",
|
||||
"<div id=\"_auto4\"></div>\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{equation}\n",
|
||||
"\\boldsymbol{0}=\\boldsymbol{\\nabla} \\mathcal{C}(\\mathbf{w}) = \\sum_{i=1}^n\\left[f(\\mathbf{x}_i^T\\mathbf{w})-y_i\\right]\\mathbf{x}_i,\n",
|
||||
"\\label{_auto4} \\tag{5}\n",
|
||||
"\\end{equation}\n",
|
||||
"\\frac{\\partial \\mathcal{C}(\\hat{\\beta})}{\\partial \\beta_0} = -\\sum_{i=1}^n \\left(y_i -\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right),\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -281,13 +310,60 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where we made use of the logistic function identity $\\partial_z f(z) =\n",
|
||||
"f(z)[1-f(z)]$. This equation defines a transcendental equation for\n",
|
||||
"$\\mathbf{w}$, the solution of which, unlike linear regression, cannot\n",
|
||||
"be written in a closed form. \n",
|
||||
"Here we need gradient descent methods!\n",
|
||||
"\n",
|
||||
"and"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\frac{\\partial \\mathcal{C}(\\hat{\\beta})}{\\partial \\beta_1} = -\\sum_{i=1}^n \\left(y_ix_i -x_i\\frac{\\exp{(\\beta_0+\\beta_1x_i)}}{1+\\exp{(\\beta_0+\\beta_1x_i)}}\\right).\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## A more compact expression\n",
|
||||
"\n",
|
||||
"Let us now define a vector $\\hat{y}$ with $n$ elements $y_i$, an\n",
|
||||
"$n\\times p$ matrix $\\hat{X}$ which contains the $x_i$ values and a\n",
|
||||
"vector $\\hat{p}$ of fitted probabilities $p(y_i\\vert x_i,\\hat{\\beta})$. We can rewrite in a more compact form the first\n",
|
||||
"derivative of cost function as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\frac{\\partial \\mathcal{C}(\\hat{\\beta})}{\\partial \\hat{\\beta}} = -\\hat{X}^T\\left(\\hat{y}-\\hat{p}\\right).\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"If we in addition define a diagonal matrix $\\hat{W}$ with elements \n",
|
||||
"$p(y_i\\vert x_i,\\hat{\\beta})(1-p(y_i\\vert x_i,\\hat{\\beta})$, we can obtain a compact expression of the second derivative as"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"\\frac{\\partial^2 \\mathcal{C}(\\hat{\\beta})}{\\partial \\hat{\\beta}\\partial \\hat{\\beta}^T} = \\hat{X}^T\\hat{W}\\hat{X}.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## A **scikit-learn** example"
|
||||
]
|
||||
},
|
||||
@@ -320,6 +396,72 @@
|
||||
"plt.plot(X_new, y_proba[:, 0], \"b--\", label=\"Not Iris-Virginica\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## A simple classification problem"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from sklearn import datasets, linear_model\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def generate_data():\n",
|
||||
" np.random.seed(0)\n",
|
||||
" X, y = datasets.make_moons(200, noise=0.20)\n",
|
||||
" return X, y\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def visualize(X, y, clf):\n",
|
||||
" # plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)\n",
|
||||
" # plt.show()\n",
|
||||
" plot_decision_boundary(lambda x: clf.predict(x), X, y)\n",
|
||||
" plt.title(\"Logistic Regression\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def plot_decision_boundary(pred_func, X, y):\n",
|
||||
" # Set min and max values and give it some padding\n",
|
||||
" x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n",
|
||||
" y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n",
|
||||
" h = 0.01\n",
|
||||
" # Generate a grid of points with distance h between them\n",
|
||||
" xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\n",
|
||||
" # Predict the function value for the whole gid\n",
|
||||
" Z = pred_func(np.c_[xx.ravel(), yy.ravel()])\n",
|
||||
" Z = Z.reshape(xx.shape)\n",
|
||||
" # Plot the contour and training examples\n",
|
||||
" plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)\n",
|
||||
" plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def classify(X, y):\n",
|
||||
" clf = linear_model.LogisticRegressionCV()\n",
|
||||
" clf.fit(X, y)\n",
|
||||
" return clf\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def main():\n",
|
||||
" X, y = generate_data()\n",
|
||||
" # visualize(X, y)\n",
|
||||
" clf = classify(X, y)\n",
|
||||
" visualize(X, y, clf)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" main()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -44,7 +44,7 @@ equation in the parameters $\hat{\beta}$. The optmization of the problem calls t
|
||||
|
||||
We note also that many of the topics discussed here
|
||||
regression are also commonly used in modern supervised Deep Learning
|
||||
models_, as we will see later.
|
||||
models, as we will see later.
|
||||
|
||||
|
||||
!split
|
||||
@@ -90,7 +90,9 @@ $n\times p$ design matrix and $\hat{\beta}$ represents our estimators.
|
||||
The main problem with our function is that it
|
||||
takes values on the entire real axis. In the case of
|
||||
logistic regression, however, the labels $y_i$ are discrete
|
||||
variables. One simple way to get a discrete output is to have sign
|
||||
variables.
|
||||
|
||||
One simple way to get a discrete output is to have sign
|
||||
functions that map the output of a linear regressor to values $\{0,1\}$,
|
||||
$f(s_i)=sign(s_i)=1$ if $s_i\ge 0$ and 0 if otherwise.
|
||||
We will encounter this model in our first demonstration of neural networks. Historically it is called the ``perceptron" model in the machine learning
|
||||
@@ -110,82 +112,124 @@ p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||||
Note that $1-p(t)= p(-t)$.
|
||||
|
||||
!split
|
||||
===== The cross-entropy as a cost function for logistic regression =====
|
||||
===== Probabilistic classifiers =====
|
||||
|
||||
The perceptron is an example of a ``hard classification'': each datapoint is deterministically assigned to a category (i.e $y_i=0$ or $y_i=1$). In many cases, it is favorable to have a ``soft'' classifier that outputs the probability of a given category rather than a single value. For example, given $\mathbf{x}_i$, the classifier outputs the probability of being in category $m$.
|
||||
Logistic regression is the most canonical example of a soft classifier. In logistic regression, the probability that a data point $\boldsymbol{x}_i$ belongs to a category $y_i=\{0,1\}$ is is given by
|
||||
The perceptron is an example of a ``hard classification'' model. We
|
||||
will encounter this model when we discuss neural networks as
|
||||
well. Each datapoint is deterministically assigned to a category (i.e
|
||||
$y_i=0$ or $y_i=1$). In many cases, it is favorable to have a ``soft''
|
||||
classifier that outputs the probability of a given category rather
|
||||
than a single value. For example, given $x_i$, the classifier
|
||||
outputs the probability of being in a category $k$. Logistic regression
|
||||
is the most common example of a so-called soft classifier. In logistic
|
||||
regression, the probability that a data point $x_i$
|
||||
belongs to a category $y_i=\{0,1\}$ is given by the so-called logit function (or Sigmoid)
|
||||
|
||||
|
||||
!split
|
||||
===== Two parameters =====
|
||||
|
||||
We assume now that we have two classes with $y_i$ either $0$ or $1$. Furthermore we assume also that we have only two parameters $\beta$ in our fitting of the Sigmoid function, that is we define probabilities
|
||||
!bt
|
||||
\begin{eqnarray}
|
||||
P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& \frac{1}{1+\mathrm{e}^{-\mathbf{x}^T_i\mathbf{w}}},\nonumber\\
|
||||
P(y_i=0|\boldsymbol{x}_i,\boldsymbol{\theta)} &=& 1 - P(y_i=1|\boldsymbol{x}_i,\boldsymbol{\theta)},
|
||||
\end{eqnarray}
|
||||
\begin{align*}
|
||||
p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\
|
||||
p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}),
|
||||
\end{align*}
|
||||
!et
|
||||
where $\boldsymbol{\theta}=\mathbf{w}$ are the weights we wish to learn from the data.
|
||||
where $\hat{\beta}$ are the weights we wish to extract from data, in our case $\beta_0$ and $\beta_1$.
|
||||
|
||||
|
||||
Notice that in terms of the logistic function, we can write
|
||||
Note that we used
|
||||
!bt
|
||||
\[
|
||||
P(y_i=1) =f(\mathbf{x}_i^T\mathbf{w})=1-P(y_i=0).
|
||||
p(y_i=1) = 1-p(y_i=0).
|
||||
\]
|
||||
!et
|
||||
|
||||
!split
|
||||
===== Maximum likelihood =====
|
||||
|
||||
We now define the cost function for logistic regression using Maximum
|
||||
Likelihood Estimation (MLE). Recall, that in MLE we choose parameters
|
||||
to maximize the probability of seeing the observed data. Consider a
|
||||
dataset $\mathcal{D}=\{(y_i,\boldsymbol{x}_i)\}$ with binary labels
|
||||
$y_i\in\{0,1\}$ where the data points are drawn independently. The
|
||||
likelihood of the seeing the data under our model is just:
|
||||
In order to define the total likelihood for all possible outcomes from a
|
||||
dataset $\mathcal{D}=\{(y_i,x_i)\}$, with the binary labels
|
||||
$y_i\in\{0,1\}$ and where the data points are drawn independently, we use the so-called Maximum Likelihood Estimation (MLE) principle.
|
||||
We aim thus at maximizing
|
||||
the probability of seeing the observed data. We can then approximate the
|
||||
likelihood in terms of the product of the individual probabilities of a specific outcome $y_i$, that is
|
||||
!bt
|
||||
\begin{align}
|
||||
P(\mathcal{D}|\mathbf{w})& = \prod_{i=1}^n \left[f(\mathbf{x}_i^T\mathbf{w})\right]^{y_i}\left[1-f(\mathbf{x}_i^T\mathbf{w})\right]^{1-y_i}\nonumber \\
|
||||
\end{align}
|
||||
\begin{align*}
|
||||
P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\
|
||||
\end{align*}
|
||||
!et
|
||||
from which we can readily compute the log-likelihood:
|
||||
from which we obtain the log-likelihood and our _cost/loss_ function
|
||||
!bt
|
||||
\begin{equation}
|
||||
l(\mathbf{w}) = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\end{equation}
|
||||
\[
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right].
|
||||
\]
|
||||
!et
|
||||
|
||||
!split
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\theta$
|
||||
===== The cost function rewritten =====
|
||||
|
||||
Reordering the logarithms, we can rewrite the _cost/loss_ function as
|
||||
!bt
|
||||
\[
|
||||
\hat{\mathbf{w}} = \sum_{i=1}^n y_i\log f(\mathbf{x}_i^T\mathbf{w}) + (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].
|
||||
\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
\]
|
||||
!et
|
||||
!split
|
||||
The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\beta$.
|
||||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
||||
!bt
|
||||
\begin{eqnarray}
|
||||
\mathcal{C}(\mathbf{w}) &=& - l(\mathbf{w}) \\
|
||||
&=& \sum_{i=1}^n -y_i\log f(\mathbf{x}_i^T\mathbf{w}) - (1-y_i)\log\left[1-f(\mathbf{x}_i^T\mathbf{w})\right].\nonumber
|
||||
\end{eqnarray}
|
||||
\[
|
||||
\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
\]
|
||||
!et
|
||||
This equation is known in statistics as the \emph{cross entropy}. Finally, we note that just as in linear regression,
|
||||
in practice we usually supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression.
|
||||
This equation is known in statistics as the _cross entropy_. Finally, we note that just as in linear regression,
|
||||
in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression.
|
||||
|
||||
!split
|
||||
===== Minimizing the cross entropy =====
|
||||
|
||||
The cross entropy is a convex function of the weights $\mathbf{w}$ and,
|
||||
therefore, any local minimizer is a global minimizer. Minimizing this
|
||||
cost function leads to the following equation
|
||||
The cross entropy is a convex function of the weights $\hat{\beta}$ and,
|
||||
therefore, any local minimizer is a global minimizer.
|
||||
|
||||
|
||||
Minimizing this
|
||||
cost function with respect to the two parameters $\beta_0$ and $\beta_1$ we obtain
|
||||
|
||||
!bt
|
||||
\begin{equation}
|
||||
\boldsymbol{0}=\boldsymbol{\nabla} \mathcal{C}(\mathbf{w}) = \sum_{i=1}^n\left[f(\mathbf{x}_i^T\mathbf{w})-y_i\right]\mathbf{x}_i,
|
||||
\end{equation}
|
||||
\[
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right),
|
||||
\]
|
||||
!et
|
||||
and
|
||||
!bt
|
||||
\[
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right).
|
||||
\]
|
||||
!et
|
||||
|
||||
where we made use of the logistic function identity $\partial_z f(z) =
|
||||
f(z)[1-f(z)]$. This equation defines a transcendental equation for
|
||||
$\mathbf{w}$, the solution of which, unlike linear regression, cannot
|
||||
be written in a closed form.
|
||||
Here we need gradient descent methods!
|
||||
!split
|
||||
===== A more compact expression =====
|
||||
|
||||
Let us now define a vector $\hat{y}$ with $n$ elements $y_i$, an
|
||||
$n\times p$ matrix $\hat{X}$ which contains the $x_i$ values and a
|
||||
vector $\hat{p}$ of fitted probabilities $p(y_i\vert x_i,\hat{\beta})$. We can rewrite in a more compact form the first
|
||||
derivative of cost function as
|
||||
|
||||
!bt
|
||||
\[
|
||||
\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right).
|
||||
\]
|
||||
!et
|
||||
|
||||
If we in addition define a diagonal matrix $\hat{W}$ with elements
|
||||
$p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta})$, we can obtain a compact expression of the second derivative as
|
||||
|
||||
!bt
|
||||
\[
|
||||
\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}.
|
||||
\]
|
||||
!et
|
||||
|
||||
|
||||
!split
|
||||
@@ -212,3 +256,58 @@ plt.plot(X_new, y_proba[:, 0], "b--", label="Not Iris-Virginica")
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== A simple classification problem =====
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
from sklearn import datasets, linear_model
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def generate_data():
|
||||
np.random.seed(0)
|
||||
X, y = datasets.make_moons(200, noise=0.20)
|
||||
return X, y
|
||||
|
||||
|
||||
def visualize(X, y, clf):
|
||||
# plt.scatter(X[:, 0], X[:, 1], s=40, c=y, cmap=plt.cm.Spectral)
|
||||
# plt.show()
|
||||
plot_decision_boundary(lambda x: clf.predict(x), X, y)
|
||||
plt.title("Logistic Regression")
|
||||
|
||||
|
||||
def plot_decision_boundary(pred_func, X, y):
|
||||
# Set min and max values and give it some padding
|
||||
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
|
||||
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
|
||||
h = 0.01
|
||||
# Generate a grid of points with distance h between them
|
||||
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
||||
# Predict the function value for the whole gid
|
||||
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
||||
Z = Z.reshape(xx.shape)
|
||||
# Plot the contour and training examples
|
||||
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
||||
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)
|
||||
plt.show()
|
||||
|
||||
|
||||
def classify(X, y):
|
||||
clf = linear_model.LogisticRegressionCV()
|
||||
clf.fit(X, y)
|
||||
return clf
|
||||
|
||||
|
||||
def main():
|
||||
X, y = generate_data()
|
||||
# visualize(X, y)
|
||||
clf = classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user