Morten Hjorth-Jensen
1989 lines
125 KiB
HTML
1989 lines
125 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week40.do.txt --pygments_html_style=default --html_style=bloodish --html_links_in_new_window --html_output=week40 --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 40: Gradient descent methods (continued) and start Neural networks">
|
|
<title>Week 40: Gradient descent methods (continued) and start Neural networks</title>
|
|
<style type="text/css">
|
|
/* bloodish style */
|
|
body {
|
|
font-family: Helvetica, Verdana, Arial, Sans-serif;
|
|
color: #404040;
|
|
background: #ffffff;
|
|
}
|
|
h1 { font-size: 1.8em; color: #8A0808; }
|
|
h2 { font-size: 1.6em; color: #8A0808; }
|
|
h3 { font-size: 1.4em; color: #8A0808; }
|
|
h4 { font-size: 1.2em; color: #8A0808; }
|
|
a { color: #8A0808; text-decoration:none; }
|
|
tt { font-family: "Courier New", Courier; }
|
|
p { text-indent: 0px; }
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-style: normal; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa; }div.highlight {
|
|
border: 1px solid #cfcfcf;
|
|
border-radius: 2px;
|
|
line-height: 1.21429em;
|
|
}
|
|
div.cell {
|
|
width: 100%;
|
|
padding: 5px 5px 5px 0;
|
|
margin: 0;
|
|
outline: none;
|
|
}
|
|
div.input {
|
|
page-break-inside: avoid;
|
|
box-orient: horizontal;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: row;
|
|
align-items: stretch;
|
|
}
|
|
div.inner_cell {
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
box-flex: 1;
|
|
flex: 1;
|
|
}
|
|
div.input_area {
|
|
border: 1px solid #cfcfcf;
|
|
border-radius: 4px;
|
|
background: #f7f7f7;
|
|
line-height: 1.21429em;
|
|
}
|
|
div.input_area > div.highlight {
|
|
margin: .4em;
|
|
border: none;
|
|
padding: 0;
|
|
background-color: transparent;
|
|
}
|
|
div.output_wrapper {
|
|
position: relative;
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
}
|
|
.output {
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
}
|
|
div.output_area {
|
|
padding: 0;
|
|
page-break-inside: avoid;
|
|
box-orient: horizontal;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: row;
|
|
align-items: stretch;
|
|
}
|
|
div.output_subarea {
|
|
padding: .4em .4em 0 .4em;
|
|
box-flex: 1;
|
|
flex: 1;
|
|
}
|
|
div.output_text {
|
|
text-align: left;
|
|
color: #000;
|
|
line-height: 1.21429em;
|
|
}
|
|
.alert-text-small { font-size: 80%; }
|
|
.alert-text-large { font-size: 130%; }
|
|
.alert-text-normal { font-size: 90%; }
|
|
.alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:1px solid #bababa;
|
|
border-radius: 4px;
|
|
-webkit-border-radius: 4px;
|
|
-moz-border-radius: 4px;
|
|
color: #555;
|
|
background-color: #f8f8f8;
|
|
background-position: 10px 5px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 55px;
|
|
width: 75%;
|
|
}
|
|
.alert-block {padding-top:14px; padding-bottom:14px}
|
|
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
.alert li {margin-top: 1em}
|
|
.alert-block p+p {margin-top:5px}
|
|
.alert-notice { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_notice.png); }
|
|
.alert-summary { background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_summary.png); }
|
|
.alert-warning { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_warning.png); }
|
|
.alert-question {background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_question.png); }
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
.tab {
|
|
padding-left: 1.5em;
|
|
}
|
|
div.toc p,a {
|
|
line-height: 1.3;
|
|
margin-top: 1.1;
|
|
margin-bottom: 1.1;
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Lecture Monday September 29, 2025',
|
|
2,
|
|
None,
|
|
'lecture-monday-september-29-2025'),
|
|
('Suggested readings and videos',
|
|
2,
|
|
None,
|
|
'suggested-readings-and-videos'),
|
|
('Lab sessions Tuesday and Wednesday',
|
|
2,
|
|
None,
|
|
'lab-sessions-tuesday-and-wednesday'),
|
|
('Logistic Regression, from last week',
|
|
2,
|
|
None,
|
|
'logistic-regression-from-last-week'),
|
|
('Classification problems', 2, None, 'classification-problems'),
|
|
('Optimization and Deep learning',
|
|
2,
|
|
None,
|
|
'optimization-and-deep-learning'),
|
|
('Basics', 2, None, 'basics'),
|
|
('Two parameters', 2, None, 'two-parameters'),
|
|
('Maximum likelihood', 2, None, 'maximum-likelihood'),
|
|
('The cost function rewritten',
|
|
2,
|
|
None,
|
|
'the-cost-function-rewritten'),
|
|
('Minimizing the cross entropy',
|
|
2,
|
|
None,
|
|
'minimizing-the-cross-entropy'),
|
|
('A more compact expression',
|
|
2,
|
|
None,
|
|
'a-more-compact-expression'),
|
|
('Extending to more predictors',
|
|
2,
|
|
None,
|
|
'extending-to-more-predictors'),
|
|
('Including more classes', 2, None, 'including-more-classes'),
|
|
('More classes', 2, None, 'more-classes'),
|
|
('Optimization, the central part of any Machine Learning '
|
|
'algortithm',
|
|
2,
|
|
None,
|
|
'optimization-the-central-part-of-any-machine-learning-algortithm'),
|
|
('Revisiting our Logistic Regression case',
|
|
2,
|
|
None,
|
|
'revisiting-our-logistic-regression-case'),
|
|
('The equations to solve', 2, None, 'the-equations-to-solve'),
|
|
("Solving using Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'solving-using-newton-raphson-s-method'),
|
|
('Example code for Logistic Regression',
|
|
2,
|
|
None,
|
|
'example-code-for-logistic-regression'),
|
|
('Synthetic data generation',
|
|
3,
|
|
None,
|
|
'synthetic-data-generation'),
|
|
('Using _Scikit-learn_', 2, None, 'using-scikit-learn'),
|
|
('Using the correlation matrix',
|
|
2,
|
|
None,
|
|
'using-the-correlation-matrix'),
|
|
('Discussing the correlation data',
|
|
2,
|
|
None,
|
|
'discussing-the-correlation-data'),
|
|
('Other measures in classification studies',
|
|
2,
|
|
None,
|
|
'other-measures-in-classification-studies'),
|
|
('Introduction to Neural networks',
|
|
2,
|
|
None,
|
|
'introduction-to-neural-networks'),
|
|
('Artificial neurons', 2, None, 'artificial-neurons'),
|
|
('Neural network types', 2, None, 'neural-network-types'),
|
|
('Feed-forward neural networks',
|
|
2,
|
|
None,
|
|
'feed-forward-neural-networks'),
|
|
('Convolutional Neural Network',
|
|
2,
|
|
None,
|
|
'convolutional-neural-network'),
|
|
('Recurrent neural networks',
|
|
2,
|
|
None,
|
|
'recurrent-neural-networks'),
|
|
('Other types of networks', 2, None, 'other-types-of-networks'),
|
|
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
|
('Why multilayer perceptrons?',
|
|
2,
|
|
None,
|
|
'why-multilayer-perceptrons'),
|
|
('Illustration of a single perceptron model and a '
|
|
'multi-perceptron model',
|
|
2,
|
|
None,
|
|
'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
|
|
('Examples of XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'examples-of-xor-or-and-and-gates'),
|
|
('Does Logistic Regression do a better Job?',
|
|
2,
|
|
None,
|
|
'does-logistic-regression-do-a-better-job'),
|
|
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
|
('Matrix-vector notation and activation',
|
|
3,
|
|
None,
|
|
'matrix-vector-notation-and-activation'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 3, None, 'relevance')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "AMS" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
<center>
|
|
<h1>Week 40: Gradient descent methods (continued) and start Neural networks</h1>
|
|
</center> <!-- document title -->
|
|
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b>
|
|
</center>
|
|
<!-- institution -->
|
|
<center>
|
|
<b>Department of Physics, University of Oslo, Norway</b>
|
|
</center>
|
|
<br>
|
|
<center>
|
|
<h4>September 29-October 3, 2025</h4>
|
|
</center> <!-- date -->
|
|
<br>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="lecture-monday-september-29-2025">Lecture Monday September 29, 2025 </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b></b>
|
|
<p>
|
|
<ol>
|
|
<li> Logistic regression and gradient descent, examples on how to code
|
|
<!-- o Automatic differentiation and gradient descent, examples using Logistic regression --></li>
|
|
<li> Start with the basics of Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model
|
|
<!-- o <a href="https://youtu.be/jdJoOrCIdII" target="_blank">Video of lecture</a> -->
|
|
<!-- o Whiteboard notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember30.pdf" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember30.pdf</tt></a> --></li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="suggested-readings-and-videos">Suggested readings and videos </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Readings and Videos:</b>
|
|
<p>
|
|
<ol>
|
|
<li> The lecture notes for week 40 (these notes)
|
|
<!-- o For a good discussion on gradient methods, we would like to recommend Goodfellow et al section 4.3-4.5 and# sections 8.3-8.6. We will come back to the latter chapter in our discussion of Neural networks as well. --></li>
|
|
<li> For neural networks we recommend Goodfellow et al chapter 6 and Raschka et al chapter 2 (contains also material about gradient descent) and chapter 11 (we will use this next week)
|
|
<!-- o Video on gradient descent at <a href="https://www.youtube.com/watch?v=sDv4f4s2SB8" target="_blank"><tt>https://www.youtube.com/watch?v=sDv4f4s2SB8</tt></a> -->
|
|
<!-- o Video on automatic differentiation at <a href="https://www.youtube.com/watch?v=wG_nF1awSSY" target="_blank"><tt>https://www.youtube.com/watch?v=wG_nF1awSSY</tt></a> --></li>
|
|
<li> Neural Networks demystified at <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_blank"><tt>https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs</tt></a></li>
|
|
<li> Building Neural Networks from scratch at URL:https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex"</li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="lab-sessions-tuesday-and-wednesday">Lab sessions Tuesday and Wednesday </h2>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Material for the active learning sessions on Tuesday and Wednesday</b>
|
|
<p>
|
|
<ul>
|
|
<li> Work on project 1 and discussions on how to structure your report</li>
|
|
<li> No weekly exercises for week 40, project work only</li>
|
|
<li> Video on how to write scientific reports recorded during one of the lab sessions at <a href="https://youtu.be/tVW1ZDmZnwM" target="_blank"><tt>https://youtu.be/tVW1ZDmZnwM</tt></a></li>
|
|
<li> A general guideline can be found at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md</tt></a>.</li>
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
<!-- !split -->
|
|
<h2 id="logistic-regression-from-last-week">Logistic Regression, from last week </h2>
|
|
|
|
<p>In linear regression our main interest was centered on learning the
|
|
coefficients of a functional fit (say a polynomial) in order to be
|
|
able to predict the response of a continuous variable on some unseen
|
|
data. The fit to the continuous variable \( y_i \) is based on some
|
|
independent variables \( \boldsymbol{x}_i \). Linear regression resulted in
|
|
analytical expressions for standard ordinary Least Squares or Ridge
|
|
regression (in terms of matrices to invert) for several quantities,
|
|
ranging from the variance and thereby the confidence intervals of the
|
|
parameters \( \boldsymbol{\theta} \) to the mean squared error. If we can invert
|
|
the product of the design matrices, linear regression gives then a
|
|
simple recipe for fitting our data.
|
|
</p>
|
|
|
|
<!-- !split -->
|
|
<h2 id="classification-problems">Classification problems </h2>
|
|
|
|
<p>Classification problems, however, are concerned with outcomes taking
|
|
the form of discrete variables (i.e. categories). We may for example,
|
|
on the basis of DNA sequencing for a number of patients, like to find
|
|
out which mutations are important for a certain disease; or based on
|
|
scans of various patients' brains, figure out if there is a tumor or
|
|
not; or given a specific physical system, we'd like to identify its
|
|
state, say whether it is an ordered or disordered system (typical
|
|
situation in solid state physics); or classify the status of a
|
|
patient, whether she/he has a stroke or not and many other similar
|
|
situations.
|
|
</p>
|
|
|
|
<p>The most common situation we encounter when we apply logistic
|
|
regression is that of two possible outcomes, normally denoted as a
|
|
binary outcome, true or false, positive or negative, success or
|
|
failure etc.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="optimization-and-deep-learning">Optimization and Deep learning </h2>
|
|
|
|
<p>Logistic regression will also serve as our stepping stone towards
|
|
neural network algorithms and supervised deep learning. For logistic
|
|
learning, the minimization of the cost function leads to a non-linear
|
|
equation in the parameters \( \boldsymbol{\theta} \). The optimization of the
|
|
problem calls therefore for minimization algorithms.
|
|
</p>
|
|
|
|
<p>As we have discussed earlier, this forms the
|
|
bottle neck of all machine learning algorithms, namely how to find
|
|
reliable minima of a multi-variable function. This leads us to the
|
|
family of gradient descent methods. The latter are the working horses
|
|
of basically all modern machine learning algorithms.
|
|
</p>
|
|
|
|
<p>We note also that many of the topics discussed here on logistic
|
|
regression are also commonly used in modern supervised Deep Learning
|
|
models, as we will see later.
|
|
</p>
|
|
|
|
<!-- !split -->
|
|
<h2 id="basics">Basics </h2>
|
|
|
|
<p>We consider the case where the outputs/targets, also called the
|
|
responses or the outcomes, \( y_i \) are discrete and only take values
|
|
from \( k=0,\dots,K-1 \) (i.e. \( K \) classes).
|
|
</p>
|
|
|
|
<p>The goal is to predict the
|
|
output classes from the design matrix \( \boldsymbol{X}\in\mathbb{R}^{n\times p} \)
|
|
made of \( n \) samples, each of which carries \( p \) features or predictors. The
|
|
primary goal is to identify the classes to which new unseen samples
|
|
belong.
|
|
</p>
|
|
|
|
<p>Last week we specialized to the case of two classes only, with outputs
|
|
\( y_i=0 \) and \( y_i=1 \). Our outcomes could represent the status of a
|
|
credit card user that could default or not on her/his credit card
|
|
debt. That is
|
|
</p>
|
|
|
|
$$
|
|
y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}.
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="two-parameters">Two parameters </h2>
|
|
|
|
<p>We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \theta \) in our fitting of the Sigmoid function, that is we define probabilities </p>
|
|
$$
|
|
\begin{align*}
|
|
p(y_i=1|x_i,\boldsymbol{\theta}) &= \frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}},\nonumber\\
|
|
p(y_i=0|x_i,\boldsymbol{\theta}) &= 1 - p(y_i=1|x_i,\boldsymbol{\theta}),
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>where \( \boldsymbol{\theta} \) are the weights we wish to extract from data, in our case \( \theta_0 \) and \( \theta_1 \). </p>
|
|
|
|
<p>Note that we used</p>
|
|
$$
|
|
p(y_i=0\vert x_i, \boldsymbol{\theta}) = 1-p(y_i=1\vert x_i, \boldsymbol{\theta}).
|
|
$$
|
|
|
|
|
|
<!-- !split -->
|
|
<h2 id="maximum-likelihood">Maximum likelihood </h2>
|
|
|
|
<p>In order to define the total likelihood for all possible outcomes from a
|
|
dataset \( \mathcal{D}=\{(y_i,x_i)\} \), with the binary labels
|
|
\( y_i\in\{0,1\} \) and where the data points are drawn independently, we use the so-called <a href="https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" target="_blank">Maximum Likelihood Estimation</a> (MLE) principle.
|
|
We aim thus at maximizing
|
|
the probability of seeing the observed data. We can then approximate the
|
|
likelihood in terms of the product of the individual probabilities of a specific outcome \( y_i \), that is
|
|
</p>
|
|
$$
|
|
\begin{align*}
|
|
P(\mathcal{D}|\boldsymbol{\theta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\boldsymbol{\theta})\right]^{y_i}\left[1-p(y_i=1|x_i,\boldsymbol{\theta}))\right]^{1-y_i}\nonumber \\
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>from which we obtain the log-likelihood and our <b>cost/loss</b> function</p>
|
|
$$
|
|
\mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\boldsymbol{\theta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\boldsymbol{\theta}))\right]\right).
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-cost-function-rewritten">The cost function rewritten </h2>
|
|
|
|
<p>Reordering the logarithms, we can rewrite the <b>cost/loss</b> function as</p>
|
|
$$
|
|
\mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^n \left(y_i(\theta_0+\theta_1x_i) -\log{(1+\exp{(\theta_0+\theta_1x_i)})}\right).
|
|
$$
|
|
|
|
<p>The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to \( \theta \).
|
|
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that
|
|
</p>
|
|
$$
|
|
\mathcal{C}(\boldsymbol{\theta})=-\sum_{i=1}^n \left(y_i(\theta_0+\theta_1x_i) -\log{(1+\exp{(\theta_0+\theta_1x_i)})}\right).
|
|
$$
|
|
|
|
<p>This equation is known in statistics as the <b>cross entropy</b>. Finally, we note that just as in linear regression,
|
|
in practice we often supplement the cross-entropy with additional regularization terms, usually \( L_1 \) and \( L_2 \) regularization as we did for Ridge and Lasso regression.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="minimizing-the-cross-entropy">Minimizing the cross entropy </h2>
|
|
|
|
<p>The cross entropy is a convex function of the weights \( \boldsymbol{\theta} \) and,
|
|
therefore, any local minimizer is a global minimizer.
|
|
</p>
|
|
|
|
<p>Minimizing this
|
|
cost function with respect to the two parameters \( \theta_0 \) and \( \theta_1 \) we obtain
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \theta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}}\right),
|
|
$$
|
|
|
|
<p>and </p>
|
|
$$
|
|
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \theta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}}\right).
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="a-more-compact-expression">A more compact expression </h2>
|
|
|
|
<p>Let us now define a vector \( \boldsymbol{y} \) with \( n \) elements \( y_i \), an
|
|
\( n\times p \) matrix \( \boldsymbol{X} \) which contains the \( x_i \) values and a
|
|
vector \( \boldsymbol{p} \) of fitted probabilities \( p(y_i\vert x_i,\boldsymbol{\theta}) \). We can rewrite in a more compact form the first
|
|
derivative of the cost function as
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}} = -\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{p}\right).
|
|
$$
|
|
|
|
<p>If we in addition define a diagonal matrix \( \boldsymbol{W} \) with elements
|
|
\( p(y_i\vert x_i,\boldsymbol{\theta})(1-p(y_i\vert x_i,\boldsymbol{\theta}) \), we can obtain a compact expression of the second derivative as
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial^2 \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}\partial \boldsymbol{\theta}^T} = \boldsymbol{X}^T\boldsymbol{W}\boldsymbol{X}.
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="extending-to-more-predictors">Extending to more predictors </h2>
|
|
|
|
<p>Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with \( p \) predictors</p>
|
|
$$
|
|
\log{ \frac{p(\boldsymbol{\theta}\boldsymbol{x})}{1-p(\boldsymbol{\theta}\boldsymbol{x})}} = \theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p.
|
|
$$
|
|
|
|
<p>Here we defined \( \boldsymbol{x}=[1,x_1,x_2,\dots,x_p] \) and \( \boldsymbol{\theta}=[\theta_0, \theta_1, \dots, \theta_p] \) leading to</p>
|
|
$$
|
|
p(\boldsymbol{\theta}\boldsymbol{x})=\frac{ \exp{(\theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p)}}{1+\exp{(\theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p)}}.
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="including-more-classes">Including more classes </h2>
|
|
|
|
<p>Till now we have mainly focused on two classes, the so-called binary
|
|
system. Suppose we wish to extend to \( K \) classes. Let us for the sake
|
|
of simplicity assume we have only two predictors. We have then following model
|
|
</p>
|
|
|
|
$$
|
|
\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \theta_{10}+\theta_{11}x_1,
|
|
$$
|
|
|
|
<p>and </p>
|
|
$$
|
|
\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \theta_{20}+\theta_{21}x_1,
|
|
$$
|
|
|
|
<p>and so on till the class \( C=K-1 \) class</p>
|
|
$$
|
|
\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \theta_{(K-1)0}+\theta_{(K-1)1}x_1,
|
|
$$
|
|
|
|
<p>and the model is specified in term of \( K-1 \) so-called log-odds or
|
|
<b>logit</b> transformations.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="more-classes">More classes </h2>
|
|
|
|
<p>In our discussion of neural networks we will encounter the above again
|
|
in terms of a slightly modified function, the so-called <b>Softmax</b> function.
|
|
</p>
|
|
|
|
<p>The softmax function is used in various multiclass classification
|
|
methods, such as multinomial logistic regression (also known as
|
|
softmax regression), multiclass linear discriminant analysis, naive
|
|
Bayes classifiers, and artificial neural networks. Specifically, in
|
|
multinomial logistic regression and linear discriminant analysis, the
|
|
input to the function is the result of \( K \) distinct linear functions,
|
|
and the predicted probability for the \( k \)-th class given a sample
|
|
vector \( \boldsymbol{x} \) and a weighting vector \( \boldsymbol{\theta} \) is (with two
|
|
predictors):
|
|
</p>
|
|
|
|
$$
|
|
p(C=k\vert \mathbf {x} )=\frac{\exp{(\theta_{k0}+\theta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\theta_{l0}+\theta_{l1}x_1)}}.
|
|
$$
|
|
|
|
<p>It is easy to extend to more predictors. The final class is </p>
|
|
$$
|
|
p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\theta_{l0}+\theta_{l1}x_1)}},
|
|
$$
|
|
|
|
<p>and they sum to one. Our earlier discussions were all specialized to
|
|
the case with two classes only. It is easy to see from the above that
|
|
what we derived earlier is compatible with these equations.
|
|
</p>
|
|
|
|
<p>To find the optimal parameters we would typically use a gradient
|
|
descent method. Newton's method and gradient descent methods are
|
|
discussed in the material on <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_blank">optimization
|
|
methods</a>.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="optimization-the-central-part-of-any-machine-learning-algortithm">Optimization, the central part of any Machine Learning algortithm </h2>
|
|
|
|
<p>Almost every problem in machine learning and data science starts with
|
|
a dataset \( X \), a model \( g(\theta) \), which is a function of the
|
|
parameters \( \theta \) and a cost function \( C(X, g(\theta)) \) that allows
|
|
us to judge how well the model \( g(\theta) \) explains the observations
|
|
\( X \). The model is fit by finding the values of \( \theta \) that minimize
|
|
the cost function. Ideally we would be able to solve for \( \theta \)
|
|
analytically, however this is not possible in general and we must use
|
|
some approximative/numerical method to compute the minimum.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="revisiting-our-logistic-regression-case">Revisiting our Logistic Regression case </h2>
|
|
|
|
<p>In our discussion on Logistic Regression we studied the
|
|
case of
|
|
two classes, with \( y_i \) either
|
|
\( 0 \) or \( 1 \). Furthermore we assumed also that we have only two
|
|
parameters \( \theta \) in our fitting, that is we
|
|
defined probabilities
|
|
</p>
|
|
|
|
$$
|
|
\begin{align*}
|
|
p(y_i=1|x_i,\boldsymbol{\theta}) &= \frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}},\nonumber\\
|
|
p(y_i=0|x_i,\boldsymbol{\theta}) &= 1 - p(y_i=1|x_i,\boldsymbol{\theta}),
|
|
\end{align*}
|
|
$$
|
|
|
|
<p>where \( \boldsymbol{\theta} \) are the weights we wish to extract from data, in our case \( \theta_0 \) and \( \theta_1 \). </p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-equations-to-solve">The equations to solve </h2>
|
|
|
|
<p>Our compact equations used a definition of a vector \( \boldsymbol{y} \) with \( n \)
|
|
elements \( y_i \), an \( n\times p \) matrix \( \boldsymbol{X} \) which contains the
|
|
\( x_i \) values and a vector \( \boldsymbol{p} \) of fitted probabilities
|
|
\( p(y_i\vert x_i,\boldsymbol{\theta}) \). We rewrote in a more compact form
|
|
the first derivative of the cost function as
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}} = -\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{p}\right).
|
|
$$
|
|
|
|
<p>If we in addition define a diagonal matrix \( \boldsymbol{W} \) with elements
|
|
\( p(y_i\vert x_i,\boldsymbol{\theta})(1-p(y_i\vert x_i,\boldsymbol{\theta}) \), we can obtain a compact expression of the second derivative as
|
|
</p>
|
|
|
|
$$
|
|
\frac{\partial^2 \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}\partial \boldsymbol{\theta}^T} = \boldsymbol{X}^T\boldsymbol{W}\boldsymbol{X}.
|
|
$$
|
|
|
|
<p>This defines what is called the Hessian matrix.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="solving-using-newton-raphson-s-method">Solving using Newton-Raphson's method </h2>
|
|
|
|
<p>If we can set up these equations, Newton-Raphson's iterative method is normally the method of choice. It requires however that we can compute in an efficient way the matrices that define the first and second derivatives. </p>
|
|
|
|
<p>Our iterative scheme is then given by</p>
|
|
|
|
$$
|
|
\boldsymbol{\theta}^{\mathrm{new}} = \boldsymbol{\theta}^{\mathrm{old}}-\left(\frac{\partial^2 \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}\partial \boldsymbol{\theta}^T}\right)^{-1}_{\boldsymbol{\theta}^{\mathrm{old}}}\times \left(\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}}\right)_{\boldsymbol{\theta}^{\mathrm{old}}},
|
|
$$
|
|
|
|
<p>or in matrix form as</p>
|
|
|
|
$$
|
|
\boldsymbol{\theta}^{\mathrm{new}} = \boldsymbol{\theta}^{\mathrm{old}}-\left(\boldsymbol{X}^T\boldsymbol{W}\boldsymbol{X} \right)^{-1}\times \left(-\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{p}) \right)_{\boldsymbol{\theta}^{\mathrm{old}}}.
|
|
$$
|
|
|
|
<p>The right-hand side is computed with the old values of \( \theta \). </p>
|
|
|
|
<p>If we can compute these matrices, in particular the Hessian, the above is often the easiest method to implement. </p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="example-code-for-logistic-regression">Example code for Logistic Regression </h2>
|
|
|
|
<p>Here we make a class for Logistic regression. The code uses a simple data set and includes both a binary case and a multiclass case.</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">LogisticRegression</span>:
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Logistic Regression for binary and multiclass classification.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, lr<span style="color: #666666">=0.01</span>, epochs<span style="color: #666666">=1000</span>, fit_intercept<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, verbose<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>):
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>lr <span style="color: #666666">=</span> lr <span style="color: #408080; font-style: italic"># Learning rate for gradient descent</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs <span style="color: #666666">=</span> epochs <span style="color: #408080; font-style: italic"># Number of iterations</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>fit_intercept <span style="color: #666666">=</span> fit_intercept <span style="color: #408080; font-style: italic"># Whether to add intercept (bias)</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>verbose <span style="color: #666666">=</span> verbose <span style="color: #408080; font-style: italic"># Print loss during training if True</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span> <span style="color: #408080; font-style: italic"># Will be determined at fit time</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_add_intercept</span>(<span style="color: #008000">self</span>, X):
|
|
<span style="color: #BA2121; font-style: italic">"""Add intercept term (column of ones) to feature matrix."""</span>
|
|
intercept <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones((X<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], <span style="color: #666666">1</span>))
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>concatenate((intercept, X), axis<span style="color: #666666">=1</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_sigmoid</span>(<span style="color: #008000">self</span>, z):
|
|
<span style="color: #BA2121; font-style: italic">"""Sigmoid function for binary logistic."""</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1</span> <span style="color: #666666">/</span> (<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z))
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_softmax</span>(<span style="color: #008000">self</span>, Z):
|
|
<span style="color: #BA2121; font-style: italic">"""Softmax function for multiclass logistic."""</span>
|
|
exp_Z <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(Z <span style="color: #666666">-</span> np<span style="color: #666666">.</span>max(Z, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>))
|
|
<span style="color: #008000; font-weight: bold">return</span> exp_Z <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_Z, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">fit</span>(<span style="color: #008000">self</span>, X, y):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Train the logistic regression model using gradient descent.</span>
|
|
<span style="color: #BA2121; font-style: italic"> Supports binary (sigmoid) and multiclass (softmax) based on y.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(X)
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y)
|
|
n_samples, n_features <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape
|
|
|
|
<span style="color: #408080; font-style: italic"># Add intercept if needed</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>fit_intercept:
|
|
X <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_add_intercept(X)
|
|
n_features <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Determine classes and mode (binary vs multiclass)</span>
|
|
unique_classes <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unique(y)
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">len</span>(unique_classes) <span style="color: #666666">></span> <span style="color: #666666">2</span>:
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">True</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># ----- Multiclass case -----</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class:
|
|
n_classes <span style="color: #666666">=</span> <span style="color: #008000">len</span>(unique_classes)
|
|
<span style="color: #408080; font-style: italic"># Map original labels to 0...n_classes-1</span>
|
|
class_to_index <span style="color: #666666">=</span> {c: idx <span style="color: #008000; font-weight: bold">for</span> idx, c <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(unique_classes)}
|
|
y_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([class_to_index[c] <span style="color: #008000; font-weight: bold">for</span> c <span style="color: #AA22FF; font-weight: bold">in</span> y])
|
|
<span style="color: #408080; font-style: italic"># Initialize weight matrix (features x classes)</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_features, n_classes))
|
|
|
|
<span style="color: #408080; font-style: italic"># One-hot encode y</span>
|
|
Y_onehot <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_samples, n_classes))
|
|
Y_onehot[np<span style="color: #666666">.</span>arange(n_samples), y_indices] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Gradient descent</span>
|
|
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
|
|
scores <span style="color: #666666">=</span> X<span style="color: #666666">.</span>dot(<span style="color: #008000">self</span><span style="color: #666666">.</span>weights) <span style="color: #408080; font-style: italic"># Linear scores (n_samples x n_classes)</span>
|
|
probs <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_softmax(scores) <span style="color: #408080; font-style: italic"># Probabilities (n_samples x n_classes)</span>
|
|
<span style="color: #408080; font-style: italic"># Compute gradient (features x classes)</span>
|
|
gradient <span style="color: #666666">=</span> (<span style="color: #666666">1</span> <span style="color: #666666">/</span> n_samples) <span style="color: #666666">*</span> X<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(probs <span style="color: #666666">-</span> Y_onehot)
|
|
<span style="color: #408080; font-style: italic"># Update weights</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lr <span style="color: #666666">*</span> gradient
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>verbose <span style="color: #AA22FF; font-weight: bold">and</span> epoch <span style="color: #666666">%</span> <span style="color: #666666">100</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
|
<span style="color: #408080; font-style: italic"># Compute current loss (categorical cross-entropy)</span>
|
|
loss <span style="color: #666666">=</span> <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sum(Y_onehot <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(probs <span style="color: #666666">+</span> <span style="color: #666666">1e-15</span>)) <span style="color: #666666">/</span> n_samples
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"[Epoch </span><span style="color: #BB6688; font-weight: bold">{</span>epoch<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">] Multiclass loss: </span><span style="color: #BB6688; font-weight: bold">{</span>loss<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.4f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># ----- Binary case -----</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
<span style="color: #408080; font-style: italic"># Convert y to 0/1 if not already</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> np<span style="color: #666666">.</span>array_equal(unique_classes, [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>]):
|
|
<span style="color: #408080; font-style: italic"># Map the two classes to 0 and 1</span>
|
|
class0, class1 <span style="color: #666666">=</span> unique_classes
|
|
y_binary <span style="color: #666666">=</span> np<span style="color: #666666">.</span>where(y <span style="color: #666666">==</span> class1, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>)
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
y_binary <span style="color: #666666">=</span> y<span style="color: #666666">.</span>copy()<span style="color: #666666">.</span>astype(<span style="color: #008000">int</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># Initialize weights vector (features,)</span>
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_features)
|
|
|
|
<span style="color: #408080; font-style: italic"># Gradient descent</span>
|
|
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
|
|
linear_model <span style="color: #666666">=</span> X<span style="color: #666666">.</span>dot(<span style="color: #008000">self</span><span style="color: #666666">.</span>weights) <span style="color: #408080; font-style: italic"># (n_samples,)</span>
|
|
probs <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_sigmoid(linear_model) <span style="color: #408080; font-style: italic"># (n_samples,)</span>
|
|
<span style="color: #408080; font-style: italic"># Gradient for binary cross-entropy</span>
|
|
gradient <span style="color: #666666">=</span> (<span style="color: #666666">1</span> <span style="color: #666666">/</span> n_samples) <span style="color: #666666">*</span> X<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(probs <span style="color: #666666">-</span> y_binary)
|
|
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lr <span style="color: #666666">*</span> gradient
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>verbose <span style="color: #AA22FF; font-weight: bold">and</span> epoch <span style="color: #666666">%</span> <span style="color: #666666">100</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
|
<span style="color: #408080; font-style: italic"># Compute binary cross-entropy loss</span>
|
|
loss <span style="color: #666666">=</span> <span style="color: #666666">-</span>np<span style="color: #666666">.</span>mean(
|
|
y_binary <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(probs <span style="color: #666666">+</span> <span style="color: #666666">1e-15</span>) <span style="color: #666666">+</span>
|
|
(<span style="color: #666666">1</span> <span style="color: #666666">-</span> y_binary) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(<span style="color: #666666">1</span> <span style="color: #666666">-</span> probs <span style="color: #666666">+</span> <span style="color: #666666">1e-15</span>)
|
|
)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"[Epoch </span><span style="color: #BB6688; font-weight: bold">{</span>epoch<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">] Binary loss: </span><span style="color: #BB6688; font-weight: bold">{</span>loss<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.4f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict_prob</span>(<span style="color: #008000">self</span>, X):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Compute probability estimates. Returns a 1D array for binary or</span>
|
|
<span style="color: #BA2121; font-style: italic"> a 2D array (n_samples x n_classes) for multiclass.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(X)
|
|
<span style="color: #408080; font-style: italic"># Add intercept if the model used it</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>fit_intercept:
|
|
X <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_add_intercept(X)
|
|
scores <span style="color: #666666">=</span> X<span style="color: #666666">.</span>dot(<span style="color: #008000">self</span><span style="color: #666666">.</span>weights)
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class:
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_softmax(scores)
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_sigmoid(scores)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(<span style="color: #008000">self</span>, X):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Predict class labels for samples in X.</span>
|
|
<span style="color: #BA2121; font-style: italic"> Returns integer class labels (0,1 for binary, or 0...C-1 for multiclass).</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
probs <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>predict_prob(X)
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>multi_class:
|
|
<span style="color: #408080; font-style: italic"># Choose class with highest probability</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probs, axis<span style="color: #666666">=1</span>)
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
<span style="color: #408080; font-style: italic"># Threshold at 0.5 for binary</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> (probs <span style="color: #666666">>=</span> <span style="color: #666666">0.5</span>)<span style="color: #666666">.</span>astype(<span style="color: #008000">int</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>The class implements the sigmoid and softmax internally. During fit(),
|
|
we check the number of classes: if more than 2, we set
|
|
self.multi_class=True and perform multinomial logistic regression. We
|
|
one-hot encode the target vector and update a weight matrix with
|
|
softmax probabilities. Otherwise, we do standard binary logistic
|
|
regression, converting labels to 0/1 if needed and updating a weight
|
|
vector. In both cases we use batch gradient descent on the
|
|
cross-entropy loss (we add a small epsilon 1e-15 to logs for numerical
|
|
stability). Progress (loss) can be printed if verbose=True.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Evaluation Metrics</span>
|
|
<span style="color: #408080; font-style: italic">#We define helper functions for accuracy and cross-entropy loss. Accuracy is the fraction of correct predictions . For loss, we compute the appropriate cross-entropy:</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_score</span>(y_true, y_pred):
|
|
<span style="color: #BA2121; font-style: italic">"""Accuracy = (# correct predictions) / (total samples)."""</span>
|
|
y_true <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y_true)
|
|
y_pred <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y_pred)
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>mean(y_true <span style="color: #666666">==</span> y_pred)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">binary_cross_entropy</span>(y_true, y_prob):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Binary cross-entropy loss.</span>
|
|
<span style="color: #BA2121; font-style: italic"> y_true: true binary labels (0 or 1), y_prob: predicted probabilities for class 1.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
y_true <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y_true)
|
|
y_prob <span style="color: #666666">=</span> np<span style="color: #666666">.</span>clip(np<span style="color: #666666">.</span>array(y_prob), <span style="color: #666666">1e-15</span>, <span style="color: #666666">1-1e-15</span>)
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">-</span>np<span style="color: #666666">.</span>mean(y_true <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(y_prob) <span style="color: #666666">+</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> y_true) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(<span style="color: #666666">1</span> <span style="color: #666666">-</span> y_prob))
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">categorical_cross_entropy</span>(y_true, y_prob):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Categorical cross-entropy loss for multiclass.</span>
|
|
<span style="color: #BA2121; font-style: italic"> y_true: true labels (0...C-1), y_prob: array of predicted probabilities (n_samples x C).</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
y_true <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y_true, dtype<span style="color: #666666">=</span><span style="color: #008000">int</span>)
|
|
y_prob <span style="color: #666666">=</span> np<span style="color: #666666">.</span>clip(np<span style="color: #666666">.</span>array(y_prob), <span style="color: #666666">1e-15</span>, <span style="color: #666666">1-1e-15</span>)
|
|
<span style="color: #408080; font-style: italic"># One-hot encode true labels</span>
|
|
n_samples, n_classes <span style="color: #666666">=</span> y_prob<span style="color: #666666">.</span>shape
|
|
one_hot <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros_like(y_prob)
|
|
one_hot[np<span style="color: #666666">.</span>arange(n_samples), y_true] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
<span style="color: #408080; font-style: italic"># Compute cross-entropy</span>
|
|
loss_vec <span style="color: #666666">=</span> <span style="color: #666666">-</span>np<span style="color: #666666">.</span>sum(one_hot <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(y_prob), axis<span style="color: #666666">=1</span>)
|
|
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>mean(loss_vec)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<h3 id="synthetic-data-generation">Synthetic data generation </h3>
|
|
|
|
<p>Binary classification data: Create two Gaussian clusters in 2D. For example, class 0 around mean [-2,-2] and class 1 around [2,2].
|
|
Multiclass data: Create several Gaussian clusters (one per class) spread out in feature space.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_binary_data</span>(n_samples<span style="color: #666666">=100</span>, n_features<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Generate synthetic binary classification data.</span>
|
|
<span style="color: #BA2121; font-style: italic"> Returns (X, y) where X is (n_samples x n_features), y in {0,1}.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
rng <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>RandomState(random_state)
|
|
<span style="color: #408080; font-style: italic"># Half samples for class 0, half for class 1</span>
|
|
n0 <span style="color: #666666">=</span> n_samples <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
|
n1 <span style="color: #666666">=</span> n_samples <span style="color: #666666">-</span> n0
|
|
<span style="color: #408080; font-style: italic"># Class 0 around mean -2, class 1 around +2</span>
|
|
mean0 <span style="color: #666666">=</span> <span style="color: #666666">-2</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>ones(n_features)
|
|
mean1 <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>ones(n_features)
|
|
X0 <span style="color: #666666">=</span> rng<span style="color: #666666">.</span>randn(n0, n_features) <span style="color: #666666">+</span> mean0
|
|
X1 <span style="color: #666666">=</span> rng<span style="color: #666666">.</span>randn(n1, n_features) <span style="color: #666666">+</span> mean1
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>vstack((X0, X1))
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([<span style="color: #666666">0</span>]<span style="color: #666666">*</span>n0 <span style="color: #666666">+</span> [<span style="color: #666666">1</span>]<span style="color: #666666">*</span>n1)
|
|
<span style="color: #008000; font-weight: bold">return</span> X, y
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_multiclass_data</span>(n_samples<span style="color: #666666">=150</span>, n_features<span style="color: #666666">=2</span>, n_classes<span style="color: #666666">=3</span>, random_state<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>):
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic"> Generate synthetic multiclass data with n_classes Gaussian clusters.</span>
|
|
<span style="color: #BA2121; font-style: italic"> """</span>
|
|
rng <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>RandomState(random_state)
|
|
X <span style="color: #666666">=</span> []
|
|
y <span style="color: #666666">=</span> []
|
|
samples_per_class <span style="color: #666666">=</span> n_samples <span style="color: #666666">//</span> n_classes
|
|
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">cls</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_classes):
|
|
<span style="color: #408080; font-style: italic"># Random cluster center for each class</span>
|
|
center <span style="color: #666666">=</span> rng<span style="color: #666666">.</span>uniform(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, size<span style="color: #666666">=</span>n_features)
|
|
Xi <span style="color: #666666">=</span> rng<span style="color: #666666">.</span>randn(samples_per_class, n_features) <span style="color: #666666">+</span> center
|
|
yi <span style="color: #666666">=</span> [<span style="color: #008000">cls</span>] <span style="color: #666666">*</span> samples_per_class
|
|
X<span style="color: #666666">.</span>append(Xi)
|
|
y<span style="color: #666666">.</span>extend(yi)
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>vstack(X)
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(y)
|
|
<span style="color: #008000; font-weight: bold">return</span> X, y
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># Generate and test on binary data</span>
|
|
X_bin, y_bin <span style="color: #666666">=</span> generate_binary_data(n_samples<span style="color: #666666">=200</span>, n_features<span style="color: #666666">=2</span>, random_state<span style="color: #666666">=42</span>)
|
|
model_bin <span style="color: #666666">=</span> LogisticRegression(lr<span style="color: #666666">=0.1</span>, epochs<span style="color: #666666">=1000</span>)
|
|
model_bin<span style="color: #666666">.</span>fit(X_bin, y_bin)
|
|
y_prob_bin <span style="color: #666666">=</span> model_bin<span style="color: #666666">.</span>predict_prob(X_bin) <span style="color: #408080; font-style: italic"># probabilities for class 1</span>
|
|
y_pred_bin <span style="color: #666666">=</span> model_bin<span style="color: #666666">.</span>predict(X_bin) <span style="color: #408080; font-style: italic"># predicted classes 0 or 1</span>
|
|
|
|
acc_bin <span style="color: #666666">=</span> accuracy_score(y_bin, y_pred_bin)
|
|
loss_bin <span style="color: #666666">=</span> binary_cross_entropy(y_bin, y_prob_bin)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Binary Classification - Accuracy: </span><span style="color: #BB6688; font-weight: bold">{</span>acc_bin<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.2f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">, Cross-Entropy Loss: </span><span style="color: #BB6688; font-weight: bold">{</span>loss_bin<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.2f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #408080; font-style: italic">#For multiclass:</span>
|
|
<span style="color: #408080; font-style: italic"># Generate and test on multiclass data</span>
|
|
X_multi, y_multi <span style="color: #666666">=</span> generate_multiclass_data(n_samples<span style="color: #666666">=300</span>, n_features<span style="color: #666666">=2</span>, n_classes<span style="color: #666666">=3</span>, random_state<span style="color: #666666">=1</span>)
|
|
model_multi <span style="color: #666666">=</span> LogisticRegression(lr<span style="color: #666666">=0.1</span>, epochs<span style="color: #666666">=1000</span>)
|
|
model_multi<span style="color: #666666">.</span>fit(X_multi, y_multi)
|
|
y_prob_multi <span style="color: #666666">=</span> model_multi<span style="color: #666666">.</span>predict_prob(X_multi) <span style="color: #408080; font-style: italic"># (n_samples x 3) probabilities</span>
|
|
y_pred_multi <span style="color: #666666">=</span> model_multi<span style="color: #666666">.</span>predict(X_multi) <span style="color: #408080; font-style: italic"># predicted labels 0,1,2</span>
|
|
|
|
acc_multi <span style="color: #666666">=</span> accuracy_score(y_multi, y_pred_multi)
|
|
loss_multi <span style="color: #666666">=</span> categorical_cross_entropy(y_multi, y_prob_multi)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Multiclass Classification - Accuracy: </span><span style="color: #BB6688; font-weight: bold">{</span>acc_multi<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.2f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">, Cross-Entropy Loss: </span><span style="color: #BB6688; font-weight: bold">{</span>loss_multi<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.2f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># CSV Export</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">csv</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Export binary results</span>
|
|
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(<span style="color: #BA2121">'binary_results.csv'</span>, mode<span style="color: #666666">=</span><span style="color: #BA2121">'w'</span>, newline<span style="color: #666666">=</span><span style="color: #BA2121">''</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
|
writer <span style="color: #666666">=</span> csv<span style="color: #666666">.</span>writer(f)
|
|
writer<span style="color: #666666">.</span>writerow([<span style="color: #BA2121">"TrueLabel"</span>, <span style="color: #BA2121">"PredictedLabel"</span>])
|
|
<span style="color: #008000; font-weight: bold">for</span> true, pred <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(y_bin, y_pred_bin):
|
|
writer<span style="color: #666666">.</span>writerow([true, pred])
|
|
|
|
<span style="color: #408080; font-style: italic"># Export multiclass results</span>
|
|
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(<span style="color: #BA2121">'multiclass_results.csv'</span>, mode<span style="color: #666666">=</span><span style="color: #BA2121">'w'</span>, newline<span style="color: #666666">=</span><span style="color: #BA2121">''</span>) <span style="color: #008000; font-weight: bold">as</span> f:
|
|
writer <span style="color: #666666">=</span> csv<span style="color: #666666">.</span>writer(f)
|
|
writer<span style="color: #666666">.</span>writerow([<span style="color: #BA2121">"TrueLabel"</span>, <span style="color: #BA2121">"PredictedLabel"</span>])
|
|
<span style="color: #008000; font-weight: bold">for</span> true, pred <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(y_multi, y_pred_multi):
|
|
writer<span style="color: #666666">.</span>writerow([true, pred])
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="using-scikit-learn">Using <b>Scikit-learn</b> </h2>
|
|
|
|
<p>We show here how we can use a logistic regression case on a data set
|
|
included in _scikit_learn_, the so-called Wisconsin breast cancer data
|
|
using Logistic regression as our algorithm for classification. This is
|
|
a widely studied data set and can easily be included in demonstrations
|
|
of classification problems.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
|
|
|
<span style="color: #408080; font-style: italic"># Load the data</span>
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
|
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
|
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
|
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
|
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="using-the-correlation-matrix">Using the correlation matrix </h2>
|
|
|
|
<p>In addition to the above scores, we could also study the covariance (and the correlation matrix).
|
|
We use <b>Pandas</b> to compute the correlation matrix.
|
|
</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
|
<span style="color: #408080; font-style: italic"># Making a data frame</span>
|
|
cancerpd <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(cancer<span style="color: #666666">.</span>data, columns<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names)
|
|
|
|
fig, axes <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(<span style="color: #666666">15</span>,<span style="color: #666666">2</span>,figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>,<span style="color: #666666">20</span>))
|
|
malignant <span style="color: #666666">=</span> cancer<span style="color: #666666">.</span>data[cancer<span style="color: #666666">.</span>target <span style="color: #666666">==</span> <span style="color: #666666">0</span>]
|
|
benign <span style="color: #666666">=</span> cancer<span style="color: #666666">.</span>data[cancer<span style="color: #666666">.</span>target <span style="color: #666666">==</span> <span style="color: #666666">1</span>]
|
|
ax <span style="color: #666666">=</span> axes<span style="color: #666666">.</span>ravel()
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">30</span>):
|
|
_, bins <span style="color: #666666">=</span> np<span style="color: #666666">.</span>histogram(cancer<span style="color: #666666">.</span>data[:,i], bins <span style="color: #666666">=50</span>)
|
|
ax[i]<span style="color: #666666">.</span>hist(malignant[:,i], bins <span style="color: #666666">=</span> bins, alpha <span style="color: #666666">=</span> <span style="color: #666666">0.5</span>)
|
|
ax[i]<span style="color: #666666">.</span>hist(benign[:,i], bins <span style="color: #666666">=</span> bins, alpha <span style="color: #666666">=</span> <span style="color: #666666">0.5</span>)
|
|
ax[i]<span style="color: #666666">.</span>set_title(cancer<span style="color: #666666">.</span>feature_names[i])
|
|
ax[i]<span style="color: #666666">.</span>set_yticks(())
|
|
ax[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"Feature magnitude"</span>)
|
|
ax[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"Frequency"</span>)
|
|
ax[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>legend([<span style="color: #BA2121">"Malignant"</span>, <span style="color: #BA2121">"Benign"</span>], loc <span style="color: #666666">=</span><span style="color: #BA2121">"best"</span>)
|
|
fig<span style="color: #666666">.</span>tight_layout()
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
|
correlation_matrix <span style="color: #666666">=</span> cancerpd<span style="color: #666666">.</span>corr()<span style="color: #666666">.</span>round(<span style="color: #666666">1</span>)
|
|
<span style="color: #408080; font-style: italic"># use the heatmap function from seaborn to plot the correlation matrix</span>
|
|
<span style="color: #408080; font-style: italic"># annot = True to print the values inside the square</span>
|
|
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">15</span>,<span style="color: #666666">8</span>))
|
|
sns<span style="color: #666666">.</span>heatmap(data<span style="color: #666666">=</span>correlation_matrix, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="discussing-the-correlation-data">Discussing the correlation data </h2>
|
|
|
|
<p>In the above example we note two things. In the first plot we display
|
|
the overlap of benign and malignant tumors as functions of the various
|
|
features in the Wisconsin data set. We see that for
|
|
some of the features we can distinguish clearly the benign and
|
|
malignant cases while for other features we cannot. This can point to
|
|
us which features may be of greater interest when we wish to classify
|
|
a benign or not benign tumour.
|
|
</p>
|
|
|
|
<p>In the second figure we have computed the so-called correlation
|
|
matrix, which in our case with thirty features becomes a \( 30\times 30 \)
|
|
matrix.
|
|
</p>
|
|
|
|
<p>We constructed this matrix using <b>pandas</b> via the statements</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;">cancerpd <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(cancer<span style="color: #666666">.</span>data, columns<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>and then</p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;">correlation_matrix <span style="color: #666666">=</span> cancerpd<span style="color: #666666">.</span>corr()<span style="color: #666666">.</span>round(<span style="color: #666666">1</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Diagonalizing this matrix we can in turn say something about which
|
|
features are of relevance and which are not. This leads us to
|
|
the classical Principal Component Analysis (PCA) theorem with
|
|
applications. This will be discussed later this semester.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="other-measures-in-classification-studies">Other measures in classification studies </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
|
|
|
<span style="color: #408080; font-style: italic"># Load the data</span>
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
|
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
|
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
|
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
|
logreg<span style="color: #666666">.</span>fit(X_train, y_train)
|
|
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
|
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
|
accuracy <span style="color: #666666">=</span> cross_validate(logreg,X_test,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
|
<span style="color: #008000">print</span>(accuracy)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test,y_test)))
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
y_pred <span style="color: #666666">=</span> logreg<span style="color: #666666">.</span>predict(X_test)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
y_probas <span style="color: #666666">=</span> logreg<span style="color: #666666">.</span>predict_proba(X_test)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="introduction-to-neural-networks">Introduction to Neural networks </h2>
|
|
|
|
<p>Artificial neural networks are computational systems that can learn to
|
|
perform tasks by considering examples, generally without being
|
|
programmed with any task-specific rules. It is supposed to mimic a
|
|
biological system, wherein neurons interact by sending signals in the
|
|
form of mathematical functions between layers. All layers can contain
|
|
an arbitrary number of neurons, and each connection is represented by
|
|
a weight variable.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="artificial-neurons">Artificial neurons </h2>
|
|
|
|
<p>The field of artificial neural networks has a long history of
|
|
development, and is closely connected with the advancement of computer
|
|
science and computers in general. A model of artificial neurons was
|
|
first developed by McCulloch and Pitts in 1943 to study signal
|
|
processing in the brain and has later been refined by others. The
|
|
general idea is to mimic neural networks in the human brain, which is
|
|
composed of billions of neurons that communicate with each other by
|
|
sending electrical signals. Each neuron accumulates its incoming
|
|
signals, which must exceed an activation threshold to yield an
|
|
output. If the threshold is not overcome, the neuron remains inactive,
|
|
i.e. has zero output.
|
|
</p>
|
|
|
|
<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
|
|
\label{artificialNeuron}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
|
|
a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
|
|
</p>
|
|
|
|
<p>Conceptually, it is helpful to divide neural networks into four
|
|
categories:
|
|
</p>
|
|
<ol>
|
|
<li> general purpose neural networks for supervised learning,</li>
|
|
<li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
|
|
<li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
|
|
<li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
|
|
</ol>
|
|
<p>In natural science, DNNs and CNNs have already found numerous
|
|
applications. In statistical physics, they have been applied to detect
|
|
phase transitions in 2D Ising and Potts models, lattice gauge
|
|
theories, and different phases of polymers, or solving the
|
|
Navier-Stokes equation in weather forecasting. Deep learning has also
|
|
found interesting applications in quantum physics. Various quantum
|
|
phase transitions can be detected and studied using DNNs and CNNs,
|
|
topological phases, and even non-equilibrium many-body
|
|
localization. Representing quantum states as DNNs quantum state
|
|
tomography are among some of the impressive achievements to reveal the
|
|
potential of DNNs to facilitate the study of quantum systems.
|
|
</p>
|
|
|
|
<p>In quantum information theory, it has been shown that one can perform
|
|
gate decompositions with the help of neural.
|
|
</p>
|
|
|
|
<p>The applications are not limited to the natural sciences. There is a
|
|
plethora of applications in essentially all disciplines, from the
|
|
humanities to life science and medicine.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="neural-network-types">Neural network types </h2>
|
|
|
|
<p>An artificial neural network (ANN), is a computational model that
|
|
consists of layers of connected neurons, or nodes or units. We will
|
|
refer to these interchangeably as units or nodes, and sometimes as
|
|
neurons.
|
|
</p>
|
|
|
|
<p>It is supposed to mimic a biological nervous system by letting each
|
|
neuron interact with other neurons by sending signals in the form of
|
|
mathematical functions between layers. A wide variety of different
|
|
ANNs have been developed, but most of them consist of an input layer,
|
|
an output layer and eventual layers in-between, called <em>hidden
|
|
layers</em>. All layers can contain an arbitrary number of nodes, and each
|
|
connection between two nodes is associated with a weight variable.
|
|
</p>
|
|
|
|
<p>Neural networks (also called neural nets) are neural-inspired
|
|
nonlinear models for supervised learning. As we will see, neural nets
|
|
can be viewed as natural, more powerful extensions of supervised
|
|
learning methods such as linear and logistic regression and soft-max
|
|
methods we discussed earlier.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="feed-forward-neural-networks">Feed-forward neural networks </h2>
|
|
|
|
<p>The feed-forward neural network (FFNN) was the first and simplest type
|
|
of ANNs that were devised. In this network, the information moves in
|
|
only one direction: forward through the layers.
|
|
</p>
|
|
|
|
<p>Nodes are represented by circles, while the arrows display the
|
|
connections between the nodes, including the direction of information
|
|
flow. Additionally, each arrow corresponds to a weight variable
|
|
(figure to come). We observe that each node in a layer is connected
|
|
to <em>all</em> nodes in the subsequent layer, making this a so-called
|
|
<em>fully-connected</em> FFNN.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="convolutional-neural-network">Convolutional Neural Network </h2>
|
|
|
|
<p>A different variant of FFNNs are <em>convolutional neural networks</em>
|
|
(CNNs), which have a connectivity pattern inspired by the animal
|
|
visual cortex. Individual neurons in the visual cortex only respond to
|
|
stimuli from small sub-regions of the visual field, called a receptive
|
|
field. This makes the neurons well-suited to exploit the strong
|
|
spatially local correlation present in natural images. The response of
|
|
each neuron can be approximated mathematically as a convolution
|
|
operation. (figure to come)
|
|
</p>
|
|
|
|
<p>Convolutional neural networks emulate the behaviour of neurons in the
|
|
visual cortex by enforcing a <em>local</em> connectivity pattern between
|
|
nodes of adjacent layers: Each node in a convolutional layer is
|
|
connected only to a subset of the nodes in the previous layer, in
|
|
contrast to the fully-connected FFNN. Often, CNNs consist of several
|
|
convolutional layers that learn local features of the input, with a
|
|
fully-connected layer at the end, which gathers all the local data and
|
|
produces the outputs. They have wide applications in image and video
|
|
recognition.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="recurrent-neural-networks">Recurrent neural networks </h2>
|
|
|
|
<p>So far we have only mentioned ANNs where information flows in one
|
|
direction: forward. <em>Recurrent neural networks</em> on the other hand,
|
|
have connections between nodes that form directed <em>cycles</em>. This
|
|
creates a form of internal memory which are able to capture
|
|
information on what has been calculated before; the output is
|
|
dependent on the previous computations. Recurrent NNs make use of
|
|
sequential information by performing the same task for every element
|
|
in a sequence, where each element depends on previous elements. An
|
|
example of such information is sentences, making recurrent NNs
|
|
especially well-suited for handwriting and speech recognition.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="other-types-of-networks">Other types of networks </h2>
|
|
|
|
<p>There are many other kinds of ANNs that have been developed. One type
|
|
that is specifically designed for interpolation in multidimensional
|
|
space is the radial basis function (RBF) network. RBFs are typically
|
|
made up of three layers: an input layer, a hidden layer with
|
|
non-linear radial symmetric activation functions and a linear output
|
|
layer (''linear'' here means that each node in the output layer has a
|
|
linear activation function). The layers are normally fully-connected
|
|
and there are no cycles, thus RBFs can be viewed as a type of
|
|
fully-connected FFNN. They are however usually treated as a separate
|
|
type of NN due the unusual activation functions.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="multilayer-perceptrons">Multilayer perceptrons </h2>
|
|
|
|
<p>One uses often so-called fully-connected feed-forward neural networks
|
|
with three or more layers (an input layer, one or more hidden layers
|
|
and an output layer) consisting of neurons that have non-linear
|
|
activation functions.
|
|
</p>
|
|
|
|
<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="why-multilayer-perceptrons">Why multilayer perceptrons? </h2>
|
|
|
|
<p>According to the <em>Universal approximation theorem</em>, a feed-forward
|
|
neural network with just a single hidden layer containing a finite
|
|
number of neurons can approximate a continuous multidimensional
|
|
function to arbitrary accuracy, assuming the activation function for
|
|
the hidden layer is a <b>non-constant, bounded and
|
|
monotonically-increasing continuous function</b>.
|
|
</p>
|
|
|
|
<p>Note that the requirements on the activation function only applies to
|
|
the hidden layer, the output nodes are always assumed to be linear, so
|
|
as to not restrict the range of output values.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model">Illustration of a single perceptron model and a multi-perceptron model </h2>
|
|
|
|
<center> <!-- FIGURE -->
|
|
<hr class="figure">
|
|
<center>
|
|
<p class="caption">Figure 1: In a) we show a single perceptron model while in b) we dispay a network with two hidden layers, an input layer and an output layer. </p>
|
|
</center>
|
|
<p><img src="figures/nns.png" width="600" align="bottom"></p>
|
|
</center>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="examples-of-xor-or-and-and-gates">Examples of XOR, OR and AND gates </h2>
|
|
|
|
<p>Let us first try to fit various gates using standard linear
|
|
regression. The gates we are thinking of are the classical XOR, OR and
|
|
AND gates, well-known elements in computer science. The tables here
|
|
show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
|
|
specific target \( y_i \).
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic">Simple code that tests XOR, OR and AND gates with linear regression</span>
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #408080; font-style: italic"># Design matrix</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The X.TX matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The invers of X.TX matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># The XOR gate </span>
|
|
yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
|
|
ThetaXOR <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># The OR gate </span>
|
|
yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
|
|
ThetaOR <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># The OR gate </span>
|
|
yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
|
|
ThetaAND <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>What is happening here?</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="does-logistic-regression-do-a-better-job">Does Logistic Regression do a better Job? </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic">Simple code that tests XOR and OR gates with linear regression</span>
|
|
<span style="color: #BA2121; font-style: italic">and logistic regression</span>
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Design matrix</span>
|
|
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The X.TX matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The invers of X.TX matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># The XOR gate </span>
|
|
yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
|
|
ThetaXOR <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># The OR gate </span>
|
|
yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
|
|
ThetaOR <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># The OR gate </span>
|
|
yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
|
|
ThetaAND <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"The linear regression prediction for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># Now we change to logistic regression</span>
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
|
logreg <span style="color: #666666">=</span> LogisticRegression()
|
|
logreg<span style="color: #666666">.</span>fit(X, yOR)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression for OR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yOR)))
|
|
|
|
logreg<span style="color: #666666">.</span>fit(X, yXOR)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression for XOR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yXOR)))
|
|
|
|
|
|
logreg<span style="color: #666666">.</span>fit(X, yAND)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Logistic Regression for AND gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yAND)))
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>Not exactly impressive, but somewhat better.</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adding-neural-networks">Adding Neural Networks </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># and now neural networks with Scikit-Learn and the XOR</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_classification
|
|
X, yXOR <span style="color: #666666">=</span> make_classification(n_samples<span style="color: #666666">=100</span>, random_state<span style="color: #666666">=1</span>)
|
|
FFNN <span style="color: #666666">=</span> MLPClassifier(random_state<span style="color: #666666">=1</span>, max_iter<span style="color: #666666">=300</span>)<span style="color: #666666">.</span>fit(X, yXOR)
|
|
FFNN<span style="color: #666666">.</span>predict_proba(X)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Test set accuracy with Feed Forward Neural Network for XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>FFNN<span style="color: #666666">.</span>score(X, yXOR)<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="mathematical-model">Mathematical model </h2>
|
|
|
|
<p>The output \( y \) is produced via the activation function \( f \)</p>
|
|
$$
|
|
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
|
|
$$
|
|
|
|
<p>This function receives \( x_i \) as inputs.
|
|
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
|
|
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
|
the neurons in the preceding layer. Furthermore, an MLP is
|
|
fully-connected, which means that each neuron receives a weighted sum
|
|
of the outputs of <em>all</em> neurons in the previous layer.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="mathematical-model">Mathematical model </h2>
|
|
|
|
<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
|
|
|
|
$$
|
|
\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
|
|
\label{_auto1}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>Here \( b_i \) is the so-called bias which is normally needed in
|
|
case of zero activation weights or inputs. How to fix the biases and
|
|
the weights will be discussed below. The value of \( z_i^1 \) is the
|
|
argument to the activation function \( f_i \) of each node \( i \), The
|
|
variable \( M \) stands for all possible inputs to a given node \( i \) in the
|
|
first layer. We define the output \( y_i^1 \) of all neurons in layer 1 as
|
|
</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j + b_i^1\right)
|
|
\label{outputLayer1}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>where we assume that all nodes in the same layer have identical
|
|
activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
|
|
In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
|
|
</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
|
|
\label{generalLayer}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
|
|
all the nodes in the first hidden layer are computed, the values of
|
|
the subsequent layer can be calculated and so forth until the output
|
|
is obtained.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="mathematical-model">Mathematical model </h2>
|
|
|
|
<p>The output of neuron \( i \) in layer 2 is thus,</p>
|
|
|
|
$$
|
|
\begin{align}
|
|
y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right)
|
|
\label{_auto2}\\
|
|
&= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
|
|
\label{outputLayer2}
|
|
\end{align}
|
|
$$
|
|
|
|
<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
|
|
|
|
$$
|
|
\begin{align}
|
|
y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right)
|
|
\label{_auto3}\\
|
|
&= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
|
|
+ b_1^3\right]
|
|
\label{_auto4}
|
|
\end{align}
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="mathematical-model">Mathematical model </h2>
|
|
|
|
<p>We can generalize this expression to an MLP with \( l \) hidden
|
|
layers. The complete functional form is,
|
|
</p>
|
|
|
|
$$
|
|
\begin{align}
|
|
&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
|
|
\label{completeNN}
|
|
\end{align}
|
|
$$
|
|
|
|
<p>which illustrates a basic property of MLPs: The only independent
|
|
variables are the input values \( x_n \).
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="mathematical-model">Mathematical model </h2>
|
|
|
|
<p>This confirms that an MLP, despite its quite convoluted mathematical
|
|
form, is nothing more than an analytic function, specifically a
|
|
mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
|
|
\hat{y} \in \mathbb{R}^m \).
|
|
</p>
|
|
|
|
<p>Furthermore, the flexibility and universality of an MLP can be
|
|
illustrated by realizing that the expression is essentially a nested
|
|
sum of scaled activation functions of the form
|
|
</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
f(x) = c_1 f(c_2 x + c_3) + c_4
|
|
\label{_auto5}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>where the parameters \( c_i \) are weights and biases. By adjusting these
|
|
parameters, the activation functions can be shifted up and down or
|
|
left and right, change slope or be rescaled which is the key to the
|
|
flexibility of a neural network.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h3 id="matrix-vector-notation">Matrix-vector notation </h3>
|
|
|
|
<p>We can introduce a more convenient notation for the activations in an A NN. </p>
|
|
|
|
<p>Additionally, we can represent the biases and activations
|
|
as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector
|
|
is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively.
|
|
</p>
|
|
|
|
<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
|
|
With this notation, the sum becomes a matrix-vector multiplication, and we can write
|
|
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
|
|
</p>
|
|
$$
|
|
\begin{equation}
|
|
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
|
f_2\left(\left[\begin{array}{ccc}
|
|
w^2_{11} &w^2_{12} &w^2_{13} \\
|
|
w^2_{21} &w^2_{22} &w^2_{23} \\
|
|
w^2_{31} &w^2_{32} &w^2_{33} \\
|
|
\end{array} \right] \cdot
|
|
\left[\begin{array}{c}
|
|
y^1_1 \\
|
|
y^1_2 \\
|
|
y^1_3 \\
|
|
\end{array}\right] +
|
|
\left[\begin{array}{c}
|
|
b^2_1 \\
|
|
b^2_2 \\
|
|
b^2_3 \\
|
|
\end{array}\right]\right).
|
|
\label{_auto6}
|
|
\end{equation}
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h3 id="matrix-vector-notation-and-activation">Matrix-vector notation and activation </h3>
|
|
|
|
<p>The activation of node \( i \) in layer 2 is</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) =
|
|
f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
|
|
\label{_auto7}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>This is not just a convenient and compact notation, but also a useful
|
|
and intuitive way to think about MLPs: The output is calculated by a
|
|
series of matrix-vector multiplications and vector additions that are
|
|
used as input to the activation functions. For each operation
|
|
\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h3 id="activation-functions">Activation functions </h3>
|
|
|
|
<p>A property that characterizes a neural network, other than its
|
|
connectivity, is the choice of activation function(s). As described
|
|
in, the following restrictions are imposed on an activation function
|
|
for a FFNN to fulfill the universal approximation theorem
|
|
</p>
|
|
|
|
<ul>
|
|
<li> Non-constant</li>
|
|
<li> Bounded</li>
|
|
<li> Monotonically-increasing</li>
|
|
<li> Continuous</li>
|
|
</ul>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h3 id="activation-functions-logistic-and-hyperbolic-ones">Activation functions, Logistic and Hyperbolic ones </h3>
|
|
|
|
<p>The second requirement excludes all linear functions. Furthermore, in
|
|
a MLP with only linear activation functions, each layer simply
|
|
performs a linear transformation of its inputs.
|
|
</p>
|
|
|
|
<p>Regardless of the number of layers, the output of the NN will be
|
|
nothing but a linear function of the inputs. Thus we need to introduce
|
|
some kind of non-linearity to the NN to be able to fit non-linear
|
|
functions Typical examples are the logistic <em>Sigmoid</em>
|
|
</p>
|
|
|
|
$$
|
|
f(x) = \frac{1}{1 + e^{-x}},
|
|
$$
|
|
|
|
<p>and the <em>hyperbolic tangent</em> function</p>
|
|
$$
|
|
f(x) = \tanh(x)
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h3 id="relevance">Relevance </h3>
|
|
|
|
<p>The <em>sigmoid</em> function are more biologically plausible because the
|
|
output of inactive neurons are zero. Such activation function are
|
|
called <em>one-sided</em>. However, it has been shown that the hyperbolic
|
|
tangent performs better than the sigmoid for training MLPs. has
|
|
become the most popular for <em>deep neural networks</em>
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">"""The sigmoid function (or the logistic curve) is a </span>
|
|
<span style="color: #BA2121; font-style: italic">function that takes any real number, z, and outputs a number (0,1).</span>
|
|
<span style="color: #BA2121; font-style: italic">It is useful in neural networks for assigning weights on a relative scale.</span>
|
|
<span style="color: #BA2121; font-style: italic">The value z is the weighted sum of parameters involved in the learning algorithm."""</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mt</span>
|
|
|
|
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.1</span>)
|
|
sigma_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1/</span>(<span style="color: #666666">1+</span>numpy<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z)))
|
|
sigma <span style="color: #666666">=</span> sigma_fn(z)
|
|
|
|
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
|
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
|
ax<span style="color: #666666">.</span>plot(z, sigma)
|
|
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.1</span>, <span style="color: #666666">1.1</span>])
|
|
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
|
|
ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
|
|
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
|
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'sigmoid function'</span>)
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
<span style="color: #BA2121; font-style: italic">"""Step Function"""</span>
|
|
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.02</span>)
|
|
step_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1.0</span> <span style="color: #008000; font-weight: bold">if</span> z <span style="color: #666666">>=</span> <span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">else</span> <span style="color: #666666">0.0</span>)
|
|
step <span style="color: #666666">=</span> step_fn(z)
|
|
|
|
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
|
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
|
ax<span style="color: #666666">.</span>plot(z, step)
|
|
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.5</span>, <span style="color: #666666">1.5</span>])
|
|
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
|
|
ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
|
|
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
|
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'step function'</span>)
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
<span style="color: #BA2121; font-style: italic">"""Sine Function"""</span>
|
|
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">0.1</span>)
|
|
t <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>sin(z)
|
|
|
|
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
|
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
|
ax<span style="color: #666666">.</span>plot(z, t)
|
|
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>])
|
|
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi,<span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi])
|
|
ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
|
|
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
|
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'sine function'</span>)
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
<span style="color: #BA2121; font-style: italic">"""Plots a graph of the squashing function used by a rectified linear</span>
|
|
<span style="color: #BA2121; font-style: italic">unit"""</span>
|
|
z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2</span>, <span style="color: #666666">2</span>, <span style="color: #666666">.1</span>)
|
|
zero <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>zeros(<span style="color: #008000">len</span>(z))
|
|
y <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>max([zero, z], axis<span style="color: #666666">=0</span>)
|
|
|
|
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
|
|
ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
|
|
ax<span style="color: #666666">.</span>plot(z, y)
|
|
ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
|
|
ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
|
|
ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
|
|
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">'z'</span>)
|
|
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">'Rectified linear unit'</span>)
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|