mhjensen
1026 lines
59 KiB
HTML
1026 lines
59 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Elements of machine learning">
|
|
|
|
<title>Data Analysis and Machine Learning: Elements of machine learning</title>
|
|
|
|
|
|
<link href="https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
|
<script src="https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
|
|
<script>hljs.initHighlightingOnLoad();</script>
|
|
|
|
<link href="https://thomasf.github.io/solarized-css/solarized-light.min.css" rel="stylesheet">
|
|
<style type="text/css">
|
|
h1 {color: #b58900;} /* yellow */
|
|
/* h1 {color: #cb4b16;} orange */
|
|
/* h1 {color: #d33682;} magenta, the original choice of thomasf */
|
|
code { padding: 0px; background-color: inherit; }
|
|
pre {
|
|
border: 0pt solid #93a1a1;
|
|
box-shadow: none;
|
|
}
|
|
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Neural networks', 2, None, '___sec0'),
|
|
('Artificial neurons', 2, None, '___sec1'),
|
|
('Neural network types', 2, None, '___sec2'),
|
|
('Feed-forward neural networks', 2, None, '___sec3'),
|
|
('Recurrent neural networks', 2, None, '___sec4'),
|
|
('Other types of networks', 2, None, '___sec5'),
|
|
('Multilayer perceptrons', 2, None, '___sec6'),
|
|
('Why multilayer perceptrons?', 2, None, '___sec7'),
|
|
('Mathematical model', 2, None, '___sec8'),
|
|
('Mathematical model', 2, None, '___sec9'),
|
|
('Mathematical model', 2, None, '___sec10'),
|
|
('Mathematical model', 2, None, '___sec11'),
|
|
('Mathematical model', 2, None, '___sec12'),
|
|
('Matrix-vector notation', 3, None, '___sec13'),
|
|
('Matrix-vector notation and activation', 3, None, '___sec14'),
|
|
('Activation functions', 3, None, '___sec15'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'___sec16'),
|
|
('Relevance', 3, None, '___sec17'),
|
|
('Setting up a Multi-layer perceptron model',
|
|
2,
|
|
None,
|
|
'___sec18'),
|
|
('Two-layer Neural Network', 2, None, '___sec19')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "AMS" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1>Data Analysis and Machine Learning: Elements of machine learning</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p>
|
|
<center><h4>Sep 28, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
<!-- add own code for DNN -->
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec0">Neural networks </h2>
|
|
|
|
<p>
|
|
Artificial neural networks are computational systems that can learn to
|
|
perform tasks by considering examples, generally without being
|
|
programmed with any task-specific rules. It is supposed to mimic a
|
|
biological system, wherein neurons interact by sending signals in the
|
|
form of mathematical functions between layers. All layers can contain
|
|
an arbitrary number of neurons, and each connection is represented by
|
|
a weight variable.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec1">Artificial neurons </h2>
|
|
|
|
<p>
|
|
The field of artificial neural networks has a long history of
|
|
development, and is closely connected with the advancement of computer
|
|
science and computers in general. A model of artificial neurons was
|
|
first developed by McCulloch and Pitts in 1943 to study signal
|
|
processing in the brain and has later been refined by others. The
|
|
general idea is to mimic neural networks in the human brain, which is
|
|
composed of billions of neurons that communicate with each other by
|
|
sending electrical signals. Each neuron accumulates its incoming
|
|
signals, which must exceed an activation threshold to yield an
|
|
output. If the threshold is not overcome, the neuron remains inactive,
|
|
i.e. has zero output.
|
|
|
|
<p>
|
|
This behaviour has inspired a simple mathematical model for an artificial neuron.
|
|
|
|
$$
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
|
|
\label{artificialNeuron}
|
|
\end{equation}
|
|
$$
|
|
|
|
Here, the output \( y \) of the neuron is the value of its activation function, which have as input
|
|
a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
|
|
|
|
<p>
|
|
Conceptually, it is helpful to divide neural networks into four
|
|
categories:
|
|
|
|
<ol>
|
|
<li> general purpose neural networks for supervised learning,</li>
|
|
<li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
|
|
<li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
|
|
<li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
|
|
</ol>
|
|
|
|
In natural science, DNNs and CNNs have already found numerous applications. In
|
|
statistical physics, they have been applied to detect phase
|
|
transitions in 2D Ising and Potts models, lattice gauge theories, and
|
|
different phases of polymers, or solving the Navier-Stokes equation in weather forecasting.
|
|
Deep learning has also found interesting applications in quantum
|
|
physics. Various quantum phase transitions can be detected and studied
|
|
using DNNs and CNNs,
|
|
topological phases, and even non-equilibrium many-body
|
|
localization. Representing quantum states as DNNs quantum state
|
|
tomography are among some of the impressive
|
|
achievements to reveal the potential of DNNs to facilitate the study
|
|
of quantum systems.
|
|
|
|
<p>
|
|
In quantum information theory, it has been shown that one can perform
|
|
gate decompositions with the help of neural. In lattice quantum chromodynamics,
|
|
DNNs have been used to learn action parameters in regions of parameter
|
|
space where PCA fails.
|
|
|
|
<p>
|
|
The applications are not limited to the natural sciences. There is a plethora of applications in essentially all disciplines, from the humanities to life science and medicine.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec2">Neural network types </h2>
|
|
|
|
<p>
|
|
An artificial neural network (NN), is a computational model that
|
|
consists of layers of connected neurons, or <em>nodes</em>. It is supposed
|
|
to mimic a biological nervous system by letting each neuron interact
|
|
with other neurons by sending signals in the form of mathematical
|
|
functions between layers. A wide variety of different NNs have been
|
|
developed, but most of them consist of an input layer, an output layer
|
|
and eventual layers in-between, called <em>hidden layers</em>. All layers can
|
|
contain an arbitrary number of nodes, and each connection between two
|
|
nodes is associated with a weight variable.
|
|
|
|
<p>
|
|
Neural networks (also called neural nets) are neural-inspired
|
|
nonlinear models for supervised learning. As we will see, neural nets
|
|
can be viewed as natural, more powerful extensions of supervised
|
|
learning methods such as linear and logistic regression and soft-max
|
|
methods.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec3">Feed-forward neural networks </h2>
|
|
|
|
<p>
|
|
The feed-forward neural network (FFNN) was the first and simplest type of NN devised. In this network,
|
|
the information moves in only one direction: forward through the layers.
|
|
|
|
<p>
|
|
Nodes are represented by circles, while the arrows display the connections between the nodes, including the
|
|
direction of information flow. Additionally, each arrow corresponds to a weight variable, not displayed here.
|
|
We observe that each node in a layer is connected to <em>all</em> nodes in the subsequent layer,
|
|
making this a so-called <em>fully-connected</em> FFNN.
|
|
|
|
<p>
|
|
A different variant of FFNNs are <em>convolutional neural networks</em> (CNNs), which have a connectivity pattern
|
|
inspired by the animal visual cortex. Individual neurons in the visual cortex only respond to stimuli from
|
|
small sub-regions of the visual field, called a receptive field. This makes the neurons well-suited to exploit the strong
|
|
spatially local correlation present in natural images. The response of each neuron can be approximated mathematically
|
|
as a convolution operation.
|
|
|
|
<p>
|
|
CNNs emulate the behaviour of neurons in the visual cortex by enforcing a <em>local</em> connectivity pattern
|
|
between nodes of adjacent layers: Each node
|
|
in a convolutional layer is connected only to a subset of the nodes in the previous layer,
|
|
in contrast to the fully-connected FFNN.
|
|
Often, CNNs
|
|
consist of several convolutional layers that learn local features of the input, with a fully-connected layer at the end,
|
|
which gathers all the local data and produces the outputs. They have wide applications in image and video recognition
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec4">Recurrent neural networks </h2>
|
|
|
|
<p>
|
|
So far we have only mentioned NNs where information flows in one direction: forward. <em>Recurrent neural networks</em> on
|
|
the other hand, have connections between nodes that form directed <em>cycles</em>. This creates a form of
|
|
internal memory which are able to capture information on what has been calculated before; the output is dependent
|
|
on the previous computations. Recurrent NNs make use of sequential information by performing the same task for
|
|
every element in a sequence, where each element depends on previous elements. An example of such information is
|
|
sentences, making recurrent NNs especially well-suited for handwriting and speech recognition.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec5">Other types of networks </h2>
|
|
|
|
<p>
|
|
There are many other kinds of NNs that have been developed. One type that is specifically designed for interpolation
|
|
in multidimensional space is the radial basis function (RBF) network. RBFs are typically made up of three layers:
|
|
an input layer, a hidden layer with non-linear radial symmetric activation functions and a linear output layer (''linear'' here
|
|
means that each node in the output layer has a linear activation function). The layers are normally fully-connected and
|
|
there are no cycles, thus RBFs can be viewed as a type of fully-connected FFNN. They are however usually treated as
|
|
a separate type of NN due the unusual activation functions.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec6">Multilayer perceptrons </h2>
|
|
|
|
<p>
|
|
One uses often so-called fully-connected feed-forward neural networks
|
|
with three or more layers (an input layer, one or more hidden layers
|
|
and an output layer) consisting of neurons that have non-linear
|
|
activation functions.
|
|
|
|
<p>
|
|
Such networks are often called <em>multilayer perceptrons</em> (MLPs)
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec7">Why multilayer perceptrons? </h2>
|
|
|
|
<p>
|
|
According to the <em>Universal approximation theorem</em>, a feed-forward neural network with just a single hidden layer containing
|
|
a finite number of neurons can approximate a continuous multidimensional function to arbitrary accuracy,
|
|
assuming the activation function for the hidden layer is a <b>non-constant, bounded and monotonically-increasing continuous function</b>.
|
|
|
|
<p>
|
|
Note that the requirements on the activation function only applies to
|
|
the hidden layer, the output nodes are always assumed to be linear, so
|
|
as to not restrict the range of output values.
|
|
|
|
<p>
|
|
We note that this theorem is only applicable to an NN with <em>one</em> hidden
|
|
layer. Therefore, we can easily construct an NN that employs
|
|
activation functions which do not satisfy the above requirements, as
|
|
long as we have at least one layer with activation functions that
|
|
<em>do</em>. Furthermore, although the universal approximation theorem lays
|
|
the theoretical foundation for regression with neural networks, it
|
|
does not say anything about how things work in practice: A neural
|
|
network can still be able to approximate a given function reasonably
|
|
well without having the flexibility to fit <em>all other</em> functions.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec8">Mathematical model </h2>
|
|
|
|
$$
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(u)
|
|
\label{artificialNeuron2}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
|
|
the neurons in the preceding layer. Furthermore, an MLP is
|
|
fully-connected, which means that each neuron receives a weighted sum
|
|
of the outputs of <em>all</em> neurons in the previous layer.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec9">Mathematical model </h2>
|
|
|
|
<p>
|
|
First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( u_i^1 \) of the input coordinates \( x_j \),
|
|
|
|
$$
|
|
\begin{equation}
|
|
u_i^1 = \sum_{j=1}^2 w_{ij}^1 x_j + b_i^1
|
|
\label{_auto1}
|
|
\end{equation}
|
|
$$
|
|
|
|
This value is the argument to the activation function \( f_1 \) of each neuron \( i \),
|
|
producing the output \( y_i^1 \) of all neurons in layer 1,
|
|
|
|
$$
|
|
\begin{equation}
|
|
y_i^1 = f_1(u_i^1) = f_1\left(\sum_{j=1}^2 w_{ij}^1 x_j + b_i^1\right)
|
|
\label{outputLayer1}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
where we assume that all nodes in the same layer have identical
|
|
activation functions, hence the notation \( f_l \)
|
|
|
|
$$
|
|
\begin{equation}
|
|
y_i^l = f_l(u_i^l) = f_l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
|
|
\label{generalLayer}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
where \( N_l \) is the number of nodes in layer \( l \). When the output of
|
|
all the nodes in the first hidden layer are computed, the values of
|
|
the subsequent layer can be calculated and so forth until the output
|
|
is obtained.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec10">Mathematical model </h2>
|
|
|
|
<p>
|
|
The output of neuron \( i \) in layer 2 is thus,
|
|
|
|
$$
|
|
\begin{align}
|
|
y_i^2 &= f_2\left(\sum_{j=1}^3 w_{ij}^2 y_j^1 + b_i^2\right)
|
|
\label{_auto2}\\
|
|
&= f_2\left[\sum_{j=1}^3 w_{ij}^2f_1\left(\sum_{k=1}^2 w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
|
|
\label{outputLayer2}
|
|
\end{align}
|
|
$$
|
|
|
|
where we have substituted \( y_m^1 \) with. Finally, the NN output yields,
|
|
|
|
$$
|
|
\begin{align}
|
|
y_1^3 &= f_3\left(\sum_{j=1}^3 w_{1m}^3 y_j^2 + b_1^3\right)
|
|
\label{_auto3}\\
|
|
&= f_3\left[\sum_{j=1}^3 w_{1j}^3 f_2\left(\sum_{k=1}^3 w_{jk}^2 f_1\left(\sum_{m=1}^2 w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
|
|
+ b_1^3\right]
|
|
\label{_auto4}
|
|
\end{align}
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec11">Mathematical model </h2>
|
|
|
|
<p>
|
|
We can generalize this expression to an MLP with \( l \) hidden
|
|
layers. The complete functional form is,
|
|
|
|
$$
|
|
\begin{align}
|
|
&y^{l+1}_1\! = \!f_{l+1}\!\left[\!\sum_{j=1}^{N_l}\! w_{1j}^3 f_l\!\left(\!\sum_{k=1}^{N_{l-1}}\! w_{jk}^2 f_{l-1}\!\left(\!
|
|
\dots \!f_1\!\left(\!\sum_{n=1}^{N_0} \!w_{mn}^1 x_n\! + \!b_m^1\!\right)
|
|
\!\dots \!\right) \!+ \!b_k^2\!\right)
|
|
\!+ \!b_1^3\!\right] &&
|
|
\label{completeNN}
|
|
\end{align}
|
|
$$
|
|
|
|
<p>
|
|
which illustrates a basic property of MLPs: The only independent
|
|
variables are the input values \( x_n \).
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec12">Mathematical model </h2>
|
|
|
|
<p>
|
|
This confirms that an MLP, despite its quite convoluted mathematical
|
|
form, is nothing more than an analytic function, specifically a
|
|
mapping of real-valued vectors \( \vec{x} \in \mathbb{R}^n \rightarrow
|
|
\vec{y} \in \mathbb{R}^m \). In our example, \( n=2 \) and
|
|
\( m=1 \). Consequentially, the number of input and output values of the
|
|
function we want to fit must be equal to the number of inputs and
|
|
outputs of our MLP.
|
|
|
|
<p>
|
|
Furthermore, the flexibility and universality of a MLP can be
|
|
illustrated by realizing that the expression is essentially a nested
|
|
sum of scaled activation functions of the form
|
|
|
|
$$
|
|
\begin{equation}
|
|
h(x) = c_1 f(c_2 x + c_3) + c_4
|
|
\label{_auto5}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
where the parameters \( c_i \) are weights and biases. By adjusting these
|
|
parameters, the activation functions can be shifted up and down or
|
|
left and right, change slope or be rescaled which is the key to the
|
|
flexibility of a neural network.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h3 id="___sec13">Matrix-vector notation </h3>
|
|
|
|
<p>
|
|
We can introduce a more convenient notation for the activations in a NN.
|
|
|
|
<p>
|
|
Additionally, we can represent the biases and activations
|
|
as layer-wise column vectors \( \vec{b}_l \) and \( \vec{y}_l \), so that the \( i \)-th element of each vector
|
|
is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively.
|
|
|
|
<p>
|
|
We have that \( \mathrm{W}_l \) is a \( N_{l-1} \times N_l \) matrix, while \( \vec{b}_l \) and \( \vec{y}_l \) are \( N_l \times 1 \) column vectors.
|
|
With this notation, the sum in becomes a matrix-vector multiplication, and we can write
|
|
the equation for the activations of hidden layer 2 in
|
|
$$
|
|
\begin{equation}
|
|
\vec{y}_2 = f_2(\mathrm{W}_2 \vec{y}_{1} + \vec{b}_{2}) =
|
|
f_2\left(\left[\begin{array}{ccc}
|
|
w^2_{11} &w^2_{12} &w^2_{13} \\
|
|
w^2_{21} &w^2_{22} &w^2_{23} \\
|
|
w^2_{31} &w^2_{32} &w^2_{33} \\
|
|
\end{array} \right] \cdot
|
|
\left[\begin{array}{c}
|
|
y^1_1 \\
|
|
y^1_2 \\
|
|
y^1_3 \\
|
|
\end{array}\right] +
|
|
\left[\begin{array}{c}
|
|
b^2_1 \\
|
|
b^2_2 \\
|
|
b^2_3 \\
|
|
\end{array}\right]\right).
|
|
\label{_auto6}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h3 id="___sec14">Matrix-vector notation and activation </h3>
|
|
|
|
<p>
|
|
The activation of node \( i \) in layer 2 is
|
|
|
|
$$
|
|
\begin{equation}
|
|
y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) =
|
|
f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
|
|
\label{_auto7}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
This is not just a convenient and compact notation, but also a useful
|
|
and intuitive way to think about MLPs: The output is calculated by a
|
|
series of matrix-vector multiplications and vector additions that are
|
|
used as input to the activation functions. For each operation
|
|
\( \mathrm{W}_l \vec{y}_{l-1} \) we move forward one layer.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h3 id="___sec15">Activation functions </h3>
|
|
|
|
<p>
|
|
A property that characterizes a neural network, other than its
|
|
connectivity, is the choice of activation function(s). As described
|
|
in, the following restrictions are imposed on an activation function
|
|
for a FFNN to fulfill the universal approximation theorem
|
|
|
|
<ul>
|
|
<li> Non-constant</li>
|
|
<li> Bounded</li>
|
|
<li> Monotonically-increasing</li>
|
|
<li> Continuous</li>
|
|
</ul>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h3 id="___sec16">Activation functions, Logistic and Hyperbolic ones </h3>
|
|
|
|
<p>
|
|
The second requirement excludes all linear functions. Furthermore, in
|
|
a MLP with only linear activation functions, each layer simply
|
|
performs a linear transformation of its inputs.
|
|
|
|
<p>
|
|
Regardless of the number of layers, the output of the NN will be
|
|
nothing but a linear function of the inputs. Thus we need to introduce
|
|
some kind of non-linearity to the NN to be able to fit non-linear
|
|
functions Typical examples are the logistic <em>Sigmoid</em>
|
|
|
|
$$
|
|
\begin{equation}
|
|
f(x) = \frac{1}{1 + e^{-x}},
|
|
\label{sigmoidActivationFunction}
|
|
\end{equation}
|
|
$$
|
|
|
|
and the <em>hyperbolic tangent</em> function
|
|
$$
|
|
\begin{equation}
|
|
f(x) = \tanh(x)
|
|
\label{tanhActivationFunction}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h3 id="___sec17">Relevance </h3>
|
|
|
|
<p>
|
|
The <em>sigmoid</em> function are more biologically plausible because the
|
|
output of inactive neurons are zero. Such activation function are
|
|
called <em>one-sided</em>. However, it has been shown that the hyperbolic
|
|
tangent performs better than the sigmoid for training MLPs. has
|
|
become the most popular for <em>deep neural networks</em>
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #CD5555">"""The sigmoid function (or the logistic curve) is a </span>
|
|
<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
|
|
<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
|
|
<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm."""</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
|
|
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">1</span>)
|
|
sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
|
|
sigma = sigma_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, sigma)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'sigmoid function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Step Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">02</span>)
|
|
step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z >= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
|
|
step = step_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, step)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'step function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Sine Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
|
|
t = numpy.sin(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, t)
|
|
ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'sine function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Plots a graph of the squashing function used by a rectified linear</span>
|
|
<span style="color: #CD5555">unit"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>, .<span style="color: #B452CD">1</span>)
|
|
zero = numpy.zeros(<span style="color: #658b00">len</span>(z))
|
|
y = numpy.max([zero, z], axis=<span style="color: #B452CD">0</span>)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, y)
|
|
ax.set_ylim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'Rectified linear unit'</span>)
|
|
|
|
plt.show()
|
|
</pre></div>
|
|
<p>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec18">Setting up a Multi-layer perceptron model </h2>
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">scipy</span> <span style="color: #8B008B; font-weight: bold">import</span> optimize
|
|
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">Neural_Network</span>(<span style="color: #658b00">object</span>):
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">__init__</span>(<span style="color: #658b00">self</span>, Lambda=<span style="color: #B452CD">0</span>):
|
|
<span style="color: #228B22">#Define Hyperparameters</span>
|
|
<span style="color: #658b00">self</span>.inputLayerSize = <span style="color: #B452CD">2</span>
|
|
<span style="color: #658b00">self</span>.outputLayerSize = <span style="color: #B452CD">1</span>
|
|
<span style="color: #658b00">self</span>.hiddenLayerSize = <span style="color: #B452CD">3</span>
|
|
|
|
<span style="color: #228B22">#Weights (parameters)</span>
|
|
<span style="color: #658b00">self</span>.W1 = np.random.randn(<span style="color: #658b00">self</span>.inputLayerSize,<span style="color: #658b00">self</span>.hiddenLayerSize)
|
|
<span style="color: #658b00">self</span>.W2 = np.random.randn(<span style="color: #658b00">self</span>.hiddenLayerSize,<span style="color: #658b00">self</span>.outputLayerSize)
|
|
|
|
<span style="color: #228B22">#Regularization Parameter:</span>
|
|
<span style="color: #658b00">self</span>.Lambda = Lambda
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">forward</span>(<span style="color: #658b00">self</span>, X):
|
|
<span style="color: #228B22">#Propogate inputs though network</span>
|
|
<span style="color: #658b00">self</span>.z2 = np.dot(X, <span style="color: #658b00">self</span>.W1)
|
|
<span style="color: #658b00">self</span>.a2 = <span style="color: #658b00">self</span>.sigmoid(<span style="color: #658b00">self</span>.z2)
|
|
<span style="color: #658b00">self</span>.z3 = np.dot(<span style="color: #658b00">self</span>.a2, <span style="color: #658b00">self</span>.W2)
|
|
yHat = <span style="color: #658b00">self</span>.sigmoid(<span style="color: #658b00">self</span>.z3)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> yHat
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(<span style="color: #658b00">self</span>, z):
|
|
<span style="color: #228B22">#Apply sigmoid activation function to scalar, vector, or matrix</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+np.exp(-z))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoidPrime</span>(<span style="color: #658b00">self</span>,z):
|
|
<span style="color: #228B22">#Gradient of sigmoid</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.exp(-z)/((<span style="color: #B452CD">1</span>+np.exp(-z))**<span style="color: #B452CD">2</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunction</span>(<span style="color: #658b00">self</span>, X, y):
|
|
<span style="color: #228B22">#Compute cost for given X,y, use weights already stored in class.</span>
|
|
<span style="color: #658b00">self</span>.yHat = <span style="color: #658b00">self</span>.forward(X)
|
|
J = <span style="color: #B452CD">0.5</span>*<span style="color: #658b00">sum</span>((y-<span style="color: #658b00">self</span>.yHat)**<span style="color: #B452CD">2</span>)/X.shape[<span style="color: #B452CD">0</span>] + (<span style="color: #658b00">self</span>.Lambda/<span style="color: #B452CD">2</span>)*(np.sum(<span style="color: #658b00">self</span>.W1**<span style="color: #B452CD">2</span>)+np.sum(<span style="color: #658b00">self</span>.W2**<span style="color: #B452CD">2</span>))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> J
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunctionPrime</span>(<span style="color: #658b00">self</span>, X, y):
|
|
<span style="color: #228B22">#Compute derivative with respect to W and W2 for a given X and y:</span>
|
|
<span style="color: #658b00">self</span>.yHat = <span style="color: #658b00">self</span>.forward(X)
|
|
|
|
delta3 = np.multiply(-(y-<span style="color: #658b00">self</span>.yHat), <span style="color: #658b00">self</span>.sigmoidPrime(<span style="color: #658b00">self</span>.z3))
|
|
<span style="color: #228B22">#Add gradient of regularization term:</span>
|
|
dJdW2 = np.dot(<span style="color: #658b00">self</span>.a2.T, delta3)/X.shape[<span style="color: #B452CD">0</span>] + <span style="color: #658b00">self</span>.Lambda*<span style="color: #658b00">self</span>.W2
|
|
|
|
delta2 = np.dot(delta3, <span style="color: #658b00">self</span>.W2.T)*<span style="color: #658b00">self</span>.sigmoidPrime(<span style="color: #658b00">self</span>.z2)
|
|
<span style="color: #228B22">#Add gradient of regularization term:</span>
|
|
dJdW1 = np.dot(X.T, delta2)/X.shape[<span style="color: #B452CD">0</span>] + <span style="color: #658b00">self</span>.Lambda*<span style="color: #658b00">self</span>.W1
|
|
|
|
<span style="color: #8B008B; font-weight: bold">return</span> dJdW1, dJdW2
|
|
|
|
<span style="color: #228B22">#Helper functions for interacting with other methods/classes</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">getParams</span>(<span style="color: #658b00">self</span>):
|
|
<span style="color: #228B22">#Get W1 and W2 Rolled into vector:</span>
|
|
params = np.concatenate((<span style="color: #658b00">self</span>.W1.ravel(), <span style="color: #658b00">self</span>.W2.ravel()))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> params
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">setParams</span>(<span style="color: #658b00">self</span>, params):
|
|
<span style="color: #228B22">#Set W1 and W2 using single parameter vector:</span>
|
|
W1_start = <span style="color: #B452CD">0</span>
|
|
W1_end = <span style="color: #658b00">self</span>.hiddenLayerSize*<span style="color: #658b00">self</span>.inputLayerSize
|
|
<span style="color: #658b00">self</span>.W1 = np.reshape(params[W1_start:W1_end], \
|
|
(<span style="color: #658b00">self</span>.inputLayerSize, <span style="color: #658b00">self</span>.hiddenLayerSize))
|
|
W2_end = W1_end + <span style="color: #658b00">self</span>.hiddenLayerSize*<span style="color: #658b00">self</span>.outputLayerSize
|
|
<span style="color: #658b00">self</span>.W2 = np.reshape(params[W1_end:W2_end], \
|
|
(<span style="color: #658b00">self</span>.hiddenLayerSize, <span style="color: #658b00">self</span>.outputLayerSize))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">computeGradients</span>(<span style="color: #658b00">self</span>, X, y):
|
|
dJdW1, dJdW2 = <span style="color: #658b00">self</span>.costFunctionPrime(X, y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.concatenate((dJdW1.ravel(), dJdW2.ravel()))
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">trainer</span>(<span style="color: #658b00">object</span>):
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">__init__</span>(<span style="color: #658b00">self</span>, N):
|
|
<span style="color: #228B22">#Make Local reference to network:</span>
|
|
<span style="color: #658b00">self</span>.N = N
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">callbackF</span>(<span style="color: #658b00">self</span>, params):
|
|
<span style="color: #658b00">self</span>.N.setParams(params)
|
|
<span style="color: #658b00">self</span>.J.append(<span style="color: #658b00">self</span>.N.costFunction(<span style="color: #658b00">self</span>.X, <span style="color: #658b00">self</span>.y))
|
|
<span style="color: #658b00">self</span>.testJ.append(<span style="color: #658b00">self</span>.N.costFunction(<span style="color: #658b00">self</span>.testX, <span style="color: #658b00">self</span>.testY))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunctionWrapper</span>(<span style="color: #658b00">self</span>, params, X, y):
|
|
<span style="color: #658b00">self</span>.N.setParams(params)
|
|
cost = <span style="color: #658b00">self</span>.N.costFunction(X, y)
|
|
grad = <span style="color: #658b00">self</span>.N.computeGradients(X,y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> cost, grad
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train</span>(<span style="color: #658b00">self</span>, trainX, trainY, testX, testY):
|
|
<span style="color: #228B22">#Make an internal variable for the callback function:</span>
|
|
<span style="color: #658b00">self</span>.X = trainX
|
|
<span style="color: #658b00">self</span>.y = trainY
|
|
|
|
<span style="color: #658b00">self</span>.testX = testX
|
|
<span style="color: #658b00">self</span>.testY = testY
|
|
|
|
<span style="color: #228B22">#Make empty list to store training costs:</span>
|
|
<span style="color: #658b00">self</span>.J = []
|
|
<span style="color: #658b00">self</span>.testJ = []
|
|
|
|
params0 = <span style="color: #658b00">self</span>.N.getParams()
|
|
|
|
options = {<span style="color: #CD5555">'maxiter'</span>: <span style="color: #B452CD">200</span>, <span style="color: #CD5555">'disp'</span> : <span style="color: #658b00">True</span>}
|
|
_res = optimize.minimize(<span style="color: #658b00">self</span>.costFunctionWrapper, params0, jac=<span style="color: #658b00">True</span>, method=<span style="color: #CD5555">'BFGS'</span>, \
|
|
args=(trainX, trainY), options=options, callback=<span style="color: #658b00">self</span>.callbackF)
|
|
|
|
<span style="color: #658b00">self</span>.N.setParams(_res.x)
|
|
<span style="color: #658b00">self</span>.optimizationResults = _res
|
|
</pre></div>
|
|
<p>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec19">Two-layer Neural Network </h2>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
<span style="color: #228B22">#sigmoid</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">nonlin</span>(x, deriv=<span style="color: #658b00">False</span>):
|
|
<span style="color: #8B008B; font-weight: bold">if</span> (deriv==<span style="color: #658b00">True</span>):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x*(<span style="color: #B452CD">1</span>-x)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+np.exp(-x))
|
|
|
|
<span style="color: #228B22">#input data</span>
|
|
x=np.array([[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>]])
|
|
|
|
<span style="color: #228B22">#output data</span>
|
|
y=np.array([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>]).T
|
|
|
|
<span style="color: #228B22">#seed random numbers to make calculation</span>
|
|
np.random.seed(<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22">#initialize weights with mean=0</span>
|
|
syn0=<span style="color: #B452CD">2</span>*np.random.random((<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>))-<span style="color: #B452CD">1</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">10000</span>):
|
|
<span style="color: #228B22">#forward propogation</span>
|
|
l0=x
|
|
l1=nonlin(np.dot(l0,syn0))
|
|
l1_error=y-l1
|
|
<span style="color: #228B22">#multiply error by slope of sigmoid at values of l1</span>
|
|
l1_delta=l1_error*nonlin(l1,<span style="color: #658b00">True</span>)
|
|
<span style="color: #228B22">#update weights</span>
|
|
syn0+=np.dot(l0.T, l1_delta)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Output after training: "</span>,l1 )
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">random</span>
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">Network</span>(<span style="color: #658b00">object</span>):
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">_init_</span>(<span style="color: #658b00">self</span>, sizes):
|
|
<span style="color: #658b00">self</span>.num_layers=<span style="color: #658b00">len</span>(sizes)
|
|
<span style="color: #658b00">self</span>.sizes=sizes
|
|
<span style="color: #658b00">self</span>.biases=[np.random.randn(y,<span style="color: #B452CD">1</span>) <span style="color: #8B008B; font-weight: bold">for</span> y <span style="color: #8B008B">in</span> sizes[<span style="color: #B452CD">1</span>:]]
|
|
<span style="color: #658b00">self</span>.weights=[np.random.randn(y,x) <span style="color: #8B008B; font-weight: bold">for</span> x,y <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(sizes[:-<span style="color: #B452CD">1</span>], sizes[<span style="color: #B452CD">1</span>:])]
|
|
|
|
<span style="color: #228B22">#sizes is the number of neurons in each layer</span>
|
|
<span style="color: #228B22">#for example, say n_1st_layer=3, n_2nd_layer=3, n_3rd_layer=1, then net=Network([3,3,1])</span>
|
|
|
|
<span style="color: #228B22">#The biases and weights are initialized randomly, using Gaussian distributions of mean=0, stdev=1</span>
|
|
<span style="color: #228B22">#z is a vector (or a np.array)</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">feedforward</span>(<span style="color: #658b00">self</span>,a):
|
|
<span style="color: #228B22">#returns output w/ 'a' as an input</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> b, w <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, <span style="color: #658b00">self</span>.weights):
|
|
a=sigmoid(np.dot(w,b)+b)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> a
|
|
|
|
<span style="color: #228B22">#Apply a Stochastic Gradient Descent (SGD) method:</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">SGD</span>(<span style="color: #658b00">self</span>, training_data, epochs, mini_batch_size, eta, test_data=<span style="color: #658b00">None</span>):
|
|
<span style="color: #CD5555">"""Trains network using batches incorporating SGD. The network will be evaluated against the</span>
|
|
<span style="color: #CD5555"> test data after each epoch, with partial progress being printed out (this is useful for tracking,</span>
|
|
<span style="color: #CD5555"> but slows the process.)"""</span>
|
|
<span style="color: #8B008B; font-weight: bold">if</span> test_data: n_test=<span style="color: #658b00">len</span>(test_data)
|
|
n=<span style="color: #658b00">len</span>(training_data)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(epochs):
|
|
random.shuffle(training_data)
|
|
mini_batches=[training_data[k:k+mini_batch_size] <span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(o,n,mini_batch_size)]
|
|
<span style="color: #8B008B; font-weight: bold">for</span> mini_batch <span style="color: #8B008B">in</span> mini_batches:
|
|
<span style="color: #658b00">self</span>.update_mini_batch(mini_batch, eta)
|
|
<span style="color: #8B008B; font-weight: bold">if</span> test_data:
|
|
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Epoch {0}: {1}/{2}"</span>.format(j, <span style="color: #658b00">self</span>.evaluate(test_data), n_test))
|
|
<span style="color: #8B008B; font-weight: bold">else</span>:
|
|
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Epoch {0} complete"</span>.format(j))
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">update_mini_batch</span>(<span style="color: #658b00">self</span>, mini_batch, eta):
|
|
<span style="color: #228B22">#updates w and b using backpropagation to a single mini batch. eta is the learning rate."</span>
|
|
nabla_b=[np.zeros(b.shape) <span style="color: #8B008B; font-weight: bold">for</span> b <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.biases]
|
|
nabla_w=[np.zeros(w.shape) <span style="color: #8B008B; font-weight: bold">for</span> w <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.weights]
|
|
<span style="color: #8B008B; font-weight: bold">for</span> x,y <span style="color: #8B008B">in</span> mini_batch:
|
|
delta_nabla_b, delta_nabla_w=<span style="color: #658b00">self</span>.backprop(x,y)
|
|
nabla_b=[nb+dnb <span style="color: #8B008B; font-weight: bold">for</span> nb, dnb <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(nabla_b, delta_nabla_b)]
|
|
nabla_w=[nw+dnw <span style="color: #8B008B; font-weight: bold">for</span> nw, dnw <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(nabla_w, delta_nabla_w)]
|
|
<span style="color: #658b00">self</span>.weights=[w-(eta/<span style="color: #658b00">len</span>(mini_batch))*nw <span style="color: #8B008B; font-weight: bold">for</span> w, nw <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.weights, nabla_w)]
|
|
<span style="color: #658b00">self</span>.biases=[b-(eta/<span style="color: #658b00">len</span>(mini_batch))*nb <span style="color: #8B008B; font-weight: bold">for</span> b, nb <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, nabla_b)]
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">backprop</span>(<span style="color: #658b00">self</span>, x, y):
|
|
<span style="color: #CD5555">"""Return a tuple ``(nabla_b, nabla_w)`` representing the</span>
|
|
<span style="color: #CD5555"> gradient for the cost function C_x. ``nabla_b`` and</span>
|
|
<span style="color: #CD5555"> ``nabla_w`` are layer-by-layer lists of numpy arrays, similar</span>
|
|
<span style="color: #CD5555"> to ``self.biases`` and ``self.weights``."""</span>
|
|
nabla_b = [np.zeros(b.shape) <span style="color: #8B008B; font-weight: bold">for</span> b <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.biases]
|
|
nabla_w = [np.zeros(w.shape) <span style="color: #8B008B; font-weight: bold">for</span> w <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.weights]
|
|
<span style="color: #228B22"># feedforward</span>
|
|
activation = x
|
|
activations = [x] <span style="color: #228B22"># list to store all the activations, layer by layer</span>
|
|
zs = [] <span style="color: #228B22"># list to store all the z vectors, layer by layer</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> b, w <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, <span style="color: #658b00">self</span>.weights):
|
|
z = np.dot(w, activation)+b
|
|
zs.append(z)
|
|
activation = sigmoid(z)
|
|
activations.append(activation)
|
|
<span style="color: #228B22"># backward pass</span>
|
|
delta = <span style="color: #658b00">self</span>.cost_derivative(activations[-<span style="color: #B452CD">1</span>], y) * \
|
|
sigmoid_prime(zs[-<span style="color: #B452CD">1</span>])
|
|
nabla_b[-<span style="color: #B452CD">1</span>] = delta
|
|
nabla_w[-<span style="color: #B452CD">1</span>] = np.dot(delta, activations[-<span style="color: #B452CD">2</span>].transpose())
|
|
<span style="color: #228B22"># Note that the variable l in the loop below is used a little</span>
|
|
<span style="color: #228B22"># differently to the notation in Chapter 2 of the book. Here,</span>
|
|
<span style="color: #228B22"># l = 1 means the last layer of neurons, l = 2 is the</span>
|
|
<span style="color: #228B22"># second-last layer, and so on. It's a renumbering of the</span>
|
|
<span style="color: #228B22"># scheme in the book, used here to take advantage of the fact</span>
|
|
<span style="color: #228B22"># that Python can use negative indices in lists.</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> l <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(<span style="color: #B452CD">2</span>, <span style="color: #658b00">self</span>.num_layers):
|
|
z = zs[-l]
|
|
sp = sigmoid_prime(z)
|
|
delta = np.dot(<span style="color: #658b00">self</span>.weights[-l+<span style="color: #B452CD">1</span>].transpose(), delta) * sp
|
|
nabla_b[-l] = delta
|
|
nabla_w[-l] = np.dot(delta, activations[-l-<span style="color: #B452CD">1</span>].transpose())
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (nabla_b, nabla_w)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">evaluate</span>(<span style="color: #658b00">self</span>, test_data):
|
|
<span style="color: #CD5555">"""Return the number of test inputs for which the neural</span>
|
|
<span style="color: #CD5555"> network outputs the correct result. Note that the neural</span>
|
|
<span style="color: #CD5555"> network's output is assumed to be the index of whichever</span>
|
|
<span style="color: #CD5555"> neuron in the final layer has the highest activation."""</span>
|
|
test_results = [(np.argmax(<span style="color: #658b00">self</span>.feedforward(x)), y)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> (x, y) <span style="color: #8B008B">in</span> test_data]
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #658b00">sum</span>(<span style="color: #658b00">int</span>(x == y) <span style="color: #8B008B; font-weight: bold">for</span> (x, y) <span style="color: #8B008B">in</span> test_results)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">cost_derivative</span>(<span style="color: #658b00">self</span>, output_activations, y):
|
|
<span style="color: #CD5555">"""Return the vector of partial derivatives \partial C_x /</span>
|
|
<span style="color: #CD5555"> \partial a for the output activations."""</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (output_activations-y)
|
|
|
|
|
|
|
|
<span style="color: #228B22">#Functions</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(z):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1.0</span>/(<span style="color: #B452CD">1.0</span>+np.exp(-z))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid_prime</span>(z):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> sigmoid(z)*(<span style="color: #B452CD">1</span>-sigmoid(z))
|
|
|
|
network=Network()
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># %load neural-networks-and-deep-learning/src/mnist_loader.py</span>
|
|
<span style="color: #CD5555">"""</span>
|
|
<span style="color: #CD5555">mnist_loader</span>
|
|
<span style="color: #CD5555">~~~~~~~~~~~~</span>
|
|
|
|
<span style="color: #CD5555">A library to load the MNIST image data. For details of the data</span>
|
|
<span style="color: #CD5555">structures that are returned, see the doc strings for ``load_data``</span>
|
|
<span style="color: #CD5555">and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the</span>
|
|
<span style="color: #CD5555">function usually called by our neural network code.</span>
|
|
<span style="color: #CD5555">"""</span>
|
|
|
|
<span style="color: #228B22">#### Libraries</span>
|
|
<span style="color: #228B22"># Standard library</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">gzip</span>
|
|
|
|
<span style="color: #228B22"># Third-party libraries</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_data</span>():
|
|
<span style="color: #CD5555">"""Return the MNIST data as a tuple containing the training data,</span>
|
|
<span style="color: #CD5555"> the validation data, and the test data.</span>
|
|
|
|
<span style="color: #CD5555"> The ``training_data`` is returned as a tuple with two entries.</span>
|
|
<span style="color: #CD5555"> The first entry contains the actual training images. This is a</span>
|
|
<span style="color: #CD5555"> numpy ndarray with 50,000 entries. Each entry is, in turn, a</span>
|
|
<span style="color: #CD5555"> numpy ndarray with 784 values, representing the 28 * 28 = 784</span>
|
|
<span style="color: #CD5555"> pixels in a single MNIST image.</span>
|
|
|
|
<span style="color: #CD5555"> The second entry in the ``training_data`` tuple is a numpy ndarray</span>
|
|
<span style="color: #CD5555"> containing 50,000 entries. Those entries are just the digit</span>
|
|
<span style="color: #CD5555"> values (0...9) for the corresponding images contained in the first</span>
|
|
<span style="color: #CD5555"> entry of the tuple.</span>
|
|
|
|
<span style="color: #CD5555"> The ``validation_data`` and ``test_data`` are similar, except</span>
|
|
<span style="color: #CD5555"> each contains only 10,000 images.</span>
|
|
|
|
<span style="color: #CD5555"> This is a nice data format, but for use in neural networks it's</span>
|
|
<span style="color: #CD5555"> helpful to modify the format of the ``training_data`` a little.</span>
|
|
<span style="color: #CD5555"> That's done in the wrapper function ``load_data_wrapper()``, see</span>
|
|
<span style="color: #CD5555"> below.</span>
|
|
<span style="color: #CD5555"> """</span>
|
|
f = gzip.open(<span style="color: #CD5555">'../data/mnist.pkl.gz'</span>, <span style="color: #CD5555">'rb'</span>)
|
|
training_data, validation_data, test_data = cPickle.load(f)
|
|
f.close()
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (training_data, validation_data, test_data)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_data_wrapper</span>():
|
|
<span style="color: #CD5555">"""Return a tuple containing ``(training_data, validation_data,</span>
|
|
<span style="color: #CD5555"> test_data)``. Based on ``load_data``, but the format is more</span>
|
|
<span style="color: #CD5555"> convenient for use in our implementation of neural networks.</span>
|
|
|
|
<span style="color: #CD5555"> In particular, ``training_data`` is a list containing 50,000</span>
|
|
<span style="color: #CD5555"> 2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray</span>
|
|
<span style="color: #CD5555"> containing the input image. ``y`` is a 10-dimensional</span>
|
|
<span style="color: #CD5555"> numpy.ndarray representing the unit vector corresponding to the</span>
|
|
<span style="color: #CD5555"> correct digit for ``x``.</span>
|
|
|
|
<span style="color: #CD5555"> ``validation_data`` and ``test_data`` are lists containing 10,000</span>
|
|
<span style="color: #CD5555"> 2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional</span>
|
|
<span style="color: #CD5555"> numpy.ndarry containing the input image, and ``y`` is the</span>
|
|
<span style="color: #CD5555"> corresponding classification, i.e., the digit values (integers)</span>
|
|
<span style="color: #CD5555"> corresponding to ``x``.</span>
|
|
|
|
<span style="color: #CD5555"> Obviously, this means we're using slightly different formats for</span>
|
|
<span style="color: #CD5555"> the training data and the validation / test data. These formats</span>
|
|
<span style="color: #CD5555"> turn out to be the most convenient for use in our neural network</span>
|
|
<span style="color: #CD5555"> code."""</span>
|
|
tr_d, va_d, te_d = load_data()
|
|
training_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> tr_d[<span style="color: #B452CD">0</span>]]
|
|
training_results = [vectorized_result(y) <span style="color: #8B008B; font-weight: bold">for</span> y <span style="color: #8B008B">in</span> tr_d[<span style="color: #B452CD">1</span>]]
|
|
training_data = <span style="color: #658b00">zip</span>(training_inputs, training_results)
|
|
validation_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> va_d[<span style="color: #B452CD">0</span>]]
|
|
validation_data = <span style="color: #658b00">zip</span>(validation_inputs, va_d[<span style="color: #B452CD">1</span>])
|
|
test_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> te_d[<span style="color: #B452CD">0</span>]]
|
|
test_data = <span style="color: #658b00">zip</span>(test_inputs, te_d[<span style="color: #B452CD">1</span>])
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (training_data, validation_data, test_data)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">vectorized_result</span>(j):
|
|
<span style="color: #CD5555">"""Return a 10-dimensional unit vector with a 1.0 in the jth</span>
|
|
<span style="color: #CD5555"> position and zeroes elsewhere. This is used to convert a digit</span>
|
|
<span style="color: #CD5555"> (0...9) into a corresponding desired output from the neural</span>
|
|
<span style="color: #CD5555"> network."""</span>
|
|
e = np.zeros((<span style="color: #B452CD">10</span>, <span style="color: #B452CD">1</span>))
|
|
e[j] = <span style="color: #B452CD">1.0</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> e
|
|
|
|
net=network.Network([<span style="color: #B452CD">784</span>,<span style="color: #B452CD">30</span>,<span style="color: #B452CD">30</span>])
|
|
net.SGD(training_data,<span style="color: #B452CD">30</span>,<span style="color: #B452CD">10</span>,<span style="color: #B452CD">3</span>,test_data=test_data)
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|