mhjensen
1250 lines
66 KiB
HTML
1250 lines
66 KiB
HTML
\
|
|
<!DOCTYPE html>
|
|
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Elements of machine learning">
|
|
|
|
<title>Data Analysis and Machine Learning: Elements of machine learning</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- reveal.js: http://lab.hakim.se/reveal-js/ -->
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
|
|
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
|
|
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<!--
|
|
<link rel="stylesheet" href="reveal.js/css/reveal.css">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
|
|
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
|
|
-->
|
|
|
|
<!-- For syntax highlighting -->
|
|
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
|
|
|
|
<!-- Printing and PDF exports -->
|
|
<script>
|
|
var link = document.createElement( 'link' );
|
|
link.rel = 'stylesheet';
|
|
link.type = 'text/css';
|
|
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
|
|
document.getElementsByTagName( 'head' )[0].appendChild( link );
|
|
</script>
|
|
|
|
<style type="text/css">
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
.reveal .alert-text-small { font-size: 80%; }
|
|
.reveal .alert-text-large { font-size: 130%; }
|
|
.reveal .alert-text-normal { font-size: 90%; }
|
|
.reveal .alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:5px solid #bababa;
|
|
-webkit-border-radius: 14px; -moz-border-radius: 14px;
|
|
border-radius:14px;
|
|
background-position: 10px 10px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 30px; /* 55px; if icon */
|
|
}
|
|
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
|
|
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
/*.reveal .alert li {margin-top: 1em}*/
|
|
.reveal .alert-block p+p {margin-top:5px}
|
|
/*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
|
|
.reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
|
|
.reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
|
|
.reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
|
|
|
|
</style>
|
|
|
|
|
|
|
|
<!-- Styles for table layout of slides -->
|
|
<style type="text/css">
|
|
td.padding {
|
|
padding-top:20px;
|
|
padding-bottom:20px;
|
|
padding-right:50px;
|
|
padding-left:50px;
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
|
|
<body>
|
|
<div class="reveal">
|
|
|
|
<!-- Any section element inside the <div class="slides"> container
|
|
is displayed as a slide -->
|
|
|
|
<div class="slides">
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<section>
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1 style="text-align: center;">Data Analysis and Machine Learning: Elements of machine learning</h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p> <br>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p> <br>
|
|
<center><h4>May 30, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec0">What is Machine Learning? </h2>
|
|
|
|
<p>
|
|
Machine learning is the science of giving computers the ability to
|
|
learn without being explicitly programmed. The idea is that there
|
|
exist generic algorithms which can be used to find patterns in a broad
|
|
class of data sets without having to write code specifically for each
|
|
problem. The algorithm will build its own logic based on the data.
|
|
|
|
<p>
|
|
Machine learning is a subfield of computer science, and is closely
|
|
related to computational statistics. It evolved from the study of
|
|
pattern recognition in artificial intelligence (AI) research, and has
|
|
made contributions to AI tasks like computer vision, natural language
|
|
processing and speech recognition. It has also, especially in later
|
|
years, found applications in a wide variety of other areas, including
|
|
bioinformatics, economy, physics, finance and marketing.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec1">Types of Machine Learning </h2>
|
|
|
|
<p>
|
|
The approaches to machine learning are many, but are often split into two main categories.
|
|
In <em>supervised learning</em> we know the answer to a problem,
|
|
and let the computer deduce the logic behind it. On the other hand, <em>unsupervised learning</em>
|
|
is a method for finding patterns and relationship in data sets without any prior knowledge of the system.
|
|
Some authours also operate with a third category, namely <em>reinforcement learning</em>. This is a paradigm
|
|
of learning inspired by behavioural psychology, where learning is achieved by trial-and-error,
|
|
solely from rewards and punishment.
|
|
|
|
<p>
|
|
Another way to categorize machine learning tasks is to consider the desired output of a system.
|
|
Some of the most common tasks are:
|
|
|
|
<ul>
|
|
|
|
<p><li> Classification: Outputs are divided into two or more classes. The goal is to produce a model that assigns inputs into one of these classes. An example is to identify digits based on pictures of hand-written ones. Classification is typically supervised learning.</li>
|
|
|
|
<p><li> Regression: Finding a functional relationship between an input data set and a reference data set. The goal is to construct a function that maps input data to continuous output values.</li>
|
|
|
|
<p><li> Clustering: Data are divided into groups with certain common traits, without knowing the different groups beforehand. It is thus a form of unsupervised learning.</li>
|
|
</ul>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec2">Artificial neurons </h2>
|
|
|
|
<p>
|
|
The field of artificial neural networks has a long history of
|
|
development, and is closely connected with the advancement of computer
|
|
science and computers in general. A model of artificial neurons was
|
|
first developed by McCulloch and Pitts in 1943 to study signal
|
|
processing in the brain and has later been refined by others. The
|
|
general idea is to mimic neural networks in the human brain, which is
|
|
composed of billions of neurons that communicate with each other by
|
|
sending electrical signals. Each neuron accumulates its incoming
|
|
signals, which must exceed an activation threshold to yield an
|
|
output. If the threshold is not overcome, the neuron remains inactive,
|
|
i.e. has zero output.
|
|
|
|
<p>
|
|
This behaviour has inspired a simple mathematical model for an artificial neuron.
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
|
|
\tag{1}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
Here, the output \( y \) of the neuron is the value of its activation function, which have as input
|
|
a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
|
|
|
|
<p>
|
|
Conceptually, it is helpful to divide neural networks into four
|
|
categories:
|
|
|
|
<ol>
|
|
<p><li> general purpose neural networks for supervised learning,</li>
|
|
<p><li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
|
|
<p><li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
|
|
<p><li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
|
|
</ol>
|
|
<p>
|
|
|
|
In physics, DNNs and CNNs have already found numerous applications. In
|
|
statistical physics, they have been applied to detect phase
|
|
transitions in 2D Ising and Potts models, lattice gauge theories, and
|
|
different phases of polymers.
|
|
Deep learning has also found interesting applications in quantum
|
|
physics. Various quantum phase transitions can be detected and studied
|
|
using DNNs and CNNs, including the transverse-field Ising model,
|
|
topological phases, and even non-equilibrium many-body
|
|
localization. Representing quantum states as DNNs quantum state
|
|
tomography are among some of the impressive
|
|
achievements to reveal the potential of DNNs to facilitate the study
|
|
of quantum systems.
|
|
|
|
<p>
|
|
In quantum information theory, it has been shown that one can perform
|
|
gate decompositions with the help of neural. In lattice quantum chromodynamics,
|
|
DNNs have been used to learn action parameters in regions of parameter
|
|
space where PCA fails. Last but not least,
|
|
DNNs also found place in the study of quantum, and in scattering theory to learn
|
|
\( s \)-wave scattering length of potentials.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec3">Neural network types </h2>
|
|
|
|
<p>
|
|
An artificial neural network (NN), is a computational model that
|
|
consists of layers of connected neurons, or <em>nodes</em>. It is supposed
|
|
to mimic a biological nervous system by letting each neuron interact
|
|
with other neurons by sending signals in the form of mathematical
|
|
functions between layers. A wide variety of different NNs have been
|
|
developed, but most of them consist of an input layer, an output layer
|
|
and eventual layers in-between, called <em>hidden layers</em>. All layers can
|
|
contain an arbitrary number of nodes, and each connection between two
|
|
nodes is associated with a weight variable.
|
|
|
|
<p>
|
|
Neural networks (also called neural nets) are neural-inspired
|
|
nonlinear models for supervised learning. As we will see, neural nets
|
|
can be viewed as natural, more powerful extensions of supervised
|
|
learning methods such as linear and logistic regression and soft-max
|
|
methods.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec4">Feed-forward neural networks </h2>
|
|
|
|
<p>
|
|
The feed-forward neural network (FFNN) was the first and simplest type of NN devised. In this network,
|
|
the information moves in only one direction: forward through the layers.
|
|
|
|
<p>
|
|
Nodes are represented by circles, while the arrows display the connections between the nodes, including the
|
|
direction of information flow. Additionally, each arrow corresponds to a weight variable, not displayed here.
|
|
We observe that each node in a layer is connected to <em>all</em> nodes in the subsequent layer,
|
|
making this a so-called <em>fully-connected</em> FFNN.
|
|
|
|
<p>
|
|
A different variant of FFNNs are <em>convolutional neural networks</em> (CNNs), which have a connectivity pattern
|
|
inspired by the animal visual cortex. Individual neurons in the visual cortex only respond to stimuli from
|
|
small sub-regions of the visual field, called a receptive field. This makes the neurons well-suited to exploit the strong
|
|
spatially local correlation present in natural images. The response of each neuron can be approximated mathematically
|
|
as a convolution operation.
|
|
|
|
<p>
|
|
CNNs emulate the behaviour of neurons in the visual cortex by enforcing a <em>local</em> connectivity pattern
|
|
between nodes of adjacent layers: Each node
|
|
in a convolutional layer is connected only to a subset of the nodes in the previous layer,
|
|
in contrast to the fully-connected FFNN.
|
|
Often, CNNs
|
|
consist of several convolutional layers that learn local features of the input, with a fully-connected layer at the end,
|
|
which gathers all the local data and produces the outputs. They have wide applications in image and video recognition
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec5">Recurrent neural networks </h2>
|
|
|
|
<p>
|
|
So far we have only mentioned NNs where information flows in one direction: forward. <em>Recurrent neural networks</em> on
|
|
the other hand, have connections between nodes that form directed <em>cycles</em>. This creates a form of
|
|
internal memory which are able to capture information on what has been calculated before; the output is dependent
|
|
on the previous computations. Recurrent NNs make use of sequential information by performing the same task for
|
|
every element in a sequence, where each element depends on previous elements. An example of such information is
|
|
sentences, making recurrent NNs especially well-suited for handwriting and speech recognition.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec6">Other types of networks </h2>
|
|
|
|
<p>
|
|
There are many other kinds of NNs that have been developed. One type that is specifically designed for interpolation
|
|
in multidimensional space is the radial basis function (RBF) network. RBFs are typically made up of three layers:
|
|
an input layer, a hidden layer with non-linear radial symmetric activation functions and a linear output layer (''linear'' here
|
|
means that each node in the output layer has a linear activation function). The layers are normally fully-connected and
|
|
there are no cycles, thus RBFs can be viewed as a type of fully-connected FFNN. They are however usually treated as
|
|
a separate type of NN due the unusual activation functions.
|
|
|
|
<p>
|
|
Other types of NNs could also be mentioned, but are outside the scope of this work. We will now move on to a detailed description
|
|
of how a fully-connected FFNN works, and how it can be used to interpolate data sets.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec7">Multilayer perceptrons </h2>
|
|
|
|
<p>
|
|
One use often so-called fully-connected feed-forward neural networks with three
|
|
or more layers (an input layer, one or more hidden layers and an output layer)
|
|
consisting of neurons that have non-linear activation functions.
|
|
|
|
<p>
|
|
Such networks are often called <em>multilayer perceptrons</em> (MLPs)
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec8">Why multilayer perceptrons? </h2>
|
|
|
|
<p>
|
|
According to the <em>Universal approximation theorem</em>, a feed-forward neural network with just a single hidden layer containing
|
|
a finite number of neurons can approximate a continuous multidimensional function to arbitrary accuracy,
|
|
assuming the activation function for the hidden layer is a <b>non-constant, bounded and monotonically-increasing continuous function</b>.
|
|
Note that the requirements on the activation function only applies to the hidden layer, the output nodes are always
|
|
assumed to be linear, so as to not restrict the range of output values.
|
|
|
|
<p>
|
|
We note that this theorem is only applicable to a NN with <em>one</em> hidden layer.
|
|
Therefore, we can easily construct an NN
|
|
that employs activation functions which do not satisfy the above requirements, as long as we have at least one layer
|
|
with activation functions that <em>do</em>. Furthermore, although the universal approximation theorem
|
|
lays the theoretical foundation for regression with neural networks, it does not say anything about how things work in practice:
|
|
A neural network can still be able to approximate a given function reasonably well without having the flexibility to fit <em>all other</em>
|
|
functions.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec9">Mathematical model </h2>
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(u)
|
|
\tag{2}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
In an FFNN of such neurons, the <em>inputs</em> \( x_i \)
|
|
are the <em>outputs</em> of the neurons in the preceding layer. Furthermore, an MLP is fully-connected,
|
|
which means that each neuron receives a weighted sum of the outputs of <em>all</em> neurons in the previous layer.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec10">Mathematical model </h2>
|
|
|
|
<p>
|
|
First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( u_i^1 \) of the input coordinates \( x_j \),
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
u_i^1 = \sum_{j=1}^2 w_{ij}^1 x_j + b_i^1
|
|
\tag{3}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
This value is the argument to the activation function \( f_1 \) of each neuron \( i \),
|
|
producing the output \( y_i^1 \) of all neurons in layer 1,
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
y_i^1 = f_1(u_i^1) = f_1\left(\sum_{j=1}^2 w_{ij}^1 x_j + b_i^1\right)
|
|
\tag{4}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
where we assume that all nodes in the same layer have identical activation functions, hence the notation \( f_l \)
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
y_i^l = f_l(u_i^l) = f_l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
|
|
\tag{5}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
where \( N_l \) is the number of nodes in layer \( l \). When the output of all the nodes in the first hidden layer are computed,
|
|
the values of the subsequent layer can be calculated and so forth until the output is obtained.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec11">Mathematical model </h2>
|
|
|
|
<p>
|
|
The output of neuron \( i \) in layer 2 is thus,
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{align}
|
|
y_i^2 &= f_2\left(\sum_{j=1}^3 w_{ij}^2 y_j^1 + b_i^2\right)
|
|
\tag{6}\\
|
|
&= f_2\left[\sum_{j=1}^3 w_{ij}^2f_1\left(\sum_{k=1}^2 w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
|
|
\tag{7}
|
|
\end{align}
|
|
$$
|
|
<p> <br>
|
|
|
|
where we have substituted \( y_m^1 \) with. Finally, the NN output yields,
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{align}
|
|
y_1^3 &= f_3\left(\sum_{j=1}^3 w_{1m}^3 y_j^2 + b_1^3\right)
|
|
\tag{8}\\
|
|
&= f_3\left[\sum_{j=1}^3 w_{1j}^3 f_2\left(\sum_{k=1}^3 w_{jk}^2 f_1\left(\sum_{m=1}^2 w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
|
|
+ b_1^3\right]
|
|
\tag{9}
|
|
\end{align}
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec12">Mathematical model </h2>
|
|
|
|
<p>
|
|
We can generalize this expression to an MLP with \( l \) hidden layers. The complete functional form
|
|
is,
|
|
<p> <br>
|
|
$$
|
|
\begin{align}
|
|
&y^{l+1}_1\! = \!f_{l+1}\!\left[\!\sum_{j=1}^{N_l}\! w_{1j}^3 f_l\!\left(\!\sum_{k=1}^{N_{l-1}}\! w_{jk}^2 f_{l-1}\!\left(\!
|
|
\dots \!f_1\!\left(\!\sum_{n=1}^{N_0} \!w_{mn}^1 x_n\! + \!b_m^1\!\right)
|
|
\!\dots \!\right) \!+ \!b_k^2\!\right)
|
|
\!+ \!b_1^3\!\right] &&
|
|
\tag{10}
|
|
\end{align}
|
|
$$
|
|
<p> <br>
|
|
|
|
which illustrates a basic property of MLPs: The only independent variables are the input values \( x_n \).
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec13">Mathematical model </h2>
|
|
|
|
<p>
|
|
This confirms that an MLP,
|
|
despite its quite convoluted mathematical form, is nothing more than an analytic function, specifically a
|
|
mapping of real-valued vectors \( \vec{x} \in \mathbb{R}^n \rightarrow \vec{y} \in \mathbb{R}^m \).
|
|
In our example, \( n=2 \) and \( m=1 \). Consequentially,
|
|
the number of input and output values of the function we want to fit must be equal to the number of inputs and outputs of our MLP.
|
|
|
|
<p>
|
|
Furthermore, the flexibility and universality of a MLP can be illustrated by realizing that
|
|
the expression is essentially a nested sum of scaled activation functions of the form
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
h(x) = c_1 f(c_2 x + c_3) + c_4
|
|
\tag{11}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
where the parameters \( c_i \) are weights and biases. By adjusting these parameters, the activation functions
|
|
can be shifted up and down or left and right, change slope or be rescaled
|
|
which is the key to the flexibility of a neural network.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
|
|
<h3 id="___sec14">Matrix-vector notation </h3>
|
|
|
|
We can introduce a more convenient notation for the activations in a NN.
|
|
|
|
<p>
|
|
Additionally, we can represent the biases and activations
|
|
as layer-wise column vectors \( \vec{b}_l \) and \( \vec{y}_l \), so that the \( i \)-th element of each vector
|
|
is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively.
|
|
|
|
<p>
|
|
We have that \( \mathrm{W}_l \) is a \( N_{l-1} \times N_l \) matrix, while \( \vec{b}_l \) and \( \vec{y}_l \) are \( N_l \times 1 \) column vectors.
|
|
With this notation, the sum in becomes a matrix-vector multiplication, and we can write
|
|
the equation for the activations of hidden layer 2 in
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
\vec{y}_2 = f_2(\mathrm{W}_2 \vec{y}_{1} + \vec{b}_{2}) =
|
|
f_2\left(\left[\begin{array}{ccc}
|
|
w^2_{11} &w^2_{12} &w^2_{13} \\
|
|
w^2_{21} &w^2_{22} &w^2_{23} \\
|
|
w^2_{31} &w^2_{32} &w^2_{33} \\
|
|
\end{array} \right] \cdot
|
|
\left[\begin{array}{c}
|
|
y^1_1 \\
|
|
y^1_2 \\
|
|
y^1_3 \\
|
|
\end{array}\right] +
|
|
\left[\begin{array}{c}
|
|
b^2_1 \\
|
|
b^2_2 \\
|
|
b^2_3 \\
|
|
\end{array}\right]\right).
|
|
\tag{12}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
|
|
<h3 id="___sec15">Matrix-vector notation and activation </h3>
|
|
|
|
The activation of node \( i \) in layer 2 is
|
|
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) =
|
|
f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
|
|
\tag{13}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
This is not just a convenient and compact notation, but also
|
|
a useful and intuitive way to think about MLPs: The output is calculated by a series of matrix-vector multiplications
|
|
and vector additions that are used as input to the activation functions. For each operation
|
|
\( \mathrm{W}_l \vec{y}_{l-1} \) we move forward one layer.
|
|
</section>
|
|
|
|
|
|
<section>
|
|
|
|
<h3 id="___sec16">Activation functions </h3>
|
|
|
|
<p>
|
|
A property that characterizes a neural network, other than its connectivity, is the choice of activation function(s).
|
|
As described in, the following restrictions are imposed on an activation function for a FFNN
|
|
to fulfill the universal approximation theorem
|
|
|
|
<ul>
|
|
|
|
<p><li> Non-constant</li>
|
|
|
|
<p><li> Bounded</li>
|
|
|
|
<p><li> Monotonically-increasing</li>
|
|
|
|
<p><li> Continuous</li>
|
|
</ul>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
|
|
<h3 id="___sec17">Activation functions, Logistic and Hyperbolic ones </h3>
|
|
|
|
<p>
|
|
The second requirement excludes all linear functions. Furthermore, in a MLP with only linear activation functions, each
|
|
layer simply performs a linear transformation of its inputs.
|
|
|
|
<p>
|
|
Regardless of the number of layers,
|
|
the output of the NN will be nothing but a linear function of the inputs. Thus we need to introduce some kind of
|
|
non-linearity to the NN to be able to fit non-linear functions
|
|
Typical examples are the logistic <em>Sigmoid</em>
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
f(x) = \frac{1}{1 + e^{-x}},
|
|
\tag{14}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
|
|
and the <em>hyperbolic tangent</em> function
|
|
<p> <br>
|
|
$$
|
|
\begin{equation}
|
|
f(x) = \tanh(x)
|
|
\tag{15}
|
|
\end{equation}
|
|
$$
|
|
<p> <br>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
|
|
<h3 id="___sec18">Relevance </h3>
|
|
|
|
The <em>sigmoid</em> function are more biologically plausible because
|
|
the output of inactive neurons are zero. Such activation function are called <em>one-sided</em>. However,
|
|
it has been shown that the hyperbolic tangent
|
|
performs better than the sigmoid for training MLPs.
|
|
has become the most popular for <em>deep neural networks</em>
|
|
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #CD5555">"""The sigmoid function (or the logistic curve) is a </span>
|
|
<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
|
|
<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
|
|
<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm."""</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
|
|
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">1</span>)
|
|
sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
|
|
sigma = sigma_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, sigma)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'sigmoid function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Step Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, .<span style="color: #B452CD">02</span>)
|
|
step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z >= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
|
|
step = step_fn(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, step)
|
|
ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'step function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Sine Function"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
|
|
t = numpy.sin(z)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, t)
|
|
ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'sine function'</span>)
|
|
|
|
plt.show()
|
|
|
|
<span style="color: #CD5555">"""Plots a graph of the squashing function used by a rectified linear</span>
|
|
<span style="color: #CD5555">unit"""</span>
|
|
z = numpy.arange(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>, .<span style="color: #B452CD">1</span>)
|
|
zero = numpy.zeros(<span style="color: #658b00">len</span>(z))
|
|
y = numpy.max([zero, z], axis=<span style="color: #B452CD">0</span>)
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
|
|
ax.plot(z, y)
|
|
ax.set_ylim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
|
|
ax.set_xlim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
|
|
ax.grid(<span style="color: #658b00">True</span>)
|
|
ax.set_xlabel(<span style="color: #CD5555">'z'</span>)
|
|
ax.set_title(<span style="color: #CD5555">'Rectified linear unit'</span>)
|
|
|
|
plt.show()
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">scipy</span> <span style="color: #8B008B; font-weight: bold">import</span> optimize
|
|
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">Neural_Network</span>(<span style="color: #658b00">object</span>):
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">__init__</span>(<span style="color: #658b00">self</span>, Lambda=<span style="color: #B452CD">0</span>):
|
|
<span style="color: #228B22">#Define Hyperparameters</span>
|
|
<span style="color: #658b00">self</span>.inputLayerSize = <span style="color: #B452CD">2</span>
|
|
<span style="color: #658b00">self</span>.outputLayerSize = <span style="color: #B452CD">1</span>
|
|
<span style="color: #658b00">self</span>.hiddenLayerSize = <span style="color: #B452CD">3</span>
|
|
|
|
<span style="color: #228B22">#Weights (parameters)</span>
|
|
<span style="color: #658b00">self</span>.W1 = np.random.randn(<span style="color: #658b00">self</span>.inputLayerSize,<span style="color: #658b00">self</span>.hiddenLayerSize)
|
|
<span style="color: #658b00">self</span>.W2 = np.random.randn(<span style="color: #658b00">self</span>.hiddenLayerSize,<span style="color: #658b00">self</span>.outputLayerSize)
|
|
|
|
<span style="color: #228B22">#Regularization Parameter:</span>
|
|
<span style="color: #658b00">self</span>.Lambda = Lambda
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">forward</span>(<span style="color: #658b00">self</span>, X):
|
|
<span style="color: #228B22">#Propogate inputs though network</span>
|
|
<span style="color: #658b00">self</span>.z2 = np.dot(X, <span style="color: #658b00">self</span>.W1)
|
|
<span style="color: #658b00">self</span>.a2 = <span style="color: #658b00">self</span>.sigmoid(<span style="color: #658b00">self</span>.z2)
|
|
<span style="color: #658b00">self</span>.z3 = np.dot(<span style="color: #658b00">self</span>.a2, <span style="color: #658b00">self</span>.W2)
|
|
yHat = <span style="color: #658b00">self</span>.sigmoid(<span style="color: #658b00">self</span>.z3)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> yHat
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(<span style="color: #658b00">self</span>, z):
|
|
<span style="color: #228B22">#Apply sigmoid activation function to scalar, vector, or matrix</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+np.exp(-z))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoidPrime</span>(<span style="color: #658b00">self</span>,z):
|
|
<span style="color: #228B22">#Gradient of sigmoid</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.exp(-z)/((<span style="color: #B452CD">1</span>+np.exp(-z))**<span style="color: #B452CD">2</span>)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunction</span>(<span style="color: #658b00">self</span>, X, y):
|
|
<span style="color: #228B22">#Compute cost for given X,y, use weights already stored in class.</span>
|
|
<span style="color: #658b00">self</span>.yHat = <span style="color: #658b00">self</span>.forward(X)
|
|
J = <span style="color: #B452CD">0.5</span>*<span style="color: #658b00">sum</span>((y-<span style="color: #658b00">self</span>.yHat)**<span style="color: #B452CD">2</span>)/X.shape[<span style="color: #B452CD">0</span>] + (<span style="color: #658b00">self</span>.Lambda/<span style="color: #B452CD">2</span>)*(np.sum(<span style="color: #658b00">self</span>.W1**<span style="color: #B452CD">2</span>)+np.sum(<span style="color: #658b00">self</span>.W2**<span style="color: #B452CD">2</span>))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> J
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunctionPrime</span>(<span style="color: #658b00">self</span>, X, y):
|
|
<span style="color: #228B22">#Compute derivative with respect to W and W2 for a given X and y:</span>
|
|
<span style="color: #658b00">self</span>.yHat = <span style="color: #658b00">self</span>.forward(X)
|
|
|
|
delta3 = np.multiply(-(y-<span style="color: #658b00">self</span>.yHat), <span style="color: #658b00">self</span>.sigmoidPrime(<span style="color: #658b00">self</span>.z3))
|
|
<span style="color: #228B22">#Add gradient of regularization term:</span>
|
|
dJdW2 = np.dot(<span style="color: #658b00">self</span>.a2.T, delta3)/X.shape[<span style="color: #B452CD">0</span>] + <span style="color: #658b00">self</span>.Lambda*<span style="color: #658b00">self</span>.W2
|
|
|
|
delta2 = np.dot(delta3, <span style="color: #658b00">self</span>.W2.T)*<span style="color: #658b00">self</span>.sigmoidPrime(<span style="color: #658b00">self</span>.z2)
|
|
<span style="color: #228B22">#Add gradient of regularization term:</span>
|
|
dJdW1 = np.dot(X.T, delta2)/X.shape[<span style="color: #B452CD">0</span>] + <span style="color: #658b00">self</span>.Lambda*<span style="color: #658b00">self</span>.W1
|
|
|
|
<span style="color: #8B008B; font-weight: bold">return</span> dJdW1, dJdW2
|
|
|
|
<span style="color: #228B22">#Helper functions for interacting with other methods/classes</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">getParams</span>(<span style="color: #658b00">self</span>):
|
|
<span style="color: #228B22">#Get W1 and W2 Rolled into vector:</span>
|
|
params = np.concatenate((<span style="color: #658b00">self</span>.W1.ravel(), <span style="color: #658b00">self</span>.W2.ravel()))
|
|
<span style="color: #8B008B; font-weight: bold">return</span> params
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">setParams</span>(<span style="color: #658b00">self</span>, params):
|
|
<span style="color: #228B22">#Set W1 and W2 using single parameter vector:</span>
|
|
W1_start = <span style="color: #B452CD">0</span>
|
|
W1_end = <span style="color: #658b00">self</span>.hiddenLayerSize*<span style="color: #658b00">self</span>.inputLayerSize
|
|
<span style="color: #658b00">self</span>.W1 = np.reshape(params[W1_start:W1_end], \
|
|
(<span style="color: #658b00">self</span>.inputLayerSize, <span style="color: #658b00">self</span>.hiddenLayerSize))
|
|
W2_end = W1_end + <span style="color: #658b00">self</span>.hiddenLayerSize*<span style="color: #658b00">self</span>.outputLayerSize
|
|
<span style="color: #658b00">self</span>.W2 = np.reshape(params[W1_end:W2_end], \
|
|
(<span style="color: #658b00">self</span>.hiddenLayerSize, <span style="color: #658b00">self</span>.outputLayerSize))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">computeGradients</span>(<span style="color: #658b00">self</span>, X, y):
|
|
dJdW1, dJdW2 = <span style="color: #658b00">self</span>.costFunctionPrime(X, y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> np.concatenate((dJdW1.ravel(), dJdW2.ravel()))
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">trainer</span>(<span style="color: #658b00">object</span>):
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">__init__</span>(<span style="color: #658b00">self</span>, N):
|
|
<span style="color: #228B22">#Make Local reference to network:</span>
|
|
<span style="color: #658b00">self</span>.N = N
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">callbackF</span>(<span style="color: #658b00">self</span>, params):
|
|
<span style="color: #658b00">self</span>.N.setParams(params)
|
|
<span style="color: #658b00">self</span>.J.append(<span style="color: #658b00">self</span>.N.costFunction(<span style="color: #658b00">self</span>.X, <span style="color: #658b00">self</span>.y))
|
|
<span style="color: #658b00">self</span>.testJ.append(<span style="color: #658b00">self</span>.N.costFunction(<span style="color: #658b00">self</span>.testX, <span style="color: #658b00">self</span>.testY))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">costFunctionWrapper</span>(<span style="color: #658b00">self</span>, params, X, y):
|
|
<span style="color: #658b00">self</span>.N.setParams(params)
|
|
cost = <span style="color: #658b00">self</span>.N.costFunction(X, y)
|
|
grad = <span style="color: #658b00">self</span>.N.computeGradients(X,y)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> cost, grad
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train</span>(<span style="color: #658b00">self</span>, trainX, trainY, testX, testY):
|
|
<span style="color: #228B22">#Make an internal variable for the callback function:</span>
|
|
<span style="color: #658b00">self</span>.X = trainX
|
|
<span style="color: #658b00">self</span>.y = trainY
|
|
|
|
<span style="color: #658b00">self</span>.testX = testX
|
|
<span style="color: #658b00">self</span>.testY = testY
|
|
|
|
<span style="color: #228B22">#Make empty list to store training costs:</span>
|
|
<span style="color: #658b00">self</span>.J = []
|
|
<span style="color: #658b00">self</span>.testJ = []
|
|
|
|
params0 = <span style="color: #658b00">self</span>.N.getParams()
|
|
|
|
options = {<span style="color: #CD5555">'maxiter'</span>: <span style="color: #B452CD">200</span>, <span style="color: #CD5555">'disp'</span> : <span style="color: #658b00">True</span>}
|
|
_res = optimize.minimize(<span style="color: #658b00">self</span>.costFunctionWrapper, params0, jac=<span style="color: #658b00">True</span>, method=<span style="color: #CD5555">'BFGS'</span>, \
|
|
args=(trainX, trainY), options=options, callback=<span style="color: #658b00">self</span>.callbackF)
|
|
|
|
<span style="color: #658b00">self</span>.N.setParams(_res.x)
|
|
<span style="color: #658b00">self</span>.optimizationResults = _res
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
<section>
|
|
<h2 id="___sec19">Two-layer Neural Network </h2>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
<span style="color: #228B22">#sigmoid</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">nonlin</span>(x, deriv=<span style="color: #658b00">False</span>):
|
|
<span style="color: #8B008B; font-weight: bold">if</span> (deriv==<span style="color: #658b00">True</span>):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> x*(<span style="color: #B452CD">1</span>-x)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+np.exp(-x))
|
|
|
|
<span style="color: #228B22">#input data</span>
|
|
x=np.array([[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>],[<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>]])
|
|
|
|
<span style="color: #228B22">#output data</span>
|
|
y=np.array([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">0</span>]).T
|
|
|
|
<span style="color: #228B22">#seed random numbers to make calculation</span>
|
|
np.random.seed(<span style="color: #B452CD">1</span>)
|
|
|
|
<span style="color: #228B22">#initialize weights with mean=0</span>
|
|
syn0=<span style="color: #B452CD">2</span>*np.random.random((<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>))-<span style="color: #B452CD">1</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">10000</span>):
|
|
<span style="color: #228B22">#forward propogation</span>
|
|
l0=x
|
|
l1=nonlin(np.dot(l0,syn0))
|
|
l1_error=y-l1
|
|
<span style="color: #228B22">#multiply error by slope of sigmoid at values of l1</span>
|
|
l1_delta=l1_error*nonlin(l1,<span style="color: #658b00">True</span>)
|
|
<span style="color: #228B22">#update weights</span>
|
|
syn0+=np.dot(l0.T, l1_delta)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"Output after training: "</span>,l1 )
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">random</span>
|
|
<span style="color: #8B008B; font-weight: bold">class</span> <span style="color: #008b45; font-weight: bold">Network</span>(<span style="color: #658b00">object</span>):
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">_init_</span>(<span style="color: #658b00">self</span>, sizes):
|
|
<span style="color: #658b00">self</span>.num_layers=<span style="color: #658b00">len</span>(sizes)
|
|
<span style="color: #658b00">self</span>.sizes=sizes
|
|
<span style="color: #658b00">self</span>.biases=[np.random.randn(y,<span style="color: #B452CD">1</span>) <span style="color: #8B008B; font-weight: bold">for</span> y <span style="color: #8B008B">in</span> sizes[<span style="color: #B452CD">1</span>:]]
|
|
<span style="color: #658b00">self</span>.weights=[np.random.randn(y,x) <span style="color: #8B008B; font-weight: bold">for</span> x,y <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(sizes[:-<span style="color: #B452CD">1</span>], sizes[<span style="color: #B452CD">1</span>:])]
|
|
|
|
<span style="color: #228B22">#sizes is the number of neurons in each layer</span>
|
|
<span style="color: #228B22">#for example, say n_1st_layer=3, n_2nd_layer=3, n_3rd_layer=1, then net=Network([3,3,1])</span>
|
|
|
|
<span style="color: #228B22">#The biases and weights are initialized randomly, using Gaussian distributions of mean=0, stdev=1</span>
|
|
<span style="color: #228B22">#z is a vector (or a np.array)</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">feedforward</span>(<span style="color: #658b00">self</span>,a):
|
|
<span style="color: #228B22">#returns output w/ 'a' as an input</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> b, w <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, <span style="color: #658b00">self</span>.weights):
|
|
a=sigmoid(np.dot(w,b)+b)
|
|
<span style="color: #8B008B; font-weight: bold">return</span> a
|
|
|
|
<span style="color: #228B22">#Apply a Stochastic Gradient Descent (SGD) method:</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">SGD</span>(<span style="color: #658b00">self</span>, training_data, epochs, mini_batch_size, eta, test_data=<span style="color: #658b00">None</span>):
|
|
<span style="color: #CD5555">"""Trains network using batches incorporating SGD. The network will be evaluated against the</span>
|
|
<span style="color: #CD5555"> test data after each epoch, with partial progress being printed out (this is useful for tracking,</span>
|
|
<span style="color: #CD5555"> but slows the process.)"""</span>
|
|
<span style="color: #8B008B; font-weight: bold">if</span> test_data: n_test=<span style="color: #658b00">len</span>(test_data)
|
|
n=<span style="color: #658b00">len</span>(training_data)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(epochs):
|
|
random.shuffle(training_data)
|
|
mini_batches=[training_data[k:k+mini_batch_size] <span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(o,n,mini_batch_size)]
|
|
<span style="color: #8B008B; font-weight: bold">for</span> mini_batch <span style="color: #8B008B">in</span> mini_batches:
|
|
<span style="color: #658b00">self</span>.update_mini_batch(mini_batch, eta)
|
|
<span style="color: #8B008B; font-weight: bold">if</span> test_data:
|
|
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Epoch {0}: {1}/{2}"</span>.format(j, <span style="color: #658b00">self</span>.evaluate(test_data), n_test))
|
|
<span style="color: #8B008B; font-weight: bold">else</span>:
|
|
<span style="color: #8B008B; font-weight: bold">print</span> (<span style="color: #CD5555">"Epoch {0} complete"</span>.format(j))
|
|
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">update_mini_batch</span>(<span style="color: #658b00">self</span>, mini_batch, eta):
|
|
<span style="color: #228B22">#updates w and b using backpropagation to a single mini batch. eta is the learning rate."</span>
|
|
nabla_b=[np.zeros(b.shape) <span style="color: #8B008B; font-weight: bold">for</span> b <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.biases]
|
|
nabla_w=[np.zeros(w.shape) <span style="color: #8B008B; font-weight: bold">for</span> w <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.weights]
|
|
<span style="color: #8B008B; font-weight: bold">for</span> x,y <span style="color: #8B008B">in</span> mini_batch:
|
|
delta_nabla_b, delta_nabla_w=<span style="color: #658b00">self</span>.backprop(x,y)
|
|
nabla_b=[nb+dnb <span style="color: #8B008B; font-weight: bold">for</span> nb, dnb <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(nabla_b, delta_nabla_b)]
|
|
nabla_w=[nw+dnw <span style="color: #8B008B; font-weight: bold">for</span> nw, dnw <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(nabla_w, delta_nabla_w)]
|
|
<span style="color: #658b00">self</span>.weights=[w-(eta/<span style="color: #658b00">len</span>(mini_batch))*nw <span style="color: #8B008B; font-weight: bold">for</span> w, nw <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.weights, nabla_w)]
|
|
<span style="color: #658b00">self</span>.biases=[b-(eta/<span style="color: #658b00">len</span>(mini_batch))*nb <span style="color: #8B008B; font-weight: bold">for</span> b, nb <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, nabla_b)]
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">backprop</span>(<span style="color: #658b00">self</span>, x, y):
|
|
<span style="color: #CD5555">"""Return a tuple ``(nabla_b, nabla_w)`` representing the</span>
|
|
<span style="color: #CD5555"> gradient for the cost function C_x. ``nabla_b`` and</span>
|
|
<span style="color: #CD5555"> ``nabla_w`` are layer-by-layer lists of numpy arrays, similar</span>
|
|
<span style="color: #CD5555"> to ``self.biases`` and ``self.weights``."""</span>
|
|
nabla_b = [np.zeros(b.shape) <span style="color: #8B008B; font-weight: bold">for</span> b <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.biases]
|
|
nabla_w = [np.zeros(w.shape) <span style="color: #8B008B; font-weight: bold">for</span> w <span style="color: #8B008B">in</span> <span style="color: #658b00">self</span>.weights]
|
|
<span style="color: #228B22"># feedforward</span>
|
|
activation = x
|
|
activations = [x] <span style="color: #228B22"># list to store all the activations, layer by layer</span>
|
|
zs = [] <span style="color: #228B22"># list to store all the z vectors, layer by layer</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> b, w <span style="color: #8B008B">in</span> <span style="color: #658b00">zip</span>(<span style="color: #658b00">self</span>.biases, <span style="color: #658b00">self</span>.weights):
|
|
z = np.dot(w, activation)+b
|
|
zs.append(z)
|
|
activation = sigmoid(z)
|
|
activations.append(activation)
|
|
<span style="color: #228B22"># backward pass</span>
|
|
delta = <span style="color: #658b00">self</span>.cost_derivative(activations[-<span style="color: #B452CD">1</span>], y) * \
|
|
sigmoid_prime(zs[-<span style="color: #B452CD">1</span>])
|
|
nabla_b[-<span style="color: #B452CD">1</span>] = delta
|
|
nabla_w[-<span style="color: #B452CD">1</span>] = np.dot(delta, activations[-<span style="color: #B452CD">2</span>].transpose())
|
|
<span style="color: #228B22"># Note that the variable l in the loop below is used a little</span>
|
|
<span style="color: #228B22"># differently to the notation in Chapter 2 of the book. Here,</span>
|
|
<span style="color: #228B22"># l = 1 means the last layer of neurons, l = 2 is the</span>
|
|
<span style="color: #228B22"># second-last layer, and so on. It's a renumbering of the</span>
|
|
<span style="color: #228B22"># scheme in the book, used here to take advantage of the fact</span>
|
|
<span style="color: #228B22"># that Python can use negative indices in lists.</span>
|
|
<span style="color: #8B008B; font-weight: bold">for</span> l <span style="color: #8B008B">in</span> <span style="color: #658b00">xrange</span>(<span style="color: #B452CD">2</span>, <span style="color: #658b00">self</span>.num_layers):
|
|
z = zs[-l]
|
|
sp = sigmoid_prime(z)
|
|
delta = np.dot(<span style="color: #658b00">self</span>.weights[-l+<span style="color: #B452CD">1</span>].transpose(), delta) * sp
|
|
nabla_b[-l] = delta
|
|
nabla_w[-l] = np.dot(delta, activations[-l-<span style="color: #B452CD">1</span>].transpose())
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (nabla_b, nabla_w)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">evaluate</span>(<span style="color: #658b00">self</span>, test_data):
|
|
<span style="color: #CD5555">"""Return the number of test inputs for which the neural</span>
|
|
<span style="color: #CD5555"> network outputs the correct result. Note that the neural</span>
|
|
<span style="color: #CD5555"> network's output is assumed to be the index of whichever</span>
|
|
<span style="color: #CD5555"> neuron in the final layer has the highest activation."""</span>
|
|
test_results = [(np.argmax(<span style="color: #658b00">self</span>.feedforward(x)), y)
|
|
<span style="color: #8B008B; font-weight: bold">for</span> (x, y) <span style="color: #8B008B">in</span> test_data]
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #658b00">sum</span>(<span style="color: #658b00">int</span>(x == y) <span style="color: #8B008B; font-weight: bold">for</span> (x, y) <span style="color: #8B008B">in</span> test_results)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">cost_derivative</span>(<span style="color: #658b00">self</span>, output_activations, y):
|
|
<span style="color: #CD5555">"""Return the vector of partial derivatives \partial C_x /</span>
|
|
<span style="color: #CD5555"> \partial a for the output activations."""</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (output_activations-y)
|
|
|
|
|
|
|
|
<span style="color: #228B22">#Functions</span>
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(z):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1.0</span>/(<span style="color: #B452CD">1.0</span>+np.exp(-z))
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid_prime</span>(z):
|
|
<span style="color: #8B008B; font-weight: bold">return</span> sigmoid(z)*(<span style="color: #B452CD">1</span>-sigmoid(z))
|
|
|
|
network=Network()
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
|
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># %load neural-networks-and-deep-learning/src/mnist_loader.py</span>
|
|
<span style="color: #CD5555">"""</span>
|
|
<span style="color: #CD5555">mnist_loader</span>
|
|
<span style="color: #CD5555">~~~~~~~~~~~~</span>
|
|
|
|
<span style="color: #CD5555">A library to load the MNIST image data. For details of the data</span>
|
|
<span style="color: #CD5555">structures that are returned, see the doc strings for ``load_data``</span>
|
|
<span style="color: #CD5555">and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the</span>
|
|
<span style="color: #CD5555">function usually called by our neural network code.</span>
|
|
<span style="color: #CD5555">"""</span>
|
|
|
|
<span style="color: #228B22">#### Libraries</span>
|
|
<span style="color: #228B22"># Standard library</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">gzip</span>
|
|
|
|
<span style="color: #228B22"># Third-party libraries</span>
|
|
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_data</span>():
|
|
<span style="color: #CD5555">"""Return the MNIST data as a tuple containing the training data,</span>
|
|
<span style="color: #CD5555"> the validation data, and the test data.</span>
|
|
|
|
<span style="color: #CD5555"> The ``training_data`` is returned as a tuple with two entries.</span>
|
|
<span style="color: #CD5555"> The first entry contains the actual training images. This is a</span>
|
|
<span style="color: #CD5555"> numpy ndarray with 50,000 entries. Each entry is, in turn, a</span>
|
|
<span style="color: #CD5555"> numpy ndarray with 784 values, representing the 28 * 28 = 784</span>
|
|
<span style="color: #CD5555"> pixels in a single MNIST image.</span>
|
|
|
|
<span style="color: #CD5555"> The second entry in the ``training_data`` tuple is a numpy ndarray</span>
|
|
<span style="color: #CD5555"> containing 50,000 entries. Those entries are just the digit</span>
|
|
<span style="color: #CD5555"> values (0...9) for the corresponding images contained in the first</span>
|
|
<span style="color: #CD5555"> entry of the tuple.</span>
|
|
|
|
<span style="color: #CD5555"> The ``validation_data`` and ``test_data`` are similar, except</span>
|
|
<span style="color: #CD5555"> each contains only 10,000 images.</span>
|
|
|
|
<span style="color: #CD5555"> This is a nice data format, but for use in neural networks it's</span>
|
|
<span style="color: #CD5555"> helpful to modify the format of the ``training_data`` a little.</span>
|
|
<span style="color: #CD5555"> That's done in the wrapper function ``load_data_wrapper()``, see</span>
|
|
<span style="color: #CD5555"> below.</span>
|
|
<span style="color: #CD5555"> """</span>
|
|
f = gzip.open(<span style="color: #CD5555">'../data/mnist.pkl.gz'</span>, <span style="color: #CD5555">'rb'</span>)
|
|
training_data, validation_data, test_data = cPickle.load(f)
|
|
f.close()
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (training_data, validation_data, test_data)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">load_data_wrapper</span>():
|
|
<span style="color: #CD5555">"""Return a tuple containing ``(training_data, validation_data,</span>
|
|
<span style="color: #CD5555"> test_data)``. Based on ``load_data``, but the format is more</span>
|
|
<span style="color: #CD5555"> convenient for use in our implementation of neural networks.</span>
|
|
|
|
<span style="color: #CD5555"> In particular, ``training_data`` is a list containing 50,000</span>
|
|
<span style="color: #CD5555"> 2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray</span>
|
|
<span style="color: #CD5555"> containing the input image. ``y`` is a 10-dimensional</span>
|
|
<span style="color: #CD5555"> numpy.ndarray representing the unit vector corresponding to the</span>
|
|
<span style="color: #CD5555"> correct digit for ``x``.</span>
|
|
|
|
<span style="color: #CD5555"> ``validation_data`` and ``test_data`` are lists containing 10,000</span>
|
|
<span style="color: #CD5555"> 2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional</span>
|
|
<span style="color: #CD5555"> numpy.ndarry containing the input image, and ``y`` is the</span>
|
|
<span style="color: #CD5555"> corresponding classification, i.e., the digit values (integers)</span>
|
|
<span style="color: #CD5555"> corresponding to ``x``.</span>
|
|
|
|
<span style="color: #CD5555"> Obviously, this means we're using slightly different formats for</span>
|
|
<span style="color: #CD5555"> the training data and the validation / test data. These formats</span>
|
|
<span style="color: #CD5555"> turn out to be the most convenient for use in our neural network</span>
|
|
<span style="color: #CD5555"> code."""</span>
|
|
tr_d, va_d, te_d = load_data()
|
|
training_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> tr_d[<span style="color: #B452CD">0</span>]]
|
|
training_results = [vectorized_result(y) <span style="color: #8B008B; font-weight: bold">for</span> y <span style="color: #8B008B">in</span> tr_d[<span style="color: #B452CD">1</span>]]
|
|
training_data = <span style="color: #658b00">zip</span>(training_inputs, training_results)
|
|
validation_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> va_d[<span style="color: #B452CD">0</span>]]
|
|
validation_data = <span style="color: #658b00">zip</span>(validation_inputs, va_d[<span style="color: #B452CD">1</span>])
|
|
test_inputs = [np.reshape(x, (<span style="color: #B452CD">784</span>, <span style="color: #B452CD">1</span>)) <span style="color: #8B008B; font-weight: bold">for</span> x <span style="color: #8B008B">in</span> te_d[<span style="color: #B452CD">0</span>]]
|
|
test_data = <span style="color: #658b00">zip</span>(test_inputs, te_d[<span style="color: #B452CD">1</span>])
|
|
<span style="color: #8B008B; font-weight: bold">return</span> (training_data, validation_data, test_data)
|
|
|
|
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">vectorized_result</span>(j):
|
|
<span style="color: #CD5555">"""Return a 10-dimensional unit vector with a 1.0 in the jth</span>
|
|
<span style="color: #CD5555"> position and zeroes elsewhere. This is used to convert a digit</span>
|
|
<span style="color: #CD5555"> (0...9) into a corresponding desired output from the neural</span>
|
|
<span style="color: #CD5555"> network."""</span>
|
|
e = np.zeros((<span style="color: #B452CD">10</span>, <span style="color: #B452CD">1</span>))
|
|
e[j] = <span style="color: #B452CD">1.0</span>
|
|
<span style="color: #8B008B; font-weight: bold">return</span> e
|
|
|
|
net=network.Network([<span style="color: #B452CD">784</span>,<span style="color: #B452CD">30</span>,<span style="color: #B452CD">30</span>])
|
|
net.SGD(training_data,<span style="color: #B452CD">30</span>,<span style="color: #B452CD">10</span>,<span style="color: #B452CD">3</span>,test_data=test_data)
|
|
</pre></div>
|
|
</section>
|
|
|
|
|
|
|
|
</div> <!-- class="slides" -->
|
|
</div> <!-- class="reveal" -->
|
|
|
|
<script src="reveal.js/lib/js/head.min.js"></script>
|
|
<script src="reveal.js/js/reveal.js"></script>
|
|
|
|
<script>
|
|
// Full list of configuration options available here:
|
|
// https://github.com/hakimel/reveal.js#configuration
|
|
Reveal.initialize({
|
|
|
|
// Display navigation controls in the bottom right corner
|
|
controls: true,
|
|
|
|
// Display progress bar (below the horiz. slider)
|
|
progress: true,
|
|
|
|
// Display the page number of the current slide
|
|
slideNumber: true,
|
|
|
|
// Push each slide change to the browser history
|
|
history: false,
|
|
|
|
// Enable keyboard shortcuts for navigation
|
|
keyboard: true,
|
|
|
|
// Enable the slide overview mode
|
|
overview: true,
|
|
|
|
// Vertical centering of slides
|
|
//center: true,
|
|
center: false,
|
|
|
|
// Enables touch navigation on devices with touch input
|
|
touch: true,
|
|
|
|
// Loop the presentation
|
|
loop: false,
|
|
|
|
// Change the presentation direction to be RTL
|
|
rtl: false,
|
|
|
|
// Turns fragments on and off globally
|
|
fragments: true,
|
|
|
|
// Flags if the presentation is running in an embedded mode,
|
|
// i.e. contained within a limited portion of the screen
|
|
embedded: false,
|
|
|
|
// Number of milliseconds between automatically proceeding to the
|
|
// next slide, disabled when set to 0, this value can be overwritten
|
|
// by using a data-autoslide attribute on your slides
|
|
autoSlide: 0,
|
|
|
|
// Stop auto-sliding after user input
|
|
autoSlideStoppable: true,
|
|
|
|
// Enable slide navigation via mouse wheel
|
|
mouseWheel: false,
|
|
|
|
// Hides the address bar on mobile devices
|
|
hideAddressBar: true,
|
|
|
|
// Opens links in an iframe preview overlay
|
|
previewLinks: false,
|
|
|
|
// Transition style
|
|
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
|
|
|
|
// Transition speed
|
|
transitionSpeed: 'default', // default/fast/slow
|
|
|
|
// Transition style for full page slide backgrounds
|
|
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
|
|
|
|
// Number of slides away from the current that are visible
|
|
viewDistance: 3,
|
|
|
|
// Parallax background image
|
|
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
|
|
|
|
// Parallax background size
|
|
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
|
|
|
|
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
|
|
transition: Reveal.getQueryHash().transition || 'default', // default/cube/page/concave/zoom/linear/none
|
|
|
|
});
|
|
|
|
Reveal.initialize({
|
|
dependencies: [
|
|
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
|
|
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
|
|
|
|
// Interpret Markdown in <section> elements
|
|
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
|
|
|
|
// Syntax highlight for <code> elements
|
|
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
|
|
|
|
// Zoom in and out with Alt+click
|
|
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Speaker notes
|
|
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// Remote control your reveal.js presentation using a touch device
|
|
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
|
|
|
|
// MathJax
|
|
//{ src: 'reveal.js/plugin/math/math.js', async: true }
|
|
]
|
|
});
|
|
|
|
Reveal.initialize({
|
|
|
|
// The "normal" size of the presentation, aspect ratio will be preserved
|
|
// when the presentation is scaled to fit different resolutions. Can be
|
|
// specified using percentage units.
|
|
width: 1170, // original: 960,
|
|
height: 700,
|
|
|
|
// Factor of the display size that should remain empty around the content
|
|
margin: 0.1,
|
|
|
|
// Bounds for smallest/largest possible scale to apply to content
|
|
minScale: 0.2,
|
|
maxScale: 1.0
|
|
|
|
});
|
|
</script>
|
|
|
|
<!-- begin footer logo
|
|
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
|
|
<img src="somelogo.png">
|
|
</div>
|
|
end footer logo -->
|
|
|
|
|
|
|
|
</body>
|
|
</html>
|