lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1b2e055ffa91e81d94d08f9b2265527ea3c4f80d
FYS-STK4155/doc/pub/NeuralNet/html/._NeuralNet-bs020.html
T
mhjensen c56b1b1611 chasing slides type
2018-09-28 12:11:44 +02:00

425 lines
38 KiB
HTML
Raw Blame History

<!--
Automatically generated HTML file from DocOnce source
(https://github.com/hplgit/doconce/)
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
<meta name="description" content="Data Analysis and Machine Learning: Elements of machine learning">
<title>Data Analysis and Machine Learning: Elements of machine learning</title>
<!-- Bootstrap style: bootstrap -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Neural networks', 2, None, '___sec0'),
('Artificial neurons', 2, None, '___sec1'),
('Neural network types', 2, None, '___sec2'),
('Feed-forward neural networks', 2, None, '___sec3'),
('Recurrent neural networks', 2, None, '___sec4'),
('Other types of networks', 2, None, '___sec5'),
('Multilayer perceptrons', 2, None, '___sec6'),
('Why multilayer perceptrons?', 2, None, '___sec7'),
('Mathematical model', 2, None, '___sec8'),
('Mathematical model', 2, None, '___sec9'),
('Mathematical model', 2, None, '___sec10'),
('Mathematical model', 2, None, '___sec11'),
('Mathematical model', 2, None, '___sec12'),
('Matrix-vector notation', 3, None, '___sec13'),
('Matrix-vector notation and activation', 3, None, '___sec14'),
('Activation functions', 3, None, '___sec15'),
('Activation functions, Logistic and Hyperbolic ones',
3,
None,
'___sec16'),
('Relevance', 3, None, '___sec17'),
('Setting up a Multi-layer perceptron model',
2,
None,
'___sec18'),
('Two-layer Neural Network', 2, None, '___sec19')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Elements of machine learning</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Other types of networks</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Mathematical model</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation and activation</a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model</b></a></li>
<!-- navigation toc: --> <li><a href="#___sec19" style="font-size: 80%;"><b>Two-layer Neural Network</b></a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0020"></a>
<!-- !split -->
<h2 id="___sec19" class="anchor">Two-layer Neural Network </h2>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #408080; font-style: italic">#sigmoid</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">nonlin</span>(x, deriv<span style="color: #666666">=</span><span style="color: #008000">False</span>):
<span style="color: #008000; font-weight: bold">if</span> (deriv<span style="color: #666666">==</span><span style="color: #008000">True</span>):
<span style="color: #008000; font-weight: bold">return</span> x<span style="color: #666666">*</span>(<span style="color: #666666">1-</span>x)
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1+</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
<span style="color: #408080; font-style: italic">#input data</span>
x<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>],[<span style="color: #666666">1</span>,<span style="color: #666666">0</span>,<span style="color: #666666">1</span>],[<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>]])
<span style="color: #408080; font-style: italic">#output data</span>
y<span style="color: #666666">=</span>np<span style="color: #666666">.</span>array([<span style="color: #666666">0</span>,<span style="color: #666666">1</span>,<span style="color: #666666">1</span>,<span style="color: #666666">0</span>])<span style="color: #666666">.</span>T
<span style="color: #408080; font-style: italic">#seed random numbers to make calculation</span>
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic">#initialize weights with mean=0</span>
syn0<span style="color: #666666">=2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>random((<span style="color: #666666">3</span>,<span style="color: #666666">4</span>))<span style="color: #666666">-1</span>
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">10000</span>):
<span style="color: #408080; font-style: italic">#forward propogation</span>
l0<span style="color: #666666">=</span>x
l1<span style="color: #666666">=</span>nonlin(np<span style="color: #666666">.</span>dot(l0,syn0))
l1_error<span style="color: #666666">=</span>y<span style="color: #666666">-</span>l1
<span style="color: #408080; font-style: italic">#multiply error by slope of sigmoid at values of l1</span>
l1_delta<span style="color: #666666">=</span>l1_error<span style="color: #666666">*</span>nonlin(l1,<span style="color: #008000">True</span>)
<span style="color: #408080; font-style: italic">#update weights</span>
syn0<span style="color: #666666">+=</span>np<span style="color: #666666">.</span>dot(l0<span style="color: #666666">.</span>T, l1_delta)
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&quot;Output after training: &quot;</span>,l1 )
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">random</span>
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Network</span>(<span style="color: #008000">object</span>):
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_init_</span>(<span style="color: #008000">self</span>, sizes):
<span style="color: #008000">self</span><span style="color: #666666">.</span>num_layers<span style="color: #666666">=</span><span style="color: #008000">len</span>(sizes)
<span style="color: #008000">self</span><span style="color: #666666">.</span>sizes<span style="color: #666666">=</span>sizes
<span style="color: #008000">self</span><span style="color: #666666">.</span>biases<span style="color: #666666">=</span>[np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(y,<span style="color: #666666">1</span>) <span style="color: #008000; font-weight: bold">for</span> y <span style="color: #AA22FF; font-weight: bold">in</span> sizes[<span style="color: #666666">1</span>:]]
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights<span style="color: #666666">=</span>[np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(y,x) <span style="color: #008000; font-weight: bold">for</span> x,y <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(sizes[:<span style="color: #666666">-1</span>], sizes[<span style="color: #666666">1</span>:])]
<span style="color: #408080; font-style: italic">#sizes is the number of neurons in each layer</span>
<span style="color: #408080; font-style: italic">#for example, say n_1st_layer=3, n_2nd_layer=3, n_3rd_layer=1, then net=Network([3,3,1])</span>
<span style="color: #408080; font-style: italic">#The biases and weights are initialized randomly, using Gaussian distributions of mean=0, stdev=1</span>
<span style="color: #408080; font-style: italic">#z is a vector (or a np.array)</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feedforward</span>(<span style="color: #008000">self</span>,a):
<span style="color: #408080; font-style: italic">#returns output w/ &#39;a&#39; as an input</span>
<span style="color: #008000; font-weight: bold">for</span> b, w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>biases, <span style="color: #008000">self</span><span style="color: #666666">.</span>weights):
a<span style="color: #666666">=</span>sigmoid(np<span style="color: #666666">.</span>dot(w,b)<span style="color: #666666">+</span>b)
<span style="color: #008000; font-weight: bold">return</span> a
<span style="color: #408080; font-style: italic">#Apply a Stochastic Gradient Descent (SGD) method:</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">SGD</span>(<span style="color: #008000">self</span>, training_data, epochs, mini_batch_size, eta, test_data<span style="color: #666666">=</span><span style="color: #008000">None</span>):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Trains network using batches incorporating SGD. The network will be evaluated against the</span>
<span style="color: #BA2121; font-style: italic"> test data after each epoch, with partial progress being printed out (this is useful for tracking,</span>
<span style="color: #BA2121; font-style: italic"> but slows the process.)&quot;&quot;&quot;</span>
<span style="color: #008000; font-weight: bold">if</span> test_data: n_test<span style="color: #666666">=</span><span style="color: #008000">len</span>(test_data)
n<span style="color: #666666">=</span><span style="color: #008000">len</span>(training_data)
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">xrange</span>(epochs):
random<span style="color: #666666">.</span>shuffle(training_data)
mini_batches<span style="color: #666666">=</span>[training_data[k:k<span style="color: #666666">+</span>mini_batch_size] <span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">xrange</span>(o,n,mini_batch_size)]
<span style="color: #008000; font-weight: bold">for</span> mini_batch <span style="color: #AA22FF; font-weight: bold">in</span> mini_batches:
<span style="color: #008000">self</span><span style="color: #666666">.</span>update_mini_batch(mini_batch, eta)
<span style="color: #008000; font-weight: bold">if</span> test_data:
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">&quot;Epoch {0}: {1}/{2}&quot;</span><span style="color: #666666">.</span>format(j, <span style="color: #008000">self</span><span style="color: #666666">.</span>evaluate(test_data), n_test))
<span style="color: #008000; font-weight: bold">else</span>:
<span style="color: #008000; font-weight: bold">print</span> (<span style="color: #BA2121">&quot;Epoch {0} complete&quot;</span><span style="color: #666666">.</span>format(j))
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_mini_batch</span>(<span style="color: #008000">self</span>, mini_batch, eta):
<span style="color: #408080; font-style: italic">#updates w and b using backpropagation to a single mini batch. eta is the learning rate.&quot;</span>
nabla_b<span style="color: #666666">=</span>[np<span style="color: #666666">.</span>zeros(b<span style="color: #666666">.</span>shape) <span style="color: #008000; font-weight: bold">for</span> b <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>biases]
nabla_w<span style="color: #666666">=</span>[np<span style="color: #666666">.</span>zeros(w<span style="color: #666666">.</span>shape) <span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights]
<span style="color: #008000; font-weight: bold">for</span> x,y <span style="color: #AA22FF; font-weight: bold">in</span> mini_batch:
delta_nabla_b, delta_nabla_w<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>backprop(x,y)
nabla_b<span style="color: #666666">=</span>[nb<span style="color: #666666">+</span>dnb <span style="color: #008000; font-weight: bold">for</span> nb, dnb <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(nabla_b, delta_nabla_b)]
nabla_w<span style="color: #666666">=</span>[nw<span style="color: #666666">+</span>dnw <span style="color: #008000; font-weight: bold">for</span> nw, dnw <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(nabla_w, delta_nabla_w)]
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights<span style="color: #666666">=</span>[w<span style="color: #666666">-</span>(eta<span style="color: #666666">/</span><span style="color: #008000">len</span>(mini_batch))<span style="color: #666666">*</span>nw <span style="color: #008000; font-weight: bold">for</span> w, nw <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>weights, nabla_w)]
<span style="color: #008000">self</span><span style="color: #666666">.</span>biases<span style="color: #666666">=</span>[b<span style="color: #666666">-</span>(eta<span style="color: #666666">/</span><span style="color: #008000">len</span>(mini_batch))<span style="color: #666666">*</span>nb <span style="color: #008000; font-weight: bold">for</span> b, nb <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>biases, nabla_b)]
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backprop</span>(<span style="color: #008000">self</span>, x, y):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return a tuple ``(nabla_b, nabla_w)`` representing the</span>
<span style="color: #BA2121; font-style: italic"> gradient for the cost function C_x. ``nabla_b`` and</span>
<span style="color: #BA2121; font-style: italic"> ``nabla_w`` are layer-by-layer lists of numpy arrays, similar</span>
<span style="color: #BA2121; font-style: italic"> to ``self.biases`` and ``self.weights``.&quot;&quot;&quot;</span>
nabla_b <span style="color: #666666">=</span> [np<span style="color: #666666">.</span>zeros(b<span style="color: #666666">.</span>shape) <span style="color: #008000; font-weight: bold">for</span> b <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>biases]
nabla_w <span style="color: #666666">=</span> [np<span style="color: #666666">.</span>zeros(w<span style="color: #666666">.</span>shape) <span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights]
<span style="color: #408080; font-style: italic"># feedforward</span>
activation <span style="color: #666666">=</span> x
activations <span style="color: #666666">=</span> [x] <span style="color: #408080; font-style: italic"># list to store all the activations, layer by layer</span>
zs <span style="color: #666666">=</span> [] <span style="color: #408080; font-style: italic"># list to store all the z vectors, layer by layer</span>
<span style="color: #008000; font-weight: bold">for</span> b, w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">zip</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>biases, <span style="color: #008000">self</span><span style="color: #666666">.</span>weights):
z <span style="color: #666666">=</span> np<span style="color: #666666">.</span>dot(w, activation)<span style="color: #666666">+</span>b
zs<span style="color: #666666">.</span>append(z)
activation <span style="color: #666666">=</span> sigmoid(z)
activations<span style="color: #666666">.</span>append(activation)
<span style="color: #408080; font-style: italic"># backward pass</span>
delta <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>cost_derivative(activations[<span style="color: #666666">-1</span>], y) <span style="color: #666666">*</span> \
sigmoid_prime(zs[<span style="color: #666666">-1</span>])
nabla_b[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> delta
nabla_w[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>dot(delta, activations[<span style="color: #666666">-2</span>]<span style="color: #666666">.</span>transpose())
<span style="color: #408080; font-style: italic"># Note that the variable l in the loop below is used a little</span>
<span style="color: #408080; font-style: italic"># differently to the notation in Chapter 2 of the book. Here,</span>
<span style="color: #408080; font-style: italic"># l = 1 means the last layer of neurons, l = 2 is the</span>
<span style="color: #408080; font-style: italic"># second-last layer, and so on. It&#39;s a renumbering of the</span>
<span style="color: #408080; font-style: italic"># scheme in the book, used here to take advantage of the fact</span>
<span style="color: #408080; font-style: italic"># that Python can use negative indices in lists.</span>
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">xrange</span>(<span style="color: #666666">2</span>, <span style="color: #008000">self</span><span style="color: #666666">.</span>num_layers):
z <span style="color: #666666">=</span> zs[<span style="color: #666666">-</span>l]
sp <span style="color: #666666">=</span> sigmoid_prime(z)
delta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>dot(<span style="color: #008000">self</span><span style="color: #666666">.</span>weights[<span style="color: #666666">-</span>l<span style="color: #666666">+1</span>]<span style="color: #666666">.</span>transpose(), delta) <span style="color: #666666">*</span> sp
nabla_b[<span style="color: #666666">-</span>l] <span style="color: #666666">=</span> delta
nabla_w[<span style="color: #666666">-</span>l] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>dot(delta, activations[<span style="color: #666666">-</span>l<span style="color: #666666">-1</span>]<span style="color: #666666">.</span>transpose())
<span style="color: #008000; font-weight: bold">return</span> (nabla_b, nabla_w)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">evaluate</span>(<span style="color: #008000">self</span>, test_data):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return the number of test inputs for which the neural</span>
<span style="color: #BA2121; font-style: italic"> network outputs the correct result. Note that the neural</span>
<span style="color: #BA2121; font-style: italic"> network&#39;s output is assumed to be the index of whichever</span>
<span style="color: #BA2121; font-style: italic"> neuron in the final layer has the highest activation.&quot;&quot;&quot;</span>
test_results <span style="color: #666666">=</span> [(np<span style="color: #666666">.</span>argmax(<span style="color: #008000">self</span><span style="color: #666666">.</span>feedforward(x)), y)
<span style="color: #008000; font-weight: bold">for</span> (x, y) <span style="color: #AA22FF; font-weight: bold">in</span> test_data]
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">sum</span>(<span style="color: #008000">int</span>(x <span style="color: #666666">==</span> y) <span style="color: #008000; font-weight: bold">for</span> (x, y) <span style="color: #AA22FF; font-weight: bold">in</span> test_results)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cost_derivative</span>(<span style="color: #008000">self</span>, output_activations, y):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return the vector of partial derivatives \partial C_x /</span>
<span style="color: #BA2121; font-style: italic"> \partial a for the output activations.&quot;&quot;&quot;</span>
<span style="color: #008000; font-weight: bold">return</span> (output_activations<span style="color: #666666">-</span>y)
<span style="color: #408080; font-style: italic">#Functions</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(z):
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1.0/</span>(<span style="color: #666666">1.0+</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z))
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid_prime</span>(z):
<span style="color: #008000; font-weight: bold">return</span> sigmoid(z)<span style="color: #666666">*</span>(<span style="color: #666666">1-</span>sigmoid(z))
network<span style="color: #666666">=</span>Network()
</pre></div>
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># %load neural-networks-and-deep-learning/src/mnist_loader.py</span>
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #BA2121; font-style: italic">mnist_loader</span>
<span style="color: #BA2121; font-style: italic">~~~~~~~~~~~~</span>
<span style="color: #BA2121; font-style: italic">A library to load the MNIST image data. For details of the data</span>
<span style="color: #BA2121; font-style: italic">structures that are returned, see the doc strings for ``load_data``</span>
<span style="color: #BA2121; font-style: italic">and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the</span>
<span style="color: #BA2121; font-style: italic">function usually called by our neural network code.</span>
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
<span style="color: #408080; font-style: italic">#### Libraries</span>
<span style="color: #408080; font-style: italic"># Standard library</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pickle</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">gzip</span>
<span style="color: #408080; font-style: italic"># Third-party libraries</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">load_data</span>():
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return the MNIST data as a tuple containing the training data,</span>
<span style="color: #BA2121; font-style: italic"> the validation data, and the test data.</span>
<span style="color: #BA2121; font-style: italic"> The ``training_data`` is returned as a tuple with two entries.</span>
<span style="color: #BA2121; font-style: italic"> The first entry contains the actual training images. This is a</span>
<span style="color: #BA2121; font-style: italic"> numpy ndarray with 50,000 entries. Each entry is, in turn, a</span>
<span style="color: #BA2121; font-style: italic"> numpy ndarray with 784 values, representing the 28 * 28 = 784</span>
<span style="color: #BA2121; font-style: italic"> pixels in a single MNIST image.</span>
<span style="color: #BA2121; font-style: italic"> The second entry in the ``training_data`` tuple is a numpy ndarray</span>
<span style="color: #BA2121; font-style: italic"> containing 50,000 entries. Those entries are just the digit</span>
<span style="color: #BA2121; font-style: italic"> values (0...9) for the corresponding images contained in the first</span>
<span style="color: #BA2121; font-style: italic"> entry of the tuple.</span>
<span style="color: #BA2121; font-style: italic"> The ``validation_data`` and ``test_data`` are similar, except</span>
<span style="color: #BA2121; font-style: italic"> each contains only 10,000 images.</span>
<span style="color: #BA2121; font-style: italic"> This is a nice data format, but for use in neural networks it&#39;s</span>
<span style="color: #BA2121; font-style: italic"> helpful to modify the format of the ``training_data`` a little.</span>
<span style="color: #BA2121; font-style: italic"> That&#39;s done in the wrapper function ``load_data_wrapper()``, see</span>
<span style="color: #BA2121; font-style: italic"> below.</span>
<span style="color: #BA2121; font-style: italic"> &quot;&quot;&quot;</span>
f <span style="color: #666666">=</span> gzip<span style="color: #666666">.</span>open(<span style="color: #BA2121">&#39;../data/mnist.pkl.gz&#39;</span>, <span style="color: #BA2121">&#39;rb&#39;</span>)
training_data, validation_data, test_data <span style="color: #666666">=</span> cPickle<span style="color: #666666">.</span>load(f)
f<span style="color: #666666">.</span>close()
<span style="color: #008000; font-weight: bold">return</span> (training_data, validation_data, test_data)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">load_data_wrapper</span>():
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return a tuple containing ``(training_data, validation_data,</span>
<span style="color: #BA2121; font-style: italic"> test_data)``. Based on ``load_data``, but the format is more</span>
<span style="color: #BA2121; font-style: italic"> convenient for use in our implementation of neural networks.</span>
<span style="color: #BA2121; font-style: italic"> In particular, ``training_data`` is a list containing 50,000</span>
<span style="color: #BA2121; font-style: italic"> 2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray</span>
<span style="color: #BA2121; font-style: italic"> containing the input image. ``y`` is a 10-dimensional</span>
<span style="color: #BA2121; font-style: italic"> numpy.ndarray representing the unit vector corresponding to the</span>
<span style="color: #BA2121; font-style: italic"> correct digit for ``x``.</span>
<span style="color: #BA2121; font-style: italic"> ``validation_data`` and ``test_data`` are lists containing 10,000</span>
<span style="color: #BA2121; font-style: italic"> 2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional</span>
<span style="color: #BA2121; font-style: italic"> numpy.ndarry containing the input image, and ``y`` is the</span>
<span style="color: #BA2121; font-style: italic"> corresponding classification, i.e., the digit values (integers)</span>
<span style="color: #BA2121; font-style: italic"> corresponding to ``x``.</span>
<span style="color: #BA2121; font-style: italic"> Obviously, this means we&#39;re using slightly different formats for</span>
<span style="color: #BA2121; font-style: italic"> the training data and the validation / test data. These formats</span>
<span style="color: #BA2121; font-style: italic"> turn out to be the most convenient for use in our neural network</span>
<span style="color: #BA2121; font-style: italic"> code.&quot;&quot;&quot;</span>
tr_d, va_d, te_d <span style="color: #666666">=</span> load_data()
training_inputs <span style="color: #666666">=</span> [np<span style="color: #666666">.</span>reshape(x, (<span style="color: #666666">784</span>, <span style="color: #666666">1</span>)) <span style="color: #008000; font-weight: bold">for</span> x <span style="color: #AA22FF; font-weight: bold">in</span> tr_d[<span style="color: #666666">0</span>]]
training_results <span style="color: #666666">=</span> [vectorized_result(y) <span style="color: #008000; font-weight: bold">for</span> y <span style="color: #AA22FF; font-weight: bold">in</span> tr_d[<span style="color: #666666">1</span>]]
training_data <span style="color: #666666">=</span> <span style="color: #008000">zip</span>(training_inputs, training_results)
validation_inputs <span style="color: #666666">=</span> [np<span style="color: #666666">.</span>reshape(x, (<span style="color: #666666">784</span>, <span style="color: #666666">1</span>)) <span style="color: #008000; font-weight: bold">for</span> x <span style="color: #AA22FF; font-weight: bold">in</span> va_d[<span style="color: #666666">0</span>]]
validation_data <span style="color: #666666">=</span> <span style="color: #008000">zip</span>(validation_inputs, va_d[<span style="color: #666666">1</span>])
test_inputs <span style="color: #666666">=</span> [np<span style="color: #666666">.</span>reshape(x, (<span style="color: #666666">784</span>, <span style="color: #666666">1</span>)) <span style="color: #008000; font-weight: bold">for</span> x <span style="color: #AA22FF; font-weight: bold">in</span> te_d[<span style="color: #666666">0</span>]]
test_data <span style="color: #666666">=</span> <span style="color: #008000">zip</span>(test_inputs, te_d[<span style="color: #666666">1</span>])
<span style="color: #008000; font-weight: bold">return</span> (training_data, validation_data, test_data)
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">vectorized_result</span>(j):
<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Return a 10-dimensional unit vector with a 1.0 in the jth</span>
<span style="color: #BA2121; font-style: italic"> position and zeroes elsewhere. This is used to convert a digit</span>
<span style="color: #BA2121; font-style: italic"> (0...9) into a corresponding desired output from the neural</span>
<span style="color: #BA2121; font-style: italic"> network.&quot;&quot;&quot;</span>
e <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #666666">10</span>, <span style="color: #666666">1</span>))
e[j] <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
<span style="color: #008000; font-weight: bold">return</span> e
net<span style="color: #666666">=</span>network<span style="color: #666666">.</span>Network([<span style="color: #666666">784</span>,<span style="color: #666666">30</span>,<span style="color: #666666">30</span>])
net<span style="color: #666666">.</span>SGD(training_data,<span style="color: #666666">30</span>,<span style="color: #666666">10</span>,<span style="color: #666666">3</span>,test_data<span style="color: #666666">=</span>test_data)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._NeuralNet-bs019.html">&laquo;</a></li>
<li><a href="._NeuralNet-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._NeuralNet-bs012.html">13</a></li>
<li><a href="._NeuralNet-bs013.html">14</a></li>
<li><a href="._NeuralNet-bs014.html">15</a></li>
<li><a href="._NeuralNet-bs015.html">16</a></li>
<li><a href="._NeuralNet-bs016.html">17</a></li>
<li><a href="._NeuralNet-bs017.html">18</a></li>
<li><a href="._NeuralNet-bs018.html">19</a></li>
<li><a href="._NeuralNet-bs019.html">20</a></li>
<li class="active"><a href="._NeuralNet-bs020.html">21</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="http://..."><img width="250" align=right src="http://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink