added files
This commit is contained in:
Morten Hjorth-Jensen
@@ -0,0 +1,413 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 40: From Stochastic Gradient Descent to Neural networks">
|
||||
|
||||
<title>Week 40: From Stochastic Gradient Descent to Neural networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 40', 2, None, 'plan-for-week-40'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('ADAM optimizer', 2, None, 'adam-optimizer'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Recommended to avoid', 2, None, 'recommended-to-avoid'),
|
||||
('Videos on Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'videos-on-neural-networks'),
|
||||
('Neural networks', 2, None, 'neural-networks'),
|
||||
('Artificial neurons', 2, None, 'artificial-neurons'),
|
||||
('Neural network types', 2, None, 'neural-network-types'),
|
||||
('Feed-forward neural networks',
|
||||
2,
|
||||
None,
|
||||
'feed-forward-neural-networks'),
|
||||
('Convolutional Neural Network',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-network'),
|
||||
('Recurrent neural networks',
|
||||
2,
|
||||
None,
|
||||
'recurrent-neural-networks'),
|
||||
('Other types of networks', 2, None, 'other-types-of-networks'),
|
||||
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
||||
('Why multilayer perceptrons?',
|
||||
2,
|
||||
None,
|
||||
'why-multilayer-perceptrons'),
|
||||
('Illustration of a single perceptropn model and a '
|
||||
'multi-perceptron model',
|
||||
2,
|
||||
None,
|
||||
'illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model'),
|
||||
('Examples of XOR, OR and AND gates',
|
||||
2,
|
||||
None,
|
||||
'examples-of-xor-or-and-and-gates'),
|
||||
('Does Logistic Regression do a better Job?',
|
||||
2,
|
||||
None,
|
||||
'does-logistic-regression-do-a-better-job'),
|
||||
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
||||
('Matrix-vector notation and activation',
|
||||
3,
|
||||
None,
|
||||
'matrix-vector-notation-and-activation'),
|
||||
('Activation functions', 3, None, 'activation-functions'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'activation-functions-logistic-and-hyperbolic-ones'),
|
||||
('Relevance', 3, None, 'relevance'),
|
||||
('The multilayer perceptron (MLP)',
|
||||
2,
|
||||
None,
|
||||
'the-multilayer-perceptron-mlp'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
||||
('Definitions', 2, None, 'definitions'),
|
||||
('Derivatives and the chain rule',
|
||||
2,
|
||||
None,
|
||||
'derivatives-and-the-chain-rule'),
|
||||
('Derivative of the cost function',
|
||||
2,
|
||||
None,
|
||||
'derivative-of-the-cost-function'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'bringing-it-together-first-back-propagation-equation'),
|
||||
('Derivatives in terms of $z_j^L$',
|
||||
2,
|
||||
None,
|
||||
'derivatives-in-terms-of-z_j-l'),
|
||||
('Bringing it together', 2, None, 'bringing-it-together'),
|
||||
('Final back propagating equation',
|
||||
2,
|
||||
None,
|
||||
'final-back-propagating-equation'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'setting-up-the-back-propagation-algorithm')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week40-bs.html">Week 40: From Stochastic Gradient Descent to Neural networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plan-for-week-40" style="font-size: 80%;"><b>Plan for week 40</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs002.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs003.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs004.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs005.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs006.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs007.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs008.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs009.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs010.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs011.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs012.html#program-for-stochastic-gradient" style="font-size: 80%;"><b>Program for stochastic gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs013.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs014.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs015.html#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs016.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs017.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs018.html#adam-optimizer" style="font-size: 80%;"><b>ADAM optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs019.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs020.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs021.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs022.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs024.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs025.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs026.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs027.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs028.html#unsupported-functions" style="font-size: 80%;"><b>Unsupported functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs029.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;"><b>The syntax a.dot(b) when finding the dot product</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs030.html#recommended-to-avoid" style="font-size: 80%;"><b>Recommended to avoid</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs031.html#videos-on-neural-networks" style="font-size: 80%;"><b>Videos on Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs032.html#neural-networks" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs033.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs034.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs035.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs036.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs037.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs038.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs039.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs040.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs041.html#illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptropn model and a multi-perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs042.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs043.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs044.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs050.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs051.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs052.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs053.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs054.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs055.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs056.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs057.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs058.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs059.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs060.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs062.html#derivatives-in-terms-of-z_j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs063.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs064.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs065.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0061"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="bringing-it-together-first-back-propagation-equation" class="anchor">Bringing it together, first back propagation equation </h2>
|
||||
|
||||
<p>
|
||||
We have thus
|
||||
$$
|
||||
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1},
|
||||
$$
|
||||
|
||||
<p>
|
||||
Defining
|
||||
$$
|
||||
\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
|
||||
$$
|
||||
|
||||
and using the Hadamard product of two vectors we can write this as
|
||||
$$
|
||||
\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
This is an important expression. The second term on the right handside
|
||||
measures how fast the cost function is changing as a function of the $j$th
|
||||
output activation. If, for example, the cost function doesn't depend
|
||||
much on a particular output node \( j \), then \( \delta_j^L \) will be small,
|
||||
which is what we would expect. The first term on the right, measures
|
||||
how fast the activation function \( f \) is changing at a given activation
|
||||
value \( z_j^L \).
|
||||
|
||||
<p>
|
||||
Notice that everything in the above equations is easily computed. In
|
||||
particular, we compute \( z_j^L \) while computing the behaviour of the
|
||||
network, and it is only a small additional overhead to compute
|
||||
\( f'(z^L_j) \). The exact form of the derivative with respect to the
|
||||
output depends on the form of the cost function.
|
||||
However, provided the cost function is known there should be little
|
||||
trouble in calculating
|
||||
|
||||
$$
|
||||
\frac{\partial {\cal C}}{\partial (a_j^L)}
|
||||
$$
|
||||
|
||||
<p>
|
||||
With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely
|
||||
$$
|
||||
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \delta_j^La_k^{L-1}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week40-bs060.html">«</a></li>
|
||||
<li><a href="._week40-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week40-bs053.html">54</a></li>
|
||||
<li><a href="._week40-bs054.html">55</a></li>
|
||||
<li><a href="._week40-bs055.html">56</a></li>
|
||||
<li><a href="._week40-bs056.html">57</a></li>
|
||||
<li><a href="._week40-bs057.html">58</a></li>
|
||||
<li><a href="._week40-bs058.html">59</a></li>
|
||||
<li><a href="._week40-bs059.html">60</a></li>
|
||||
<li><a href="._week40-bs060.html">61</a></li>
|
||||
<li class="active"><a href="._week40-bs061.html">62</a></li>
|
||||
<li><a href="._week40-bs062.html">63</a></li>
|
||||
<li><a href="._week40-bs063.html">64</a></li>
|
||||
<li><a href="._week40-bs064.html">65</a></li>
|
||||
<li><a href="._week40-bs065.html">66</a></li>
|
||||
<li><a href="._week40-bs062.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 40: From Stochastic Gradient Descent to Neural networks">
|
||||
|
||||
<title>Week 40: From Stochastic Gradient Descent to Neural networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 40', 2, None, 'plan-for-week-40'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('ADAM optimizer', 2, None, 'adam-optimizer'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Recommended to avoid', 2, None, 'recommended-to-avoid'),
|
||||
('Videos on Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'videos-on-neural-networks'),
|
||||
('Neural networks', 2, None, 'neural-networks'),
|
||||
('Artificial neurons', 2, None, 'artificial-neurons'),
|
||||
('Neural network types', 2, None, 'neural-network-types'),
|
||||
('Feed-forward neural networks',
|
||||
2,
|
||||
None,
|
||||
'feed-forward-neural-networks'),
|
||||
('Convolutional Neural Network',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-network'),
|
||||
('Recurrent neural networks',
|
||||
2,
|
||||
None,
|
||||
'recurrent-neural-networks'),
|
||||
('Other types of networks', 2, None, 'other-types-of-networks'),
|
||||
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
||||
('Why multilayer perceptrons?',
|
||||
2,
|
||||
None,
|
||||
'why-multilayer-perceptrons'),
|
||||
('Illustration of a single perceptropn model and a '
|
||||
'multi-perceptron model',
|
||||
2,
|
||||
None,
|
||||
'illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model'),
|
||||
('Examples of XOR, OR and AND gates',
|
||||
2,
|
||||
None,
|
||||
'examples-of-xor-or-and-and-gates'),
|
||||
('Does Logistic Regression do a better Job?',
|
||||
2,
|
||||
None,
|
||||
'does-logistic-regression-do-a-better-job'),
|
||||
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
||||
('Matrix-vector notation and activation',
|
||||
3,
|
||||
None,
|
||||
'matrix-vector-notation-and-activation'),
|
||||
('Activation functions', 3, None, 'activation-functions'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'activation-functions-logistic-and-hyperbolic-ones'),
|
||||
('Relevance', 3, None, 'relevance'),
|
||||
('The multilayer perceptron (MLP)',
|
||||
2,
|
||||
None,
|
||||
'the-multilayer-perceptron-mlp'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
||||
('Definitions', 2, None, 'definitions'),
|
||||
('Derivatives and the chain rule',
|
||||
2,
|
||||
None,
|
||||
'derivatives-and-the-chain-rule'),
|
||||
('Derivative of the cost function',
|
||||
2,
|
||||
None,
|
||||
'derivative-of-the-cost-function'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'bringing-it-together-first-back-propagation-equation'),
|
||||
('Derivatives in terms of $z_j^L$',
|
||||
2,
|
||||
None,
|
||||
'derivatives-in-terms-of-z_j-l'),
|
||||
('Bringing it together', 2, None, 'bringing-it-together'),
|
||||
('Final back propagating equation',
|
||||
2,
|
||||
None,
|
||||
'final-back-propagating-equation'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'setting-up-the-back-propagation-algorithm')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week40-bs.html">Week 40: From Stochastic Gradient Descent to Neural networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plan-for-week-40" style="font-size: 80%;"><b>Plan for week 40</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs002.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs003.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs004.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs005.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs006.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs007.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs008.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs009.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs010.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs011.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs012.html#program-for-stochastic-gradient" style="font-size: 80%;"><b>Program for stochastic gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs013.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs014.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs015.html#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs016.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs017.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs018.html#adam-optimizer" style="font-size: 80%;"><b>ADAM optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs019.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs020.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs021.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs022.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs024.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs025.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs026.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs027.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs028.html#unsupported-functions" style="font-size: 80%;"><b>Unsupported functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs029.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;"><b>The syntax a.dot(b) when finding the dot product</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs030.html#recommended-to-avoid" style="font-size: 80%;"><b>Recommended to avoid</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs031.html#videos-on-neural-networks" style="font-size: 80%;"><b>Videos on Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs032.html#neural-networks" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs033.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs034.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs035.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs036.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs037.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs038.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs039.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs040.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs041.html#illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptropn model and a multi-perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs042.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs043.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs044.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs050.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs051.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs052.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs053.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs054.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs055.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs056.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs057.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs058.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs059.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs060.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs061.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#derivatives-in-terms-of-z_j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs063.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs064.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs065.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0062"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="derivatives-in-terms-of-z_j-l" class="anchor">Derivatives in terms of \( z_j^L \) </h2>
|
||||
|
||||
<p>
|
||||
It is also easy to see that our previous equation can be written as
|
||||
|
||||
$$
|
||||
\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
|
||||
$$
|
||||
|
||||
which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely
|
||||
$$
|
||||
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
|
||||
$$
|
||||
|
||||
That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias.
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week40-bs061.html">«</a></li>
|
||||
<li><a href="._week40-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week40-bs054.html">55</a></li>
|
||||
<li><a href="._week40-bs055.html">56</a></li>
|
||||
<li><a href="._week40-bs056.html">57</a></li>
|
||||
<li><a href="._week40-bs057.html">58</a></li>
|
||||
<li><a href="._week40-bs058.html">59</a></li>
|
||||
<li><a href="._week40-bs059.html">60</a></li>
|
||||
<li><a href="._week40-bs060.html">61</a></li>
|
||||
<li><a href="._week40-bs061.html">62</a></li>
|
||||
<li class="active"><a href="._week40-bs062.html">63</a></li>
|
||||
<li><a href="._week40-bs063.html">64</a></li>
|
||||
<li><a href="._week40-bs064.html">65</a></li>
|
||||
<li><a href="._week40-bs065.html">66</a></li>
|
||||
<li><a href="._week40-bs063.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 40: From Stochastic Gradient Descent to Neural networks">
|
||||
|
||||
<title>Week 40: From Stochastic Gradient Descent to Neural networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 40', 2, None, 'plan-for-week-40'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('ADAM optimizer', 2, None, 'adam-optimizer'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Recommended to avoid', 2, None, 'recommended-to-avoid'),
|
||||
('Videos on Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'videos-on-neural-networks'),
|
||||
('Neural networks', 2, None, 'neural-networks'),
|
||||
('Artificial neurons', 2, None, 'artificial-neurons'),
|
||||
('Neural network types', 2, None, 'neural-network-types'),
|
||||
('Feed-forward neural networks',
|
||||
2,
|
||||
None,
|
||||
'feed-forward-neural-networks'),
|
||||
('Convolutional Neural Network',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-network'),
|
||||
('Recurrent neural networks',
|
||||
2,
|
||||
None,
|
||||
'recurrent-neural-networks'),
|
||||
('Other types of networks', 2, None, 'other-types-of-networks'),
|
||||
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
||||
('Why multilayer perceptrons?',
|
||||
2,
|
||||
None,
|
||||
'why-multilayer-perceptrons'),
|
||||
('Illustration of a single perceptropn model and a '
|
||||
'multi-perceptron model',
|
||||
2,
|
||||
None,
|
||||
'illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model'),
|
||||
('Examples of XOR, OR and AND gates',
|
||||
2,
|
||||
None,
|
||||
'examples-of-xor-or-and-and-gates'),
|
||||
('Does Logistic Regression do a better Job?',
|
||||
2,
|
||||
None,
|
||||
'does-logistic-regression-do-a-better-job'),
|
||||
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
||||
('Matrix-vector notation and activation',
|
||||
3,
|
||||
None,
|
||||
'matrix-vector-notation-and-activation'),
|
||||
('Activation functions', 3, None, 'activation-functions'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'activation-functions-logistic-and-hyperbolic-ones'),
|
||||
('Relevance', 3, None, 'relevance'),
|
||||
('The multilayer perceptron (MLP)',
|
||||
2,
|
||||
None,
|
||||
'the-multilayer-perceptron-mlp'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
||||
('Definitions', 2, None, 'definitions'),
|
||||
('Derivatives and the chain rule',
|
||||
2,
|
||||
None,
|
||||
'derivatives-and-the-chain-rule'),
|
||||
('Derivative of the cost function',
|
||||
2,
|
||||
None,
|
||||
'derivative-of-the-cost-function'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'bringing-it-together-first-back-propagation-equation'),
|
||||
('Derivatives in terms of $z_j^L$',
|
||||
2,
|
||||
None,
|
||||
'derivatives-in-terms-of-z_j-l'),
|
||||
('Bringing it together', 2, None, 'bringing-it-together'),
|
||||
('Final back propagating equation',
|
||||
2,
|
||||
None,
|
||||
'final-back-propagating-equation'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'setting-up-the-back-propagation-algorithm')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week40-bs.html">Week 40: From Stochastic Gradient Descent to Neural networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plan-for-week-40" style="font-size: 80%;"><b>Plan for week 40</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs002.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs003.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs004.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs005.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs006.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs007.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs008.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs009.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs010.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs011.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs012.html#program-for-stochastic-gradient" style="font-size: 80%;"><b>Program for stochastic gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs013.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs014.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs015.html#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs016.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs017.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs018.html#adam-optimizer" style="font-size: 80%;"><b>ADAM optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs019.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs020.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs021.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs022.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs024.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs025.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs026.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs027.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs028.html#unsupported-functions" style="font-size: 80%;"><b>Unsupported functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs029.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;"><b>The syntax a.dot(b) when finding the dot product</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs030.html#recommended-to-avoid" style="font-size: 80%;"><b>Recommended to avoid</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs031.html#videos-on-neural-networks" style="font-size: 80%;"><b>Videos on Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs032.html#neural-networks" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs033.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs034.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs035.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs036.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs037.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs038.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs039.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs040.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs041.html#illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptropn model and a multi-perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs042.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs043.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs044.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs050.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs051.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs052.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs053.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs054.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs055.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs056.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs057.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs058.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs059.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs060.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs061.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs062.html#derivatives-in-terms-of-z_j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs064.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs065.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0063"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="bringing-it-together" class="anchor">Bringing it together </h2>
|
||||
|
||||
<p>
|
||||
We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \delta_j^La_k^{L-1},
|
||||
\tag{18}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
and
|
||||
$$
|
||||
\begin{equation}
|
||||
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
|
||||
\tag{19}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
and
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
|
||||
\tag{20}
|
||||
\end{equation}
|
||||
$$
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
An interesting consequence of the above equations is that when the
|
||||
activation \( a_k^{L-1} \) is small, the gradient term, that is the
|
||||
derivative of the cost function with respect to the weights, will also
|
||||
tend to be small. We say then that the weight learns slowly, meaning
|
||||
that it changes slowly when we minimize the weights via say gradient
|
||||
descent. In this case we say the system learns slowly.
|
||||
|
||||
<p>
|
||||
Another interesting feature is that is when the activation function,
|
||||
represented by the sigmoid function here, is rather flat when we move towards
|
||||
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
||||
cases, the derivatives of the activation function will also be close
|
||||
to zero, meaning again that the gradients will be small and the
|
||||
network learns slowly again.
|
||||
|
||||
<p>
|
||||
We need a fourth equation and we are set. We are going to propagate
|
||||
backwards in order to the determine the weights and biases. In order
|
||||
to do so we need to represent the error in the layer before the final
|
||||
one \( L-1 \) in terms of the errors in the final output layer.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week40-bs062.html">«</a></li>
|
||||
<li><a href="._week40-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week40-bs055.html">56</a></li>
|
||||
<li><a href="._week40-bs056.html">57</a></li>
|
||||
<li><a href="._week40-bs057.html">58</a></li>
|
||||
<li><a href="._week40-bs058.html">59</a></li>
|
||||
<li><a href="._week40-bs059.html">60</a></li>
|
||||
<li><a href="._week40-bs060.html">61</a></li>
|
||||
<li><a href="._week40-bs061.html">62</a></li>
|
||||
<li><a href="._week40-bs062.html">63</a></li>
|
||||
<li class="active"><a href="._week40-bs063.html">64</a></li>
|
||||
<li><a href="._week40-bs064.html">65</a></li>
|
||||
<li><a href="._week40-bs065.html">66</a></li>
|
||||
<li><a href="._week40-bs064.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 40: From Stochastic Gradient Descent to Neural networks">
|
||||
|
||||
<title>Week 40: From Stochastic Gradient Descent to Neural networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 40', 2, None, 'plan-for-week-40'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('ADAM optimizer', 2, None, 'adam-optimizer'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Recommended to avoid', 2, None, 'recommended-to-avoid'),
|
||||
('Videos on Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'videos-on-neural-networks'),
|
||||
('Neural networks', 2, None, 'neural-networks'),
|
||||
('Artificial neurons', 2, None, 'artificial-neurons'),
|
||||
('Neural network types', 2, None, 'neural-network-types'),
|
||||
('Feed-forward neural networks',
|
||||
2,
|
||||
None,
|
||||
'feed-forward-neural-networks'),
|
||||
('Convolutional Neural Network',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-network'),
|
||||
('Recurrent neural networks',
|
||||
2,
|
||||
None,
|
||||
'recurrent-neural-networks'),
|
||||
('Other types of networks', 2, None, 'other-types-of-networks'),
|
||||
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
||||
('Why multilayer perceptrons?',
|
||||
2,
|
||||
None,
|
||||
'why-multilayer-perceptrons'),
|
||||
('Illustration of a single perceptropn model and a '
|
||||
'multi-perceptron model',
|
||||
2,
|
||||
None,
|
||||
'illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model'),
|
||||
('Examples of XOR, OR and AND gates',
|
||||
2,
|
||||
None,
|
||||
'examples-of-xor-or-and-and-gates'),
|
||||
('Does Logistic Regression do a better Job?',
|
||||
2,
|
||||
None,
|
||||
'does-logistic-regression-do-a-better-job'),
|
||||
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
||||
('Matrix-vector notation and activation',
|
||||
3,
|
||||
None,
|
||||
'matrix-vector-notation-and-activation'),
|
||||
('Activation functions', 3, None, 'activation-functions'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'activation-functions-logistic-and-hyperbolic-ones'),
|
||||
('Relevance', 3, None, 'relevance'),
|
||||
('The multilayer perceptron (MLP)',
|
||||
2,
|
||||
None,
|
||||
'the-multilayer-perceptron-mlp'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
||||
('Definitions', 2, None, 'definitions'),
|
||||
('Derivatives and the chain rule',
|
||||
2,
|
||||
None,
|
||||
'derivatives-and-the-chain-rule'),
|
||||
('Derivative of the cost function',
|
||||
2,
|
||||
None,
|
||||
'derivative-of-the-cost-function'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'bringing-it-together-first-back-propagation-equation'),
|
||||
('Derivatives in terms of $z_j^L$',
|
||||
2,
|
||||
None,
|
||||
'derivatives-in-terms-of-z_j-l'),
|
||||
('Bringing it together', 2, None, 'bringing-it-together'),
|
||||
('Final back propagating equation',
|
||||
2,
|
||||
None,
|
||||
'final-back-propagating-equation'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'setting-up-the-back-propagation-algorithm')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week40-bs.html">Week 40: From Stochastic Gradient Descent to Neural networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plan-for-week-40" style="font-size: 80%;"><b>Plan for week 40</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs002.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs003.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs004.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs005.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs006.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs007.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs008.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs009.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs010.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs011.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs012.html#program-for-stochastic-gradient" style="font-size: 80%;"><b>Program for stochastic gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs013.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs014.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs015.html#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs016.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs017.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs018.html#adam-optimizer" style="font-size: 80%;"><b>ADAM optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs019.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs020.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs021.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs022.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs024.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs025.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs026.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs027.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs028.html#unsupported-functions" style="font-size: 80%;"><b>Unsupported functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs029.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;"><b>The syntax a.dot(b) when finding the dot product</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs030.html#recommended-to-avoid" style="font-size: 80%;"><b>Recommended to avoid</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs031.html#videos-on-neural-networks" style="font-size: 80%;"><b>Videos on Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs032.html#neural-networks" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs033.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs034.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs035.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs036.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs037.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs038.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs039.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs040.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs041.html#illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptropn model and a multi-perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs042.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs043.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs044.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs050.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs051.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs052.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs053.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs054.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs055.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs056.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs057.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs058.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs059.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs060.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs061.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs062.html#derivatives-in-terms-of-z_j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs063.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs065.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0064"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="final-back-propagating-equation" class="anchor">Final back propagating equation </h2>
|
||||
|
||||
<p>
|
||||
We have that (replacing \( L \) with a general layer \( l \))
|
||||
$$
|
||||
\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
|
||||
$$
|
||||
|
||||
We want to express this in terms of the equations for layer \( l+1 \). Using the chain rule and summing over all \( k \) entries we have
|
||||
|
||||
$$
|
||||
\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
|
||||
$$
|
||||
|
||||
and recalling that
|
||||
$$
|
||||
z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
|
||||
$$
|
||||
|
||||
with \( M_l \) being the number of nodes in layer \( l \), we obtain
|
||||
$$
|
||||
\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
|
||||
$$
|
||||
|
||||
This is our final equation.
|
||||
|
||||
<p>
|
||||
We are now ready to set up the algorithm for back propagation and learning the weights and biases.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week40-bs063.html">«</a></li>
|
||||
<li><a href="._week40-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week40-bs056.html">57</a></li>
|
||||
<li><a href="._week40-bs057.html">58</a></li>
|
||||
<li><a href="._week40-bs058.html">59</a></li>
|
||||
<li><a href="._week40-bs059.html">60</a></li>
|
||||
<li><a href="._week40-bs060.html">61</a></li>
|
||||
<li><a href="._week40-bs061.html">62</a></li>
|
||||
<li><a href="._week40-bs062.html">63</a></li>
|
||||
<li><a href="._week40-bs063.html">64</a></li>
|
||||
<li class="active"><a href="._week40-bs064.html">65</a></li>
|
||||
<li><a href="._week40-bs065.html">66</a></li>
|
||||
<li><a href="._week40-bs065.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/doconce/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 40: From Stochastic Gradient Descent to Neural networks">
|
||||
|
||||
<title>Week 40: From Stochastic Gradient Descent to Neural networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 40', 2, None, 'plan-for-week-40'),
|
||||
('Overview video on Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'overview-video-on-stochastic-gradient-descent'),
|
||||
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
||||
('Stochastic Gradient Descent (SGD)',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent-sgd'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
2,
|
||||
None,
|
||||
'more-on-momentum-based-approaches'),
|
||||
('Momentum parameter', 2, None, 'momentum-parameter'),
|
||||
('Second moment of the gradient',
|
||||
2,
|
||||
None,
|
||||
'second-moment-of-the-gradient'),
|
||||
('RMS prop', 2, None, 'rms-prop'),
|
||||
('ADAM optimizer', 2, None, 'adam-optimizer'),
|
||||
('Practical tips', 2, None, 'practical-tips'),
|
||||
('Automatic differentiation',
|
||||
2,
|
||||
None,
|
||||
'automatic-differentiation'),
|
||||
('Using autograd', 2, None, 'using-autograd'),
|
||||
('Autograd with more complicated functions',
|
||||
2,
|
||||
None,
|
||||
'autograd-with-more-complicated-functions'),
|
||||
('More complicated functions using the elements of their '
|
||||
'arguments directly',
|
||||
2,
|
||||
None,
|
||||
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
||||
('Functions using mathematical functions from Numpy',
|
||||
2,
|
||||
None,
|
||||
'functions-using-mathematical-functions-from-numpy'),
|
||||
('More autograd', 2, None, 'more-autograd'),
|
||||
('And with loops', 2, None, 'and-with-loops'),
|
||||
('Using recursion', 2, None, 'using-recursion'),
|
||||
('Unsupported functions', 2, None, 'unsupported-functions'),
|
||||
('The syntax a.dot(b) when finding the dot product',
|
||||
2,
|
||||
None,
|
||||
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
||||
('Recommended to avoid', 2, None, 'recommended-to-avoid'),
|
||||
('Videos on Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'videos-on-neural-networks'),
|
||||
('Neural networks', 2, None, 'neural-networks'),
|
||||
('Artificial neurons', 2, None, 'artificial-neurons'),
|
||||
('Neural network types', 2, None, 'neural-network-types'),
|
||||
('Feed-forward neural networks',
|
||||
2,
|
||||
None,
|
||||
'feed-forward-neural-networks'),
|
||||
('Convolutional Neural Network',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-network'),
|
||||
('Recurrent neural networks',
|
||||
2,
|
||||
None,
|
||||
'recurrent-neural-networks'),
|
||||
('Other types of networks', 2, None, 'other-types-of-networks'),
|
||||
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
||||
('Why multilayer perceptrons?',
|
||||
2,
|
||||
None,
|
||||
'why-multilayer-perceptrons'),
|
||||
('Illustration of a single perceptropn model and a '
|
||||
'multi-perceptron model',
|
||||
2,
|
||||
None,
|
||||
'illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model'),
|
||||
('Examples of XOR, OR and AND gates',
|
||||
2,
|
||||
None,
|
||||
'examples-of-xor-or-and-and-gates'),
|
||||
('Does Logistic Regression do a better Job?',
|
||||
2,
|
||||
None,
|
||||
'does-logistic-regression-do-a-better-job'),
|
||||
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Mathematical model', 2, None, 'mathematical-model'),
|
||||
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
||||
('Matrix-vector notation and activation',
|
||||
3,
|
||||
None,
|
||||
'matrix-vector-notation-and-activation'),
|
||||
('Activation functions', 3, None, 'activation-functions'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'activation-functions-logistic-and-hyperbolic-ones'),
|
||||
('Relevance', 3, None, 'relevance'),
|
||||
('The multilayer perceptron (MLP)',
|
||||
2,
|
||||
None,
|
||||
'the-multilayer-perceptron-mlp'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
||||
('Definitions', 2, None, 'definitions'),
|
||||
('Derivatives and the chain rule',
|
||||
2,
|
||||
None,
|
||||
'derivatives-and-the-chain-rule'),
|
||||
('Derivative of the cost function',
|
||||
2,
|
||||
None,
|
||||
'derivative-of-the-cost-function'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'bringing-it-together-first-back-propagation-equation'),
|
||||
('Derivatives in terms of $z_j^L$',
|
||||
2,
|
||||
None,
|
||||
'derivatives-in-terms-of-z_j-l'),
|
||||
('Bringing it together', 2, None, 'bringing-it-together'),
|
||||
('Final back propagating equation',
|
||||
2,
|
||||
None,
|
||||
'final-back-propagating-equation'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'setting-up-the-back-propagation-algorithm')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week40-bs.html">Week 40: From Stochastic Gradient Descent to Neural networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plan-for-week-40" style="font-size: 80%;"><b>Plan for week 40</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs002.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs003.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs004.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs005.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs006.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs007.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs008.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs009.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs010.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs011.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs012.html#program-for-stochastic-gradient" style="font-size: 80%;"><b>Program for stochastic gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs013.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs014.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs015.html#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs016.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs017.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs018.html#adam-optimizer" style="font-size: 80%;"><b>ADAM optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs019.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs020.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs021.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs022.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs024.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs025.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs026.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs027.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs028.html#unsupported-functions" style="font-size: 80%;"><b>Unsupported functions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs029.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;"><b>The syntax a.dot(b) when finding the dot product</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs030.html#recommended-to-avoid" style="font-size: 80%;"><b>Recommended to avoid</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs031.html#videos-on-neural-networks" style="font-size: 80%;"><b>Videos on Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs032.html#neural-networks" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs033.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs034.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs035.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs036.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs037.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs038.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs039.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs040.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs041.html#illustration-of-a-single-perceptropn-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptropn model and a multi-perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs042.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs043.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs044.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs049.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs050.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs051.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs052.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs053.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs054.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs055.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs056.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs057.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs058.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs059.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs060.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs061.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs062.html#derivatives-in-terms-of-z_j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs063.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week40-bs064.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0065"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="setting-up-the-back-propagation-algorithm" class="anchor">Setting up the Back propagation algorithm </h2>
|
||||
|
||||
<p>
|
||||
The four equations provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
First, we set up the input data \( \hat{x} \) and the activations
|
||||
\( \hat{z}_1 \) of the input layer and compute the activation function and
|
||||
the pertinent outputs \( \hat{a}^1 \).
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
Secondly, we perform then the feed forward till we reach the output
|
||||
layer and compute all \( \hat{z}_l \) of the input layer and compute the
|
||||
activation function and the pertinent outputs \( \hat{a}^l \) for
|
||||
\( l=2,3,\dots,L \).
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
Thereafter we compute the ouput error \( \hat{\delta}^L \) by computing all
|
||||
$$
|
||||
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
|
||||
$$
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as
|
||||
$$
|
||||
\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
|
||||
$$
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules
|
||||
$$
|
||||
w_{jk}^l\leftarrow = w_{jk}^l- \eta \delta_j^la_k^{l-1},
|
||||
$$
|
||||
|
||||
|
||||
$$
|
||||
b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
|
||||
$$
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
|
||||
Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
|
||||
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._week40-bs064.html">«</a></li>
|
||||
<li><a href="._week40-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week40-bs057.html">58</a></li>
|
||||
<li><a href="._week40-bs058.html">59</a></li>
|
||||
<li><a href="._week40-bs059.html">60</a></li>
|
||||
<li><a href="._week40-bs060.html">61</a></li>
|
||||
<li><a href="._week40-bs061.html">62</a></li>
|
||||
<li><a href="._week40-bs062.html">63</a></li>
|
||||
<li><a href="._week40-bs063.html">64</a></li>
|
||||
<li><a href="._week40-bs064.html">65</a></li>
|
||||
<li class="active"><a href="._week40-bs065.html">66</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user