Morten Hjorth-Jensen
431 lines
25 KiB
HTML
431 lines
25 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
|
|
<title>Week 41 Neural networks and constructing a neural network code</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
|
|
('Lecture Thursday October 12',
|
|
2,
|
|
None,
|
|
'lecture-thursday-october-12'),
|
|
('Introduction to Neural networks',
|
|
2,
|
|
None,
|
|
'introduction-to-neural-networks'),
|
|
('Artificial neurons', 2, None, 'artificial-neurons'),
|
|
('Neural network types', 2, None, 'neural-network-types'),
|
|
('Feed-forward neural networks',
|
|
2,
|
|
None,
|
|
'feed-forward-neural-networks'),
|
|
('Convolutional Neural Network',
|
|
2,
|
|
None,
|
|
'convolutional-neural-network'),
|
|
('Recurrent neural networks',
|
|
2,
|
|
None,
|
|
'recurrent-neural-networks'),
|
|
('Other types of networks', 2, None, 'other-types-of-networks'),
|
|
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
|
('Why multilayer perceptrons?',
|
|
2,
|
|
None,
|
|
'why-multilayer-perceptrons'),
|
|
('Illustration of a single perceptron model and a '
|
|
'multi-perceptron model',
|
|
2,
|
|
None,
|
|
'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
|
|
('Examples of XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'examples-of-xor-or-and-and-gates'),
|
|
('Does Logistic Regression do a better Job?',
|
|
2,
|
|
None,
|
|
'does-logistic-regression-do-a-better-job'),
|
|
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
|
('Matrix-vector notation and activation',
|
|
3,
|
|
None,
|
|
'matrix-vector-notation-and-activation'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 3, None, 'relevance'),
|
|
('The multilayer perceptron (MLP)',
|
|
2,
|
|
None,
|
|
'the-multilayer-perceptron-mlp'),
|
|
('From one to many layers, the universal approximation theorem',
|
|
2,
|
|
None,
|
|
'from-one-to-many-layers-the-universal-approximation-theorem'),
|
|
('Deriving the back propagation code for a multilayer perceptron '
|
|
'model',
|
|
2,
|
|
None,
|
|
'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
|
|
('Definitions', 2, None, 'definitions'),
|
|
('Derivatives and the chain rule',
|
|
2,
|
|
None,
|
|
'derivatives-and-the-chain-rule'),
|
|
('Derivative of the cost function',
|
|
2,
|
|
None,
|
|
'derivative-of-the-cost-function'),
|
|
('Bringing it together, first back propagation equation',
|
|
2,
|
|
None,
|
|
'bringing-it-together-first-back-propagation-equation'),
|
|
('Derivatives in terms of $z_j^L$',
|
|
2,
|
|
None,
|
|
'derivatives-in-terms-of-z-j-l'),
|
|
('Bringing it together', 2, None, 'bringing-it-together'),
|
|
('Final back propagating equation',
|
|
2,
|
|
None,
|
|
'final-back-propagating-equation'),
|
|
('Setting up the Back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up the Back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up the Back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up a Multi-layer perceptron model for classification',
|
|
2,
|
|
None,
|
|
'setting-up-a-multi-layer-perceptron-model-for-classification'),
|
|
('Defining the cost function',
|
|
2,
|
|
None,
|
|
'defining-the-cost-function'),
|
|
('Example: binary classification problem',
|
|
2,
|
|
None,
|
|
'example-binary-classification-problem'),
|
|
('The Softmax function', 2, None, 'the-softmax-function'),
|
|
('Developing a code for doing neural networks with back '
|
|
'propagation',
|
|
2,
|
|
None,
|
|
'developing-a-code-for-doing-neural-networks-with-back-propagation'),
|
|
('Collect and pre-process data',
|
|
2,
|
|
None,
|
|
'collect-and-pre-process-data'),
|
|
('Train and test datasets', 2, None, 'train-and-test-datasets'),
|
|
('Define model and architecture',
|
|
2,
|
|
None,
|
|
'define-model-and-architecture'),
|
|
('Layers', 2, None, 'layers'),
|
|
('Weights and biases', 2, None, 'weights-and-biases'),
|
|
('Feed-forward pass', 2, None, 'feed-forward-pass'),
|
|
('Matrix multiplications', 2, None, 'matrix-multiplications'),
|
|
('Choose cost function and optimizer',
|
|
2,
|
|
None,
|
|
'choose-cost-function-and-optimizer'),
|
|
('Optimizing the cost function',
|
|
2,
|
|
None,
|
|
'optimizing-the-cost-function'),
|
|
('Regularization', 2, None, 'regularization'),
|
|
('Matrix multiplication', 2, None, 'matrix-multiplication'),
|
|
('Improving performance', 2, None, 'improving-performance'),
|
|
('Full object-oriented implementation',
|
|
2,
|
|
None,
|
|
'full-object-oriented-implementation'),
|
|
('Evaluate model performance on test data',
|
|
2,
|
|
None,
|
|
'evaluate-model-performance-on-test-data'),
|
|
('Adjust hyperparameters', 2, None, 'adjust-hyperparameters'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('scikit-learn implementation',
|
|
2,
|
|
None,
|
|
'scikit-learn-implementation'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('Testing our code for the XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'testing-our-code-for-the-xor-or-and-and-gates'),
|
|
('The AND and XOR Gates', 2, None, 'the-and-and-xor-gates'),
|
|
('Representing the Data Sets',
|
|
2,
|
|
None,
|
|
'representing-the-data-sets'),
|
|
('Setting up the Neural Network',
|
|
2,
|
|
None,
|
|
'setting-up-the-neural-network'),
|
|
('The Code using Scikit-Learn',
|
|
2,
|
|
None,
|
|
'the-code-using-scikit-learn')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0034"></a>
|
|
<!-- !split -->
|
|
<h2 id="bringing-it-together" class="anchor">Bringing it together </h2>
|
|
|
|
<p>We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are</p>
|
|
|
|
<div class="panel panel-default">
|
|
<div class="panel-body">
|
|
<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
|
|
|
$$
|
|
\begin{equation}
|
|
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \delta_j^La_k^{L-1},
|
|
\tag{13}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>and</p>
|
|
$$
|
|
\begin{equation}
|
|
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
|
|
\tag{14}
|
|
\end{equation}
|
|
$$
|
|
|
|
<p>and</p>
|
|
|
|
$$
|
|
\begin{equation}
|
|
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
|
|
\tag{15}
|
|
\end{equation}
|
|
$$
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<p>An interesting consequence of the above equations is that when the
|
|
activation \( a_k^{L-1} \) is small, the gradient term, that is the
|
|
derivative of the cost function with respect to the weights, will also
|
|
tend to be small. We say then that the weight learns slowly, meaning
|
|
that it changes slowly when we minimize the weights via say gradient
|
|
descent. In this case we say the system learns slowly.
|
|
</p>
|
|
|
|
<p>Another interesting feature is that is when the activation function,
|
|
represented by the sigmoid function here, is rather flat when we move towards
|
|
its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
|
|
cases, the derivatives of the activation function will also be close
|
|
to zero, meaning again that the gradients will be small and the
|
|
network learns slowly again.
|
|
</p>
|
|
|
|
<p>We need a fourth equation and we are set. We are going to propagate
|
|
backwards in order to the determine the weights and biases. In order
|
|
to do so we need to represent the error in the layer before the final
|
|
one \( L-1 \) in terms of the errors in the final output layer.
|
|
</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week41-bs033.html">«</a></li>
|
|
<li><a href="._week41-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs026.html">27</a></li>
|
|
<li><a href="._week41-bs027.html">28</a></li>
|
|
<li><a href="._week41-bs028.html">29</a></li>
|
|
<li><a href="._week41-bs029.html">30</a></li>
|
|
<li><a href="._week41-bs030.html">31</a></li>
|
|
<li><a href="._week41-bs031.html">32</a></li>
|
|
<li><a href="._week41-bs032.html">33</a></li>
|
|
<li><a href="._week41-bs033.html">34</a></li>
|
|
<li class="active"><a href="._week41-bs034.html">35</a></li>
|
|
<li><a href="._week41-bs035.html">36</a></li>
|
|
<li><a href="._week41-bs036.html">37</a></li>
|
|
<li><a href="._week41-bs037.html">38</a></li>
|
|
<li><a href="._week41-bs038.html">39</a></li>
|
|
<li><a href="._week41-bs039.html">40</a></li>
|
|
<li><a href="._week41-bs040.html">41</a></li>
|
|
<li><a href="._week41-bs041.html">42</a></li>
|
|
<li><a href="._week41-bs042.html">43</a></li>
|
|
<li><a href="._week41-bs043.html">44</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs066.html">67</a></li>
|
|
<li><a href="._week41-bs035.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|