Morten Hjorth-Jensen
584 lines
37 KiB
HTML
584 lines
37 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
|
|
<title>Week 41 Neural networks and constructing a neural network code</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 41, October 7-11',
|
|
2,
|
|
None,
|
|
'plan-for-week-41-october-7-11'),
|
|
('Material for the lecture on Monday October 7, 2024',
|
|
2,
|
|
None,
|
|
'material-for-the-lecture-on-monday-october-7-2024'),
|
|
('Material for the active learning sessions on Tuesday and '
|
|
'Wednesday',
|
|
2,
|
|
None,
|
|
'material-for-the-active-learning-sessions-on-tuesday-and-wednesday'),
|
|
('Lecture Monday October 7',
|
|
2,
|
|
None,
|
|
'lecture-monday-october-7'),
|
|
('Introduction to Neural networks',
|
|
2,
|
|
None,
|
|
'introduction-to-neural-networks'),
|
|
('Artificial neurons', 2, None, 'artificial-neurons'),
|
|
('Neural network types', 2, None, 'neural-network-types'),
|
|
('Feed-forward neural networks',
|
|
2,
|
|
None,
|
|
'feed-forward-neural-networks'),
|
|
('Convolutional Neural Network',
|
|
2,
|
|
None,
|
|
'convolutional-neural-network'),
|
|
('Recurrent neural networks',
|
|
2,
|
|
None,
|
|
'recurrent-neural-networks'),
|
|
('Other types of networks', 2, None, 'other-types-of-networks'),
|
|
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
|
('Why multilayer perceptrons?',
|
|
2,
|
|
None,
|
|
'why-multilayer-perceptrons'),
|
|
('Illustration of a single perceptron model and a '
|
|
'multi-perceptron model',
|
|
2,
|
|
None,
|
|
'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
|
|
('Examples of XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'examples-of-xor-or-and-and-gates'),
|
|
('Does Logistic Regression do a better Job?',
|
|
2,
|
|
None,
|
|
'does-logistic-regression-do-a-better-job'),
|
|
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
|
('Mathematics of deep learning',
|
|
2,
|
|
None,
|
|
'mathematics-of-deep-learning'),
|
|
('Reminder on books with hands-on material and codes',
|
|
2,
|
|
None,
|
|
'reminder-on-books-with-hands-on-material-and-codes'),
|
|
('Reading recommendations', 2, None, 'reading-recommendations'),
|
|
('Mathematics of deep learning and neural networks',
|
|
2,
|
|
None,
|
|
'mathematics-of-deep-learning-and-neural-networks'),
|
|
('Basics of an NN', 2, None, 'basics-of-an-nn'),
|
|
('Overarching view of a neural network',
|
|
2,
|
|
None,
|
|
'overarching-view-of-a-neural-network'),
|
|
('The optimization problem', 2, None, 'the-optimization-problem'),
|
|
('Parameters of neural networks',
|
|
2,
|
|
None,
|
|
'parameters-of-neural-networks'),
|
|
('Other ingredients of a neural network',
|
|
2,
|
|
None,
|
|
'other-ingredients-of-a-neural-network'),
|
|
('Other parameters', 2, None, 'other-parameters'),
|
|
('Universal approximation theorem',
|
|
2,
|
|
None,
|
|
'universal-approximation-theorem'),
|
|
('Some parallels from real analysis',
|
|
2,
|
|
None,
|
|
'some-parallels-from-real-analysis'),
|
|
('The approximation theorem in words',
|
|
2,
|
|
None,
|
|
'the-approximation-theorem-in-words'),
|
|
('More on the general approximation theorem',
|
|
2,
|
|
None,
|
|
'more-on-the-general-approximation-theorem'),
|
|
('Class of functions we can approximate',
|
|
2,
|
|
None,
|
|
'class-of-functions-we-can-approximate'),
|
|
('Setting up the equations for a neural network',
|
|
2,
|
|
None,
|
|
'setting-up-the-equations-for-a-neural-network'),
|
|
('Layout of a neural network with three hidden layers',
|
|
2,
|
|
None,
|
|
'layout-of-a-neural-network-with-three-hidden-layers'),
|
|
('Definitions', 2, None, 'definitions'),
|
|
('Inputs to the activation function',
|
|
2,
|
|
None,
|
|
'inputs-to-the-activation-function'),
|
|
('Derivatives and the chain rule',
|
|
2,
|
|
None,
|
|
'derivatives-and-the-chain-rule'),
|
|
('Derivative of the cost function',
|
|
2,
|
|
None,
|
|
'derivative-of-the-cost-function'),
|
|
('Simpler examples first, and automatic differentiation',
|
|
2,
|
|
None,
|
|
'simpler-examples-first-and-automatic-differentiation'),
|
|
('Reminder on the chain rule and gradients',
|
|
2,
|
|
None,
|
|
'reminder-on-the-chain-rule-and-gradients'),
|
|
('Multivariable functions', 2, None, 'multivariable-functions'),
|
|
('Automatic differentiation through examples',
|
|
2,
|
|
None,
|
|
'automatic-differentiation-through-examples'),
|
|
('Simple example', 2, None, 'simple-example'),
|
|
('Smarter way of evaluating the above function',
|
|
2,
|
|
None,
|
|
'smarter-way-of-evaluating-the-above-function'),
|
|
('Reducing the number of operations',
|
|
2,
|
|
None,
|
|
'reducing-the-number-of-operations'),
|
|
('Chain rule, forward and reverse modes',
|
|
2,
|
|
None,
|
|
'chain-rule-forward-and-reverse-modes'),
|
|
('Forward and reverse modes',
|
|
2,
|
|
None,
|
|
'forward-and-reverse-modes'),
|
|
('More complicated function',
|
|
2,
|
|
None,
|
|
'more-complicated-function'),
|
|
('Counting the number of floating point operations',
|
|
2,
|
|
None,
|
|
'counting-the-number-of-floating-point-operations'),
|
|
('Defining intermediate operations',
|
|
2,
|
|
None,
|
|
'defining-intermediate-operations'),
|
|
('New expression for the derivative',
|
|
2,
|
|
None,
|
|
'new-expression-for-the-derivative'),
|
|
('Final derivatives', 2, None, 'final-derivatives'),
|
|
('In general not this simple',
|
|
2,
|
|
None,
|
|
'in-general-not-this-simple'),
|
|
('Automatic differentiation',
|
|
2,
|
|
None,
|
|
'automatic-differentiation'),
|
|
('Chain rule', 2, None, 'chain-rule'),
|
|
('First network example, simple percepetron with one input',
|
|
2,
|
|
None,
|
|
'first-network-example-simple-percepetron-with-one-input'),
|
|
('Layout of a simple neural network with no hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-no-hidden-layer'),
|
|
('Optimizing the parameters',
|
|
2,
|
|
None,
|
|
'optimizing-the-parameters'),
|
|
('Adding a hidden layer', 2, None, 'adding-a-hidden-layer'),
|
|
('Layout of a simple neural network with one hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-one-hidden-layer'),
|
|
('The derivatives', 2, None, 'the-derivatives'),
|
|
('Important observations', 2, None, 'important-observations'),
|
|
('The training', 2, None, 'the-training'),
|
|
('Code example', 2, None, 'code-example'),
|
|
('Exercise 1: Including more data',
|
|
2,
|
|
None,
|
|
'exercise-1-including-more-data'),
|
|
('Simple neural network and the back propagation equations',
|
|
2,
|
|
None,
|
|
'simple-neural-network-and-the-back-propagation-equations'),
|
|
('Layout of a simple neural network with two input nodes, one '
|
|
'hidden layer and one output node',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-and-one-output-node'),
|
|
('The ouput layer', 2, None, 'the-ouput-layer'),
|
|
('Compact expressions', 2, None, 'compact-expressions'),
|
|
('Output layer', 2, None, 'output-layer'),
|
|
('Explicit derivatives', 2, None, 'explicit-derivatives'),
|
|
('Derivatives of the hidden layer',
|
|
2,
|
|
None,
|
|
'derivatives-of-the-hidden-layer'),
|
|
('Final expression', 2, None, 'final-expression'),
|
|
('Completing the list', 2, None, 'completing-the-list'),
|
|
('Final expressions for the biases of the hidden layer',
|
|
2,
|
|
None,
|
|
'final-expressions-for-the-biases-of-the-hidden-layer'),
|
|
('Gradient expressions', 2, None, 'gradient-expressions'),
|
|
('Exercise 2: Extended program',
|
|
2,
|
|
None,
|
|
'exercise-2-extended-program'),
|
|
('Getting serious, the back propagation equations for a neural '
|
|
'network',
|
|
2,
|
|
None,
|
|
'getting-serious-the-back-propagation-equations-for-a-neural-network'),
|
|
('Analyzing the last results',
|
|
2,
|
|
None,
|
|
'analyzing-the-last-results'),
|
|
('More considerations', 2, None, 'more-considerations'),
|
|
('Derivatives in terms of $z_j^L$',
|
|
2,
|
|
None,
|
|
'derivatives-in-terms-of-z-j-l'),
|
|
('Bringing it together', 2, None, 'bringing-it-together'),
|
|
('Final back propagating equation',
|
|
2,
|
|
None,
|
|
'final-back-propagating-equation'),
|
|
('Using the chain rule and summing over all $k$ entries',
|
|
2,
|
|
None,
|
|
'using-the-chain-rule-and-summing-over-all-k-entries'),
|
|
('Setting up the back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up the back propagation algorithm, part 2',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-2'),
|
|
('Setting up the Back propagation algorithm, part 3',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-3'),
|
|
('Updating the gradients', 2, None, 'updating-the-gradients'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 3, None, 'relevance'),
|
|
('Fine-tuning neural network hyperparameters',
|
|
2,
|
|
None,
|
|
'fine-tuning-neural-network-hyperparameters'),
|
|
('Hidden layers', 2, None, 'hidden-layers'),
|
|
('Vanishing gradients', 2, None, 'vanishing-gradients'),
|
|
('Exploding gradients', 2, None, 'exploding-gradients'),
|
|
('Is the Logistic activation function (Sigmoid) our choice?',
|
|
2,
|
|
None,
|
|
'is-the-logistic-activation-function-sigmoid-our-choice'),
|
|
('Logistic function as the root of problems',
|
|
2,
|
|
None,
|
|
'logistic-function-as-the-root-of-problems'),
|
|
('The derivative of the Logistic funtion',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-logistic-funtion'),
|
|
('Insights from the paper by Glorot and Bengio',
|
|
2,
|
|
None,
|
|
'insights-from-the-paper-by-glorot-and-bengio'),
|
|
('The RELU function family', 2, None, 'the-relu-function-family'),
|
|
('ELU function', 2, None, 'elu-function'),
|
|
('Which activation function should we use?',
|
|
2,
|
|
None,
|
|
'which-activation-function-should-we-use'),
|
|
('More on activation functions, output layers',
|
|
2,
|
|
None,
|
|
'more-on-activation-functions-output-layers'),
|
|
('Batch Normalization', 2, None, 'batch-normalization'),
|
|
('Dropout', 2, None, 'dropout'),
|
|
('Gradient Clipping', 2, None, 'gradient-clipping'),
|
|
('A top-down perspective on Neural networks',
|
|
2,
|
|
None,
|
|
'a-top-down-perspective-on-neural-networks'),
|
|
('More top-down perspectives',
|
|
2,
|
|
None,
|
|
'more-top-down-perspectives'),
|
|
('Limitations of supervised learning with deep networks',
|
|
2,
|
|
None,
|
|
'limitations-of-supervised-learning-with-deep-networks'),
|
|
('Limitations of NNs', 2, None, 'limitations-of-nns'),
|
|
('Homogeneous data', 2, None, 'homogeneous-data'),
|
|
('More limitations', 2, None, 'more-limitations')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41-october-7-11" style="font-size: 80%;"><b>Plan for week 41, October 7-11</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs002.html#material-for-the-lecture-on-monday-october-7-2024" style="font-size: 80%;"><b>Material for the lecture on Monday October 7, 2024</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs003.html#material-for-the-active-learning-sessions-on-tuesday-and-wednesday" style="font-size: 80%;"><b>Material for the active learning sessions on Tuesday and Wednesday</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs004.html#lecture-monday-october-7" style="font-size: 80%;"><b>Lecture Monday October 7</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs005.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs006.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs007.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs008.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs009.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs010.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs011.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs012.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs013.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs014.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs015.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs016.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs017.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs018.html#mathematics-of-deep-learning" style="font-size: 80%;"><b>Mathematics of deep learning</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs019.html#reminder-on-books-with-hands-on-material-and-codes" style="font-size: 80%;"><b>Reminder on books with hands-on material and codes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#reading-recommendations" style="font-size: 80%;"><b>Reading recommendations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs021.html#mathematics-of-deep-learning-and-neural-networks" style="font-size: 80%;"><b>Mathematics of deep learning and neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs022.html#basics-of-an-nn" style="font-size: 80%;"><b>Basics of an NN</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs023.html#overarching-view-of-a-neural-network" style="font-size: 80%;"><b>Overarching view of a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs024.html#the-optimization-problem" style="font-size: 80%;"><b>The optimization problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs025.html#parameters-of-neural-networks" style="font-size: 80%;"><b>Parameters of neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs026.html#other-ingredients-of-a-neural-network" style="font-size: 80%;"><b>Other ingredients of a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs027.html#other-parameters" style="font-size: 80%;"><b>Other parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs028.html#universal-approximation-theorem" style="font-size: 80%;"><b>Universal approximation theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs029.html#some-parallels-from-real-analysis" style="font-size: 80%;"><b>Some parallels from real analysis</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs030.html#the-approximation-theorem-in-words" style="font-size: 80%;"><b>The approximation theorem in words</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs031.html#more-on-the-general-approximation-theorem" style="font-size: 80%;"><b>More on the general approximation theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs032.html#class-of-functions-we-can-approximate" style="font-size: 80%;"><b>Class of functions we can approximate</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs033.html#setting-up-the-equations-for-a-neural-network" style="font-size: 80%;"><b>Setting up the equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs034.html#layout-of-a-neural-network-with-three-hidden-layers" style="font-size: 80%;"><b>Layout of a neural network with three hidden layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs035.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs036.html#inputs-to-the-activation-function" style="font-size: 80%;"><b>Inputs to the activation function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs039.html#simpler-examples-first-and-automatic-differentiation" style="font-size: 80%;"><b>Simpler examples first, and automatic differentiation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs040.html#reminder-on-the-chain-rule-and-gradients" style="font-size: 80%;"><b>Reminder on the chain rule and gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs041.html#multivariable-functions" style="font-size: 80%;"><b>Multivariable functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs042.html#automatic-differentiation-through-examples" style="font-size: 80%;"><b>Automatic differentiation through examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs043.html#simple-example" style="font-size: 80%;"><b>Simple example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs044.html#smarter-way-of-evaluating-the-above-function" style="font-size: 80%;"><b>Smarter way of evaluating the above function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs045.html#reducing-the-number-of-operations" style="font-size: 80%;"><b>Reducing the number of operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs046.html#chain-rule-forward-and-reverse-modes" style="font-size: 80%;"><b>Chain rule, forward and reverse modes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs047.html#forward-and-reverse-modes" style="font-size: 80%;"><b>Forward and reverse modes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs048.html#more-complicated-function" style="font-size: 80%;"><b>More complicated function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs049.html#counting-the-number-of-floating-point-operations" style="font-size: 80%;"><b>Counting the number of floating point operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs050.html#defining-intermediate-operations" style="font-size: 80%;"><b>Defining intermediate operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs051.html#new-expression-for-the-derivative" style="font-size: 80%;"><b>New expression for the derivative</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs052.html#final-derivatives" style="font-size: 80%;"><b>Final derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs053.html#in-general-not-this-simple" style="font-size: 80%;"><b>In general not this simple</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs054.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs055.html#chain-rule" style="font-size: 80%;"><b>Chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs056.html#first-network-example-simple-percepetron-with-one-input" style="font-size: 80%;"><b>First network example, simple percepetron with one input</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs057.html#layout-of-a-simple-neural-network-with-no-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with no hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs058.html#optimizing-the-parameters" style="font-size: 80%;"><b>Optimizing the parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs059.html#adding-a-hidden-layer" style="font-size: 80%;"><b>Adding a hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs060.html#layout-of-a-simple-neural-network-with-one-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with one hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs061.html#the-derivatives" style="font-size: 80%;"><b>The derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs062.html#important-observations" style="font-size: 80%;"><b>Important observations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs063.html#the-training" style="font-size: 80%;"><b>The training</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs064.html#code-example" style="font-size: 80%;"><b>Code example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs065.html#exercise-1-including-more-data" style="font-size: 80%;"><b>Exercise 1: Including more data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs066.html#simple-neural-network-and-the-back-propagation-equations" style="font-size: 80%;"><b>Simple neural network and the back propagation equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs067.html#layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-and-one-output-node" style="font-size: 80%;"><b>Layout of a simple neural network with two input nodes, one hidden layer and one output node</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs068.html#the-ouput-layer" style="font-size: 80%;"><b>The ouput layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs069.html#compact-expressions" style="font-size: 80%;"><b>Compact expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs070.html#output-layer" style="font-size: 80%;"><b>Output layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs071.html#explicit-derivatives" style="font-size: 80%;"><b>Explicit derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs072.html#derivatives-of-the-hidden-layer" style="font-size: 80%;"><b>Derivatives of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs073.html#final-expression" style="font-size: 80%;"><b>Final expression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs074.html#completing-the-list" style="font-size: 80%;"><b>Completing the list</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs075.html#final-expressions-for-the-biases-of-the-hidden-layer" style="font-size: 80%;"><b>Final expressions for the biases of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs076.html#gradient-expressions" style="font-size: 80%;"><b>Gradient expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs077.html#exercise-2-extended-program" style="font-size: 80%;"><b>Exercise 2: Extended program</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs078.html#getting-serious-the-back-propagation-equations-for-a-neural-network" style="font-size: 80%;"><b>Getting serious, the back propagation equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs079.html#analyzing-the-last-results" style="font-size: 80%;"><b>Analyzing the last results</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs080.html#more-considerations" style="font-size: 80%;"><b>More considerations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs081.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs082.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs083.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs084.html#using-the-chain-rule-and-summing-over-all-k-entries" style="font-size: 80%;"><b>Using the chain rule and summing over all \( k \) entries</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs085.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the back propagation algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-back-propagation-algorithm-part-2" style="font-size: 80%;"><b>Setting up the back propagation algorithm, part 2</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs087.html#setting-up-the-back-propagation-algorithm-part-3" style="font-size: 80%;"><b>Setting up the Back propagation algorithm, part 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs088.html#updating-the-gradients" style="font-size: 80%;"><b>Updating the gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs089.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs090.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs091.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs092.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;"><b>Fine-tuning neural network hyperparameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs093.html#hidden-layers" style="font-size: 80%;"><b>Hidden layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs094.html#vanishing-gradients" style="font-size: 80%;"><b>Vanishing gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs095.html#exploding-gradients" style="font-size: 80%;"><b>Exploding gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs096.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs097.html#logistic-function-as-the-root-of-problems" style="font-size: 80%;"><b>Logistic function as the root of problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs098.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#insights-from-the-paper-by-glorot-and-bengio" style="font-size: 80%;"><b>Insights from the paper by Glorot and Bengio</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs100.html#the-relu-function-family" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs101.html#elu-function" style="font-size: 80%;"><b>ELU function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs102.html#which-activation-function-should-we-use" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs103.html#more-on-activation-functions-output-layers" style="font-size: 80%;"><b>More on activation functions, output layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs104.html#batch-normalization" style="font-size: 80%;"><b>Batch Normalization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs105.html#dropout" style="font-size: 80%;"><b>Dropout</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs106.html#gradient-clipping" style="font-size: 80%;"><b>Gradient Clipping</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs107.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs108.html#more-top-down-perspectives" style="font-size: 80%;"><b>More top-down perspectives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs109.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs110.html#limitations-of-nns" style="font-size: 80%;"><b>Limitations of NNs</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs111.html#homogeneous-data" style="font-size: 80%;"><b>Homogeneous data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs112.html#more-limitations" style="font-size: 80%;"><b>More limitations</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0037"></a>
|
|
<!-- !split -->
|
|
<h2 id="derivatives-and-the-chain-rule" class="anchor">Derivatives and the chain rule </h2>
|
|
|
|
<p>From the definition of the activation \( z_j^l \) we have</p>
|
|
$$
|
|
\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
|
|
$$
|
|
|
|
<p>and</p>
|
|
$$
|
|
\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l.
|
|
$$
|
|
|
|
<p>With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))</p>
|
|
$$
|
|
\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=\sigma(z_j^l)(1-\sigma(z_j^l)).
|
|
$$
|
|
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week41-bs036.html">«</a></li>
|
|
<li><a href="._week41-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs029.html">30</a></li>
|
|
<li><a href="._week41-bs030.html">31</a></li>
|
|
<li><a href="._week41-bs031.html">32</a></li>
|
|
<li><a href="._week41-bs032.html">33</a></li>
|
|
<li><a href="._week41-bs033.html">34</a></li>
|
|
<li><a href="._week41-bs034.html">35</a></li>
|
|
<li><a href="._week41-bs035.html">36</a></li>
|
|
<li><a href="._week41-bs036.html">37</a></li>
|
|
<li class="active"><a href="._week41-bs037.html">38</a></li>
|
|
<li><a href="._week41-bs038.html">39</a></li>
|
|
<li><a href="._week41-bs039.html">40</a></li>
|
|
<li><a href="._week41-bs040.html">41</a></li>
|
|
<li><a href="._week41-bs041.html">42</a></li>
|
|
<li><a href="._week41-bs042.html">43</a></li>
|
|
<li><a href="._week41-bs043.html">44</a></li>
|
|
<li><a href="._week41-bs044.html">45</a></li>
|
|
<li><a href="._week41-bs045.html">46</a></li>
|
|
<li><a href="._week41-bs046.html">47</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs112.html">113</a></li>
|
|
<li><a href="._week41-bs038.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|