Morten Hjorth-Jensen
662 lines
45 KiB
HTML
662 lines
45 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
|
|
<title>Week 41 Neural networks and constructing a neural network code</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 41, October 7-11',
|
|
2,
|
|
None,
|
|
'plan-for-week-41-october-7-11'),
|
|
('Material for the lecture on Monday October 7, 2024',
|
|
2,
|
|
None,
|
|
'material-for-the-lecture-on-monday-october-7-2024'),
|
|
('Material for the active learning sessions on Tuesday and '
|
|
'Wednesday',
|
|
2,
|
|
None,
|
|
'material-for-the-active-learning-sessions-on-tuesday-and-wednesday'),
|
|
('Lecture Monday October 7',
|
|
2,
|
|
None,
|
|
'lecture-monday-october-7'),
|
|
('Introduction to Neural networks',
|
|
2,
|
|
None,
|
|
'introduction-to-neural-networks'),
|
|
('Artificial neurons', 2, None, 'artificial-neurons'),
|
|
('Neural network types', 2, None, 'neural-network-types'),
|
|
('Feed-forward neural networks',
|
|
2,
|
|
None,
|
|
'feed-forward-neural-networks'),
|
|
('Convolutional Neural Network',
|
|
2,
|
|
None,
|
|
'convolutional-neural-network'),
|
|
('Recurrent neural networks',
|
|
2,
|
|
None,
|
|
'recurrent-neural-networks'),
|
|
('Other types of networks', 2, None, 'other-types-of-networks'),
|
|
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
|
('Why multilayer perceptrons?',
|
|
2,
|
|
None,
|
|
'why-multilayer-perceptrons'),
|
|
('Illustration of a single perceptron model and a '
|
|
'multi-perceptron model',
|
|
2,
|
|
None,
|
|
'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
|
|
('Examples of XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'examples-of-xor-or-and-and-gates'),
|
|
('Does Logistic Regression do a better Job?',
|
|
2,
|
|
None,
|
|
'does-logistic-regression-do-a-better-job'),
|
|
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
|
('Mathematics of deep learning',
|
|
2,
|
|
None,
|
|
'mathematics-of-deep-learning'),
|
|
('Reminder on books with hands-on material and codes',
|
|
2,
|
|
None,
|
|
'reminder-on-books-with-hands-on-material-and-codes'),
|
|
('Reading recommendations', 2, None, 'reading-recommendations'),
|
|
('Mathematics of deep learning and neural networks',
|
|
2,
|
|
None,
|
|
'mathematics-of-deep-learning-and-neural-networks'),
|
|
('Basics of an NN', 2, None, 'basics-of-an-nn'),
|
|
('Overarching view of a neural network',
|
|
2,
|
|
None,
|
|
'overarching-view-of-a-neural-network'),
|
|
('The optimization problem', 2, None, 'the-optimization-problem'),
|
|
('Parameters of neural networks',
|
|
2,
|
|
None,
|
|
'parameters-of-neural-networks'),
|
|
('Other ingredients of a neural network',
|
|
2,
|
|
None,
|
|
'other-ingredients-of-a-neural-network'),
|
|
('Other parameters', 2, None, 'other-parameters'),
|
|
('Universal approximation theorem',
|
|
2,
|
|
None,
|
|
'universal-approximation-theorem'),
|
|
('Some parallels from real analysis',
|
|
2,
|
|
None,
|
|
'some-parallels-from-real-analysis'),
|
|
('The approximation theorem in words',
|
|
2,
|
|
None,
|
|
'the-approximation-theorem-in-words'),
|
|
('More on the general approximation theorem',
|
|
2,
|
|
None,
|
|
'more-on-the-general-approximation-theorem'),
|
|
('Class of functions we can approximate',
|
|
2,
|
|
None,
|
|
'class-of-functions-we-can-approximate'),
|
|
('Setting up the equations for a neural network',
|
|
2,
|
|
None,
|
|
'setting-up-the-equations-for-a-neural-network'),
|
|
('Layout of a neural network with three hidden layers',
|
|
2,
|
|
None,
|
|
'layout-of-a-neural-network-with-three-hidden-layers'),
|
|
('Definitions', 2, None, 'definitions'),
|
|
('Inputs to the activation function',
|
|
2,
|
|
None,
|
|
'inputs-to-the-activation-function'),
|
|
('Derivatives and the chain rule',
|
|
2,
|
|
None,
|
|
'derivatives-and-the-chain-rule'),
|
|
('Derivative of the cost function',
|
|
2,
|
|
None,
|
|
'derivative-of-the-cost-function'),
|
|
('Simpler examples first, and automatic differentiation',
|
|
2,
|
|
None,
|
|
'simpler-examples-first-and-automatic-differentiation'),
|
|
('Reminder on the chain rule and gradients',
|
|
2,
|
|
None,
|
|
'reminder-on-the-chain-rule-and-gradients'),
|
|
('Multivariable functions', 2, None, 'multivariable-functions'),
|
|
('Automatic differentiation through examples',
|
|
2,
|
|
None,
|
|
'automatic-differentiation-through-examples'),
|
|
('Simple example', 2, None, 'simple-example'),
|
|
('Smarter way of evaluating the above function',
|
|
2,
|
|
None,
|
|
'smarter-way-of-evaluating-the-above-function'),
|
|
('Reducing the number of operations',
|
|
2,
|
|
None,
|
|
'reducing-the-number-of-operations'),
|
|
('Chain rule, forward and reverse modes',
|
|
2,
|
|
None,
|
|
'chain-rule-forward-and-reverse-modes'),
|
|
('Forward and reverse modes',
|
|
2,
|
|
None,
|
|
'forward-and-reverse-modes'),
|
|
('More complicated function',
|
|
2,
|
|
None,
|
|
'more-complicated-function'),
|
|
('Counting the number of floating point operations',
|
|
2,
|
|
None,
|
|
'counting-the-number-of-floating-point-operations'),
|
|
('Defining intermediate operations',
|
|
2,
|
|
None,
|
|
'defining-intermediate-operations'),
|
|
('New expression for the derivative',
|
|
2,
|
|
None,
|
|
'new-expression-for-the-derivative'),
|
|
('Final derivatives', 2, None, 'final-derivatives'),
|
|
('In general not this simple',
|
|
2,
|
|
None,
|
|
'in-general-not-this-simple'),
|
|
('Automatic differentiation',
|
|
2,
|
|
None,
|
|
'automatic-differentiation'),
|
|
('Chain rule', 2, None, 'chain-rule'),
|
|
('First network example, simple percepetron with one input',
|
|
2,
|
|
None,
|
|
'first-network-example-simple-percepetron-with-one-input'),
|
|
('Layout of a simple neural network with no hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-no-hidden-layer'),
|
|
('Optimizing the parameters',
|
|
2,
|
|
None,
|
|
'optimizing-the-parameters'),
|
|
('Adding a hidden layer', 2, None, 'adding-a-hidden-layer'),
|
|
('Layout of a simple neural network with one hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-one-hidden-layer'),
|
|
('The derivatives', 2, None, 'the-derivatives'),
|
|
('Important observations', 2, None, 'important-observations'),
|
|
('The training', 2, None, 'the-training'),
|
|
('Code example', 2, None, 'code-example'),
|
|
('Exercise 1: Including more data',
|
|
2,
|
|
None,
|
|
'exercise-1-including-more-data'),
|
|
('Simple neural network and the back propagation equations',
|
|
2,
|
|
None,
|
|
'simple-neural-network-and-the-back-propagation-equations'),
|
|
('Layout of a simple neural network with two input nodes, one '
|
|
'hidden layer and one output node',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-and-one-output-node'),
|
|
('The ouput layer', 2, None, 'the-ouput-layer'),
|
|
('Compact expressions', 2, None, 'compact-expressions'),
|
|
('Output layer', 2, None, 'output-layer'),
|
|
('Explicit derivatives', 2, None, 'explicit-derivatives'),
|
|
('Derivatives of the hidden layer',
|
|
2,
|
|
None,
|
|
'derivatives-of-the-hidden-layer'),
|
|
('Final expression', 2, None, 'final-expression'),
|
|
('Completing the list', 2, None, 'completing-the-list'),
|
|
('Final expressions for the biases of the hidden layer',
|
|
2,
|
|
None,
|
|
'final-expressions-for-the-biases-of-the-hidden-layer'),
|
|
('Gradient expressions', 2, None, 'gradient-expressions'),
|
|
('Exercise 2: Extended program',
|
|
2,
|
|
None,
|
|
'exercise-2-extended-program'),
|
|
('Getting serious, the back propagation equations for a neural '
|
|
'network',
|
|
2,
|
|
None,
|
|
'getting-serious-the-back-propagation-equations-for-a-neural-network'),
|
|
('Analyzing the last results',
|
|
2,
|
|
None,
|
|
'analyzing-the-last-results'),
|
|
('More considerations', 2, None, 'more-considerations'),
|
|
('Derivatives in terms of $z_j^L$',
|
|
2,
|
|
None,
|
|
'derivatives-in-terms-of-z-j-l'),
|
|
('Bringing it together', 2, None, 'bringing-it-together'),
|
|
('Final back propagating equation',
|
|
2,
|
|
None,
|
|
'final-back-propagating-equation'),
|
|
('Using the chain rule and summing over all $k$ entries',
|
|
2,
|
|
None,
|
|
'using-the-chain-rule-and-summing-over-all-k-entries'),
|
|
('Setting up the back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up the back propagation algorithm, part 2',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-2'),
|
|
('Setting up the Back propagation algorithm, part 3',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-3'),
|
|
('Updating the gradients', 2, None, 'updating-the-gradients'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 3, None, 'relevance'),
|
|
('Fine-tuning neural network hyperparameters',
|
|
2,
|
|
None,
|
|
'fine-tuning-neural-network-hyperparameters'),
|
|
('Hidden layers', 2, None, 'hidden-layers'),
|
|
('Vanishing gradients', 2, None, 'vanishing-gradients'),
|
|
('Exploding gradients', 2, None, 'exploding-gradients'),
|
|
('Is the Logistic activation function (Sigmoid) our choice?',
|
|
2,
|
|
None,
|
|
'is-the-logistic-activation-function-sigmoid-our-choice'),
|
|
('Logistic function as the root of problems',
|
|
2,
|
|
None,
|
|
'logistic-function-as-the-root-of-problems'),
|
|
('The derivative of the Logistic funtion',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-logistic-funtion'),
|
|
('Insights from the paper by Glorot and Bengio',
|
|
2,
|
|
None,
|
|
'insights-from-the-paper-by-glorot-and-bengio'),
|
|
('The RELU function family', 2, None, 'the-relu-function-family'),
|
|
('ELU function', 2, None, 'elu-function'),
|
|
('Which activation function should we use?',
|
|
2,
|
|
None,
|
|
'which-activation-function-should-we-use'),
|
|
('More on activation functions, output layers',
|
|
2,
|
|
None,
|
|
'more-on-activation-functions-output-layers'),
|
|
('Batch Normalization', 2, None, 'batch-normalization'),
|
|
('Dropout', 2, None, 'dropout'),
|
|
('Gradient Clipping', 2, None, 'gradient-clipping'),
|
|
('A top-down perspective on Neural networks',
|
|
2,
|
|
None,
|
|
'a-top-down-perspective-on-neural-networks'),
|
|
('More top-down perspectives',
|
|
2,
|
|
None,
|
|
'more-top-down-perspectives'),
|
|
('Limitations of supervised learning with deep networks',
|
|
2,
|
|
None,
|
|
'limitations-of-supervised-learning-with-deep-networks'),
|
|
('Limitations of NNs', 2, None, 'limitations-of-nns'),
|
|
('Homogeneous data', 2, None, 'homogeneous-data'),
|
|
('More limitations', 2, None, 'more-limitations')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41-october-7-11" style="font-size: 80%;"><b>Plan for week 41, October 7-11</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs002.html#material-for-the-lecture-on-monday-october-7-2024" style="font-size: 80%;"><b>Material for the lecture on Monday October 7, 2024</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs003.html#material-for-the-active-learning-sessions-on-tuesday-and-wednesday" style="font-size: 80%;"><b>Material for the active learning sessions on Tuesday and Wednesday</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs004.html#lecture-monday-october-7" style="font-size: 80%;"><b>Lecture Monday October 7</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs005.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs006.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs007.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs008.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs009.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs010.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs011.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs012.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs013.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs014.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs015.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs016.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs017.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs018.html#mathematics-of-deep-learning" style="font-size: 80%;"><b>Mathematics of deep learning</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs019.html#reminder-on-books-with-hands-on-material-and-codes" style="font-size: 80%;"><b>Reminder on books with hands-on material and codes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#reading-recommendations" style="font-size: 80%;"><b>Reading recommendations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs021.html#mathematics-of-deep-learning-and-neural-networks" style="font-size: 80%;"><b>Mathematics of deep learning and neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs022.html#basics-of-an-nn" style="font-size: 80%;"><b>Basics of an NN</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs023.html#overarching-view-of-a-neural-network" style="font-size: 80%;"><b>Overarching view of a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs024.html#the-optimization-problem" style="font-size: 80%;"><b>The optimization problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs025.html#parameters-of-neural-networks" style="font-size: 80%;"><b>Parameters of neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs026.html#other-ingredients-of-a-neural-network" style="font-size: 80%;"><b>Other ingredients of a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs027.html#other-parameters" style="font-size: 80%;"><b>Other parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs028.html#universal-approximation-theorem" style="font-size: 80%;"><b>Universal approximation theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs029.html#some-parallels-from-real-analysis" style="font-size: 80%;"><b>Some parallels from real analysis</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs030.html#the-approximation-theorem-in-words" style="font-size: 80%;"><b>The approximation theorem in words</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs031.html#more-on-the-general-approximation-theorem" style="font-size: 80%;"><b>More on the general approximation theorem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs032.html#class-of-functions-we-can-approximate" style="font-size: 80%;"><b>Class of functions we can approximate</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs033.html#setting-up-the-equations-for-a-neural-network" style="font-size: 80%;"><b>Setting up the equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs034.html#layout-of-a-neural-network-with-three-hidden-layers" style="font-size: 80%;"><b>Layout of a neural network with three hidden layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs035.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs036.html#inputs-to-the-activation-function" style="font-size: 80%;"><b>Inputs to the activation function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs037.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs039.html#simpler-examples-first-and-automatic-differentiation" style="font-size: 80%;"><b>Simpler examples first, and automatic differentiation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs040.html#reminder-on-the-chain-rule-and-gradients" style="font-size: 80%;"><b>Reminder on the chain rule and gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs041.html#multivariable-functions" style="font-size: 80%;"><b>Multivariable functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs042.html#automatic-differentiation-through-examples" style="font-size: 80%;"><b>Automatic differentiation through examples</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs043.html#simple-example" style="font-size: 80%;"><b>Simple example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs044.html#smarter-way-of-evaluating-the-above-function" style="font-size: 80%;"><b>Smarter way of evaluating the above function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs045.html#reducing-the-number-of-operations" style="font-size: 80%;"><b>Reducing the number of operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs046.html#chain-rule-forward-and-reverse-modes" style="font-size: 80%;"><b>Chain rule, forward and reverse modes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs047.html#forward-and-reverse-modes" style="font-size: 80%;"><b>Forward and reverse modes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs048.html#more-complicated-function" style="font-size: 80%;"><b>More complicated function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs049.html#counting-the-number-of-floating-point-operations" style="font-size: 80%;"><b>Counting the number of floating point operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs050.html#defining-intermediate-operations" style="font-size: 80%;"><b>Defining intermediate operations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs051.html#new-expression-for-the-derivative" style="font-size: 80%;"><b>New expression for the derivative</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs052.html#final-derivatives" style="font-size: 80%;"><b>Final derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs053.html#in-general-not-this-simple" style="font-size: 80%;"><b>In general not this simple</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs054.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs055.html#chain-rule" style="font-size: 80%;"><b>Chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs056.html#first-network-example-simple-percepetron-with-one-input" style="font-size: 80%;"><b>First network example, simple percepetron with one input</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs057.html#layout-of-a-simple-neural-network-with-no-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with no hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs058.html#optimizing-the-parameters" style="font-size: 80%;"><b>Optimizing the parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs059.html#adding-a-hidden-layer" style="font-size: 80%;"><b>Adding a hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs060.html#layout-of-a-simple-neural-network-with-one-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with one hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs061.html#the-derivatives" style="font-size: 80%;"><b>The derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs062.html#important-observations" style="font-size: 80%;"><b>Important observations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs063.html#the-training" style="font-size: 80%;"><b>The training</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#code-example" style="font-size: 80%;"><b>Code example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs065.html#exercise-1-including-more-data" style="font-size: 80%;"><b>Exercise 1: Including more data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs066.html#simple-neural-network-and-the-back-propagation-equations" style="font-size: 80%;"><b>Simple neural network and the back propagation equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs067.html#layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-and-one-output-node" style="font-size: 80%;"><b>Layout of a simple neural network with two input nodes, one hidden layer and one output node</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs068.html#the-ouput-layer" style="font-size: 80%;"><b>The ouput layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs069.html#compact-expressions" style="font-size: 80%;"><b>Compact expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs070.html#output-layer" style="font-size: 80%;"><b>Output layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs071.html#explicit-derivatives" style="font-size: 80%;"><b>Explicit derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs072.html#derivatives-of-the-hidden-layer" style="font-size: 80%;"><b>Derivatives of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs073.html#final-expression" style="font-size: 80%;"><b>Final expression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs074.html#completing-the-list" style="font-size: 80%;"><b>Completing the list</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs075.html#final-expressions-for-the-biases-of-the-hidden-layer" style="font-size: 80%;"><b>Final expressions for the biases of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs076.html#gradient-expressions" style="font-size: 80%;"><b>Gradient expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs077.html#exercise-2-extended-program" style="font-size: 80%;"><b>Exercise 2: Extended program</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs078.html#getting-serious-the-back-propagation-equations-for-a-neural-network" style="font-size: 80%;"><b>Getting serious, the back propagation equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs079.html#analyzing-the-last-results" style="font-size: 80%;"><b>Analyzing the last results</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs080.html#more-considerations" style="font-size: 80%;"><b>More considerations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs081.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs082.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs083.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs084.html#using-the-chain-rule-and-summing-over-all-k-entries" style="font-size: 80%;"><b>Using the chain rule and summing over all \( k \) entries</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs085.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the back propagation algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-back-propagation-algorithm-part-2" style="font-size: 80%;"><b>Setting up the back propagation algorithm, part 2</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs087.html#setting-up-the-back-propagation-algorithm-part-3" style="font-size: 80%;"><b>Setting up the Back propagation algorithm, part 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs088.html#updating-the-gradients" style="font-size: 80%;"><b>Updating the gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs089.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs090.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs091.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs092.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;"><b>Fine-tuning neural network hyperparameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs093.html#hidden-layers" style="font-size: 80%;"><b>Hidden layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs094.html#vanishing-gradients" style="font-size: 80%;"><b>Vanishing gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs095.html#exploding-gradients" style="font-size: 80%;"><b>Exploding gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs096.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs097.html#logistic-function-as-the-root-of-problems" style="font-size: 80%;"><b>Logistic function as the root of problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs098.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#insights-from-the-paper-by-glorot-and-bengio" style="font-size: 80%;"><b>Insights from the paper by Glorot and Bengio</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs100.html#the-relu-function-family" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs101.html#elu-function" style="font-size: 80%;"><b>ELU function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs102.html#which-activation-function-should-we-use" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs103.html#more-on-activation-functions-output-layers" style="font-size: 80%;"><b>More on activation functions, output layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs104.html#batch-normalization" style="font-size: 80%;"><b>Batch Normalization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs105.html#dropout" style="font-size: 80%;"><b>Dropout</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs106.html#gradient-clipping" style="font-size: 80%;"><b>Gradient Clipping</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs107.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs108.html#more-top-down-perspectives" style="font-size: 80%;"><b>More top-down perspectives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs109.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs110.html#limitations-of-nns" style="font-size: 80%;"><b>Limitations of NNs</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs111.html#homogeneous-data" style="font-size: 80%;"><b>Homogeneous data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs112.html#more-limitations" style="font-size: 80%;"><b>More limitations</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0064"></a>
|
|
<!-- !split -->
|
|
<h2 id="code-example" class="anchor">Code example </h2>
|
|
|
|
<p>The code here implements the above model with one hidden layer and
|
|
scalar variables for the same function we studied in the previous
|
|
example. The code is however set up so that we can add multiple
|
|
inputs \( x \) and target values \( y \). Note also that we have the
|
|
possibility of defining a feature matrix \( \boldsymbol{X} \) with more than just
|
|
one column for the input values. This will turn useful in our next example. We have also defined matrices and vectors for all of our operations although it is not necessary here.
|
|
</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #408080; font-style: italic"># We use the Sigmoid function as activation function</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(z):
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1.0/</span>(<span style="color: #666666">1.0+</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z))
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">forwardpropagation</span>(x):
|
|
<span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
|
|
z_1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(x, w_1) <span style="color: #666666">+</span> b_1
|
|
<span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
|
|
a_1 <span style="color: #666666">=</span> sigmoid(z_1)
|
|
<span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
|
|
z_2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_1, w_2) <span style="color: #666666">+</span> b_2
|
|
a_2 <span style="color: #666666">=</span> z_2
|
|
<span style="color: #008000; font-weight: bold">return</span> a_1, a_2
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backpropagation</span>(x, y):
|
|
a_1, a_2 <span style="color: #666666">=</span> forwardpropagation(x)
|
|
<span style="color: #408080; font-style: italic"># parameter delta for the output layer, note that a_2=z_2 and its derivative wrt z_2 is just 1</span>
|
|
delta_2 <span style="color: #666666">=</span> a_2 <span style="color: #666666">-</span> y
|
|
<span style="color: #008000">print</span>(<span style="color: #666666">0.5*</span>((a_2<span style="color: #666666">-</span>y)<span style="color: #666666">**2</span>))
|
|
<span style="color: #408080; font-style: italic"># delta for the hidden layer</span>
|
|
delta_1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(delta_2, w_2<span style="color: #666666">.</span>T) <span style="color: #666666">*</span> a_1 <span style="color: #666666">*</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> a_1)
|
|
<span style="color: #408080; font-style: italic"># gradients for the output layer</span>
|
|
output_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_1<span style="color: #666666">.</span>T, delta_2)
|
|
output_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(delta_2, axis<span style="color: #666666">=0</span>)
|
|
<span style="color: #408080; font-style: italic"># gradient for the hidden layer</span>
|
|
hidden_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(x<span style="color: #666666">.</span>T, delta_1)
|
|
hidden_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(delta_1, axis<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000; font-weight: bold">return</span> output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
|
|
|
|
|
|
<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
|
|
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
|
<span style="color: #408080; font-style: italic"># Input variable</span>
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([<span style="color: #666666">4.0</span>],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
|
|
<span style="color: #408080; font-style: italic"># Target values</span>
|
|
y <span style="color: #666666">=</span> <span style="color: #666666">2*</span>x<span style="color: #666666">+1.0</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Defining the neural network, only scalars here</span>
|
|
n_inputs <span style="color: #666666">=</span> x<span style="color: #666666">.</span>shape
|
|
n_features <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
n_outputs <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Initialize the network</span>
|
|
<span style="color: #408080; font-style: italic"># weights and bias in the hidden layer</span>
|
|
w_1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_features, n_hidden_neurons)
|
|
b_1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># weights and bias in the output layer</span>
|
|
w_2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_hidden_neurons, n_outputs)
|
|
b_2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_outputs) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
|
|
|
|
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
|
|
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">50</span>):
|
|
<span style="color: #408080; font-style: italic"># calculate gradients</span>
|
|
derivW2, derivB2, derivW1, derivB1 <span style="color: #666666">=</span> backpropagation(x, y)
|
|
<span style="color: #408080; font-style: italic"># update weights and biases</span>
|
|
w_2 <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> derivW2
|
|
b_2 <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> derivB2
|
|
w_1 <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> derivW1
|
|
b_1 <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> derivB1
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<p>We see that after some few iterations (the results do depend on the learning rate however), we get an error which is rather small.</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week41-bs063.html">«</a></li>
|
|
<li><a href="._week41-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs056.html">57</a></li>
|
|
<li><a href="._week41-bs057.html">58</a></li>
|
|
<li><a href="._week41-bs058.html">59</a></li>
|
|
<li><a href="._week41-bs059.html">60</a></li>
|
|
<li><a href="._week41-bs060.html">61</a></li>
|
|
<li><a href="._week41-bs061.html">62</a></li>
|
|
<li><a href="._week41-bs062.html">63</a></li>
|
|
<li><a href="._week41-bs063.html">64</a></li>
|
|
<li class="active"><a href="._week41-bs064.html">65</a></li>
|
|
<li><a href="._week41-bs065.html">66</a></li>
|
|
<li><a href="._week41-bs066.html">67</a></li>
|
|
<li><a href="._week41-bs067.html">68</a></li>
|
|
<li><a href="._week41-bs068.html">69</a></li>
|
|
<li><a href="._week41-bs069.html">70</a></li>
|
|
<li><a href="._week41-bs070.html">71</a></li>
|
|
<li><a href="._week41-bs071.html">72</a></li>
|
|
<li><a href="._week41-bs072.html">73</a></li>
|
|
<li><a href="._week41-bs073.html">74</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs112.html">113</a></li>
|
|
<li><a href="._week41-bs065.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|