Morten Hjorth-Jensen
530 lines
35 KiB
HTML
530 lines
35 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week42-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 42 Constructing a Neural Network code with examples">
|
|
<title>Week 42 Constructing a Neural Network code with examples</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week42-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Lecture October 14, 2024', 2, None, 'lecture-october-14-2024'),
|
|
('Material for the active learning sessions on Tuesday and '
|
|
'Wednesday',
|
|
2,
|
|
None,
|
|
'material-for-the-active-learning-sessions-on-tuesday-and-wednesday'),
|
|
('Writing a code which implements a feed-forward neural network',
|
|
2,
|
|
None,
|
|
'writing-a-code-which-implements-a-feed-forward-neural-network'),
|
|
('Mathematics of deep learning',
|
|
2,
|
|
None,
|
|
'mathematics-of-deep-learning'),
|
|
('Reminder on books with hands-on material and codes',
|
|
2,
|
|
None,
|
|
'reminder-on-books-with-hands-on-material-and-codes'),
|
|
('Reading recommendations', 2, None, 'reading-recommendations'),
|
|
('First network example, simple percepetron with one input',
|
|
2,
|
|
None,
|
|
'first-network-example-simple-percepetron-with-one-input'),
|
|
('Layout of a simple neural network with no hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-no-hidden-layer'),
|
|
('Optimizing the parameters',
|
|
2,
|
|
None,
|
|
'optimizing-the-parameters'),
|
|
('Adding a hidden layer', 2, None, 'adding-a-hidden-layer'),
|
|
('Layout of a simple neural network with one hidden layer',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-one-hidden-layer'),
|
|
('The derivatives', 2, None, 'the-derivatives'),
|
|
('Important observations', 2, None, 'important-observations'),
|
|
('The training', 2, None, 'the-training'),
|
|
('Code example', 2, None, 'code-example'),
|
|
('Simple neural network and the back propagation equations',
|
|
2,
|
|
None,
|
|
'simple-neural-network-and-the-back-propagation-equations'),
|
|
('Layout of a simple neural network with two input nodes, one '
|
|
'hidden layer with two hidden noeds and one output node',
|
|
2,
|
|
None,
|
|
'layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-with-two-hidden-noeds-and-one-output-node'),
|
|
('The ouput layer', 2, None, 'the-ouput-layer'),
|
|
('Compact expressions', 2, None, 'compact-expressions'),
|
|
('Output layer', 2, None, 'output-layer'),
|
|
('Explicit derivatives', 2, None, 'explicit-derivatives'),
|
|
('Derivatives of the hidden layer',
|
|
2,
|
|
None,
|
|
'derivatives-of-the-hidden-layer'),
|
|
('Final expression', 2, None, 'final-expression'),
|
|
('Completing the list', 2, None, 'completing-the-list'),
|
|
('Final expressions for the biases of the hidden layer',
|
|
2,
|
|
None,
|
|
'final-expressions-for-the-biases-of-the-hidden-layer'),
|
|
('Gradient expressions', 2, None, 'gradient-expressions'),
|
|
('Setting up the equations for a neural network',
|
|
2,
|
|
None,
|
|
'setting-up-the-equations-for-a-neural-network'),
|
|
('Layout of a neural network with three hidden layers (last '
|
|
'later = $l=L=4$, first layer $l=0$)',
|
|
2,
|
|
None,
|
|
'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'),
|
|
('Definitions', 2, None, 'definitions'),
|
|
('Inputs to the activation function',
|
|
2,
|
|
None,
|
|
'inputs-to-the-activation-function'),
|
|
('Layout of input to first hidden layer $l=1$ from input layer '
|
|
'$l=0$',
|
|
2,
|
|
None,
|
|
'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'),
|
|
('Derivatives and the chain rule',
|
|
2,
|
|
None,
|
|
'derivatives-and-the-chain-rule'),
|
|
('Derivative of the cost function',
|
|
2,
|
|
None,
|
|
'derivative-of-the-cost-function'),
|
|
('The back propagation equations for a neural network',
|
|
2,
|
|
None,
|
|
'the-back-propagation-equations-for-a-neural-network'),
|
|
('Analyzing the last results',
|
|
2,
|
|
None,
|
|
'analyzing-the-last-results'),
|
|
('More considerations', 2, None, 'more-considerations'),
|
|
('Derivatives in terms of $z_j^L$',
|
|
2,
|
|
None,
|
|
'derivatives-in-terms-of-z-j-l'),
|
|
('Bringing it together', 2, None, 'bringing-it-together'),
|
|
('Final back propagating equation',
|
|
2,
|
|
None,
|
|
'final-back-propagating-equation'),
|
|
('Using the chain rule and summing over all $k$ entries',
|
|
2,
|
|
None,
|
|
'using-the-chain-rule-and-summing-over-all-k-entries'),
|
|
('Setting up the back propagation algorithm and algorithm for a '
|
|
'feed forward NN, initalizations',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-and-algorithm-for-a-feed-forward-nn-initalizations'),
|
|
('Setting up the back propagation algorithm, part 1',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-1'),
|
|
('Setting up the back propagation algorithm, part 2',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-2'),
|
|
('Setting up the Back propagation algorithm, part 3',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm-part-3'),
|
|
('Updating the gradients', 2, None, 'updating-the-gradients'),
|
|
('Activation functions', 2, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 2, None, 'relevance'),
|
|
('Vanishing gradients', 2, None, 'vanishing-gradients'),
|
|
('Exploding gradients', 2, None, 'exploding-gradients'),
|
|
('Is the Logistic activation function (Sigmoid) our choice?',
|
|
2,
|
|
None,
|
|
'is-the-logistic-activation-function-sigmoid-our-choice'),
|
|
('Logistic function as the root of problems',
|
|
2,
|
|
None,
|
|
'logistic-function-as-the-root-of-problems'),
|
|
('The derivative of the Logistic funtion',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-logistic-funtion'),
|
|
('Insights from the paper by Glorot and Bengio',
|
|
2,
|
|
None,
|
|
'insights-from-the-paper-by-glorot-and-bengio'),
|
|
('The RELU function family', 2, None, 'the-relu-function-family'),
|
|
('ELU function', 2, None, 'elu-function'),
|
|
('Which activation function should we use?',
|
|
2,
|
|
None,
|
|
'which-activation-function-should-we-use'),
|
|
('More on activation functions, output layers',
|
|
2,
|
|
None,
|
|
'more-on-activation-functions-output-layers'),
|
|
('Fine-tuning neural network hyperparameters',
|
|
2,
|
|
None,
|
|
'fine-tuning-neural-network-hyperparameters'),
|
|
('Hidden layers', 2, None, 'hidden-layers'),
|
|
('Batch Normalization', 2, None, 'batch-normalization'),
|
|
('Dropout', 2, None, 'dropout'),
|
|
('Gradient Clipping', 2, None, 'gradient-clipping'),
|
|
('A top-down perspective on Neural networks',
|
|
2,
|
|
None,
|
|
'a-top-down-perspective-on-neural-networks'),
|
|
('More top-down perspectives',
|
|
2,
|
|
None,
|
|
'more-top-down-perspectives'),
|
|
('Limitations of supervised learning with deep networks',
|
|
2,
|
|
None,
|
|
'limitations-of-supervised-learning-with-deep-networks'),
|
|
('Limitations of NNs', 2, None, 'limitations-of-nns'),
|
|
('Homogeneous data', 2, None, 'homogeneous-data'),
|
|
('More limitations', 2, None, 'more-limitations'),
|
|
('Setting up a Multi-layer perceptron model for classification',
|
|
2,
|
|
None,
|
|
'setting-up-a-multi-layer-perceptron-model-for-classification'),
|
|
('Defining the cost function',
|
|
2,
|
|
None,
|
|
'defining-the-cost-function'),
|
|
('Example: binary classification problem',
|
|
2,
|
|
None,
|
|
'example-binary-classification-problem'),
|
|
('The Softmax function', 2, None, 'the-softmax-function'),
|
|
('Developing a code for doing neural networks with back '
|
|
'propagation',
|
|
2,
|
|
None,
|
|
'developing-a-code-for-doing-neural-networks-with-back-propagation'),
|
|
('Collect and pre-process data',
|
|
2,
|
|
None,
|
|
'collect-and-pre-process-data'),
|
|
('Train and test datasets', 2, None, 'train-and-test-datasets'),
|
|
('Define model and architecture',
|
|
2,
|
|
None,
|
|
'define-model-and-architecture'),
|
|
('Layers', 2, None, 'layers'),
|
|
('Weights and biases', 2, None, 'weights-and-biases'),
|
|
('Feed-forward pass', 2, None, 'feed-forward-pass'),
|
|
('Matrix multiplications', 2, None, 'matrix-multiplications'),
|
|
('Choose cost function and optimizer',
|
|
2,
|
|
None,
|
|
'choose-cost-function-and-optimizer'),
|
|
('Optimizing the cost function',
|
|
2,
|
|
None,
|
|
'optimizing-the-cost-function'),
|
|
('Regularization', 2, None, 'regularization'),
|
|
('Matrix multiplication', 2, None, 'matrix-multiplication'),
|
|
('Improving performance', 2, None, 'improving-performance'),
|
|
('Full object-oriented implementation',
|
|
2,
|
|
None,
|
|
'full-object-oriented-implementation'),
|
|
('Evaluate model performance on test data',
|
|
2,
|
|
None,
|
|
'evaluate-model-performance-on-test-data'),
|
|
('Adjust hyperparameters', 2, None, 'adjust-hyperparameters'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('scikit-learn implementation',
|
|
2,
|
|
None,
|
|
'scikit-learn-implementation'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('Building neural networks in Tensorflow and Keras',
|
|
2,
|
|
None,
|
|
'building-neural-networks-in-tensorflow-and-keras'),
|
|
('Tensorflow', 2, None, 'tensorflow'),
|
|
('Using Keras', 2, None, 'using-keras'),
|
|
('Collect and pre-process data',
|
|
2,
|
|
None,
|
|
'collect-and-pre-process-data'),
|
|
('The Breast Cancer Data, now with Keras',
|
|
2,
|
|
None,
|
|
'the-breast-cancer-data-now-with-keras'),
|
|
('Building a neural network code',
|
|
2,
|
|
None,
|
|
'building-a-neural-network-code'),
|
|
('Learning rate methods', 3, None, 'learning-rate-methods'),
|
|
('Usage of the above learning rate schedulers',
|
|
3,
|
|
None,
|
|
'usage-of-the-above-learning-rate-schedulers'),
|
|
('Cost functions', 3, None, 'cost-functions'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('The Neural Network', 3, None, 'the-neural-network'),
|
|
('Multiclass classification',
|
|
3,
|
|
None,
|
|
'multiclass-classification'),
|
|
('Testing the XOR gate and other gates',
|
|
2,
|
|
None,
|
|
'testing-the-xor-gate-and-other-gates')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week42-bs.html">Week 42 Constructing a Neural Network code with examples</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week42-bs001.html#lecture-october-14-2024" style="font-size: 80%;"><b>Lecture October 14, 2024</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs002.html#material-for-the-active-learning-sessions-on-tuesday-and-wednesday" style="font-size: 80%;"><b>Material for the active learning sessions on Tuesday and Wednesday</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs003.html#writing-a-code-which-implements-a-feed-forward-neural-network" style="font-size: 80%;"><b>Writing a code which implements a feed-forward neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs004.html#mathematics-of-deep-learning" style="font-size: 80%;"><b>Mathematics of deep learning</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs005.html#reminder-on-books-with-hands-on-material-and-codes" style="font-size: 80%;"><b>Reminder on books with hands-on material and codes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs006.html#reading-recommendations" style="font-size: 80%;"><b>Reading recommendations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs007.html#first-network-example-simple-percepetron-with-one-input" style="font-size: 80%;"><b>First network example, simple percepetron with one input</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs008.html#layout-of-a-simple-neural-network-with-no-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with no hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs009.html#optimizing-the-parameters" style="font-size: 80%;"><b>Optimizing the parameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs010.html#adding-a-hidden-layer" style="font-size: 80%;"><b>Adding a hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs011.html#layout-of-a-simple-neural-network-with-one-hidden-layer" style="font-size: 80%;"><b>Layout of a simple neural network with one hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs012.html#the-derivatives" style="font-size: 80%;"><b>The derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs013.html#important-observations" style="font-size: 80%;"><b>Important observations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs014.html#the-training" style="font-size: 80%;"><b>The training</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs015.html#code-example" style="font-size: 80%;"><b>Code example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs016.html#simple-neural-network-and-the-back-propagation-equations" style="font-size: 80%;"><b>Simple neural network and the back propagation equations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs017.html#layout-of-a-simple-neural-network-with-two-input-nodes-one-hidden-layer-with-two-hidden-noeds-and-one-output-node" style="font-size: 80%;"><b>Layout of a simple neural network with two input nodes, one hidden layer with two hidden noeds and one output node</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs018.html#the-ouput-layer" style="font-size: 80%;"><b>The ouput layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs019.html#compact-expressions" style="font-size: 80%;"><b>Compact expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs020.html#output-layer" style="font-size: 80%;"><b>Output layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs021.html#explicit-derivatives" style="font-size: 80%;"><b>Explicit derivatives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs022.html#derivatives-of-the-hidden-layer" style="font-size: 80%;"><b>Derivatives of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs023.html#final-expression" style="font-size: 80%;"><b>Final expression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs024.html#completing-the-list" style="font-size: 80%;"><b>Completing the list</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs025.html#final-expressions-for-the-biases-of-the-hidden-layer" style="font-size: 80%;"><b>Final expressions for the biases of the hidden layer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs026.html#gradient-expressions" style="font-size: 80%;"><b>Gradient expressions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs027.html#setting-up-the-equations-for-a-neural-network" style="font-size: 80%;"><b>Setting up the equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs028.html#layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0" style="font-size: 80%;"><b>Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs030.html#inputs-to-the-activation-function" style="font-size: 80%;"><b>Inputs to the activation function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs031.html#layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0" style="font-size: 80%;"><b>Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs032.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs033.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs034.html#the-back-propagation-equations-for-a-neural-network" style="font-size: 80%;"><b>The back propagation equations for a neural network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs035.html#analyzing-the-last-results" style="font-size: 80%;"><b>Analyzing the last results</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs036.html#more-considerations" style="font-size: 80%;"><b>More considerations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs037.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs038.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs039.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs040.html#using-the-chain-rule-and-summing-over-all-k-entries" style="font-size: 80%;"><b>Using the chain rule and summing over all \( k \) entries</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs041.html#setting-up-the-back-propagation-algorithm-and-algorithm-for-a-feed-forward-nn-initalizations" style="font-size: 80%;"><b>Setting up the back propagation algorithm and algorithm for a feed forward NN, initalizations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs042.html#setting-up-the-back-propagation-algorithm-part-1" style="font-size: 80%;"><b>Setting up the back propagation algorithm, part 1</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs043.html#setting-up-the-back-propagation-algorithm-part-2" style="font-size: 80%;"><b>Setting up the back propagation algorithm, part 2</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs044.html#setting-up-the-back-propagation-algorithm-part-3" style="font-size: 80%;"><b>Setting up the Back propagation algorithm, part 3</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs045.html#updating-the-gradients" style="font-size: 80%;"><b>Updating the gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#activation-functions" style="font-size: 80%;"><b>Activation functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs047.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs048.html#relevance" style="font-size: 80%;"><b>Relevance</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs049.html#vanishing-gradients" style="font-size: 80%;"><b>Vanishing gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs050.html#exploding-gradients" style="font-size: 80%;"><b>Exploding gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs051.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs052.html#logistic-function-as-the-root-of-problems" style="font-size: 80%;"><b>Logistic function as the root of problems</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs053.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs054.html#insights-from-the-paper-by-glorot-and-bengio" style="font-size: 80%;"><b>Insights from the paper by Glorot and Bengio</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs055.html#the-relu-function-family" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs056.html#elu-function" style="font-size: 80%;"><b>ELU function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs057.html#which-activation-function-should-we-use" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs058.html#more-on-activation-functions-output-layers" style="font-size: 80%;"><b>More on activation functions, output layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs059.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;"><b>Fine-tuning neural network hyperparameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs060.html#hidden-layers" style="font-size: 80%;"><b>Hidden layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs061.html#batch-normalization" style="font-size: 80%;"><b>Batch Normalization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs062.html#dropout" style="font-size: 80%;"><b>Dropout</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs063.html#gradient-clipping" style="font-size: 80%;"><b>Gradient Clipping</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#a-top-down-perspective-on-neural-networks" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs065.html#more-top-down-perspectives" style="font-size: 80%;"><b>More top-down perspectives</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs066.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs067.html#limitations-of-nns" style="font-size: 80%;"><b>Limitations of NNs</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs068.html#homogeneous-data" style="font-size: 80%;"><b>Homogeneous data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs069.html#more-limitations" style="font-size: 80%;"><b>More limitations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs070.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs071.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs072.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs073.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs074.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs096.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs076.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs077.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs078.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs079.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs080.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs081.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs082.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs083.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs084.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs085.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs086.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs087.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs088.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs089.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs092.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs091.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs092.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs093.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs094.html#tensorflow" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs095.html#using-keras" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs096.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs097.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;"><b>The Breast Cancer Data, now with Keras</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#building-a-neural-network-code" style="font-size: 80%;"><b>Building a neural network code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#learning-rate-methods" style="font-size: 80%;"> Learning rate methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#usage-of-the-above-learning-rate-schedulers" style="font-size: 80%;"> Usage of the above learning rate schedulers</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#cost-functions" style="font-size: 80%;"> Cost functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#the-neural-network" style="font-size: 80%;"> The Neural Network</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs098.html#multiclass-classification" style="font-size: 80%;"> Multiclass classification</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week42-bs099.html#testing-the-xor-gate-and-other-gates" style="font-size: 80%;"><b>Testing the XOR gate and other gates</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0064"></a>
|
|
<!-- !split -->
|
|
<h2 id="a-top-down-perspective-on-neural-networks" class="anchor">A top-down perspective on Neural networks </h2>
|
|
|
|
<p>The first thing we would like to do is divide the data into two or
|
|
three parts. A training set, a validation or dev (development) set,
|
|
and a test set. The test set is the data on which we want to make
|
|
predictions. The dev set is a subset of the training data we use to
|
|
check how well we are doing out-of-sample, after training the model on
|
|
the training dataset. We use the validation error as a proxy for the
|
|
test error in order to make tweaks to our model. It is crucial that we
|
|
do not use any of the test data to train the algorithm. This is a
|
|
cardinal sin in ML. Then:
|
|
</p>
|
|
|
|
<ol>
|
|
<li> Estimate optimal error rate</li>
|
|
<li> Minimize underfitting (bias) on training data set.</li>
|
|
<li> Make sure you are not overfitting.</li>
|
|
</ol>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week42-bs063.html">«</a></li>
|
|
<li><a href="._week42-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week42-bs056.html">57</a></li>
|
|
<li><a href="._week42-bs057.html">58</a></li>
|
|
<li><a href="._week42-bs058.html">59</a></li>
|
|
<li><a href="._week42-bs059.html">60</a></li>
|
|
<li><a href="._week42-bs060.html">61</a></li>
|
|
<li><a href="._week42-bs061.html">62</a></li>
|
|
<li><a href="._week42-bs062.html">63</a></li>
|
|
<li><a href="._week42-bs063.html">64</a></li>
|
|
<li class="active"><a href="._week42-bs064.html">65</a></li>
|
|
<li><a href="._week42-bs065.html">66</a></li>
|
|
<li><a href="._week42-bs066.html">67</a></li>
|
|
<li><a href="._week42-bs067.html">68</a></li>
|
|
<li><a href="._week42-bs068.html">69</a></li>
|
|
<li><a href="._week42-bs069.html">70</a></li>
|
|
<li><a href="._week42-bs070.html">71</a></li>
|
|
<li><a href="._week42-bs071.html">72</a></li>
|
|
<li><a href="._week42-bs072.html">73</a></li>
|
|
<li><a href="._week42-bs073.html">74</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week42-bs099.html">100</a></li>
|
|
<li><a href="._week42-bs065.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|