added dot files
This commit is contained in:
mhjensen
@@ -0,0 +1,442 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0088"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec87" class="anchor">Applications: solving ordinary differential equations with Neural Networks </h2>
|
||||
|
||||
<p>
|
||||
We end our discussion on neural networks with a discussion on how to solve differential equations. Here we focus
|
||||
first on the classical exponential decay in one dimension. Thereafter we switch to the Poisson equation in one dimension.
|
||||
|
||||
<p>
|
||||
The aim is to see if we can use a neural network to solve
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\tag{16}
|
||||
g'(x) = -\gamma g(x)
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( g(0) = g_0 \) with \( \gamma \) and \( g_0 \) being some chosen
|
||||
values. This equation is an ordinary differential equation since the
|
||||
function we have to solve for, \( g(x) \), is of one variable.
|
||||
|
||||
<p>
|
||||
Here we set \( \gamma = 2 \) and \( g_0 = 10 \) but feel free to change
|
||||
them and see how the neural network performs.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs087.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs080.html">81</a></li>
|
||||
<li><a href="._NeuralNet-bs081.html">82</a></li>
|
||||
<li><a href="._NeuralNet-bs082.html">83</a></li>
|
||||
<li><a href="._NeuralNet-bs083.html">84</a></li>
|
||||
<li><a href="._NeuralNet-bs084.html">85</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0089"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec88" class="anchor">Trial solution </h2>
|
||||
|
||||
<p>
|
||||
To begin with, a trial solution \( g_t(t) \) must be chosen. A general
|
||||
trial solution for ordinary differential equations could be
|
||||
|
||||
$$
|
||||
g_t(x, P) = h_1(x) + h_2(x, N(x, P)),
|
||||
$$
|
||||
|
||||
<p>
|
||||
with \( h_1(x) \) ensuring that \( g_t(x) \) satisfies some conditions and
|
||||
\( h_2(x,N(x, P)) \) an expression involving \( x \) and the output from the
|
||||
neural network \( N(x,P) \) with \( P \) being the collection of the weights
|
||||
and biases for each layer.
|
||||
|
||||
<p>
|
||||
It is assumed that there are no weights and
|
||||
bias at the input layer, so \( P = \{ P_{\text{hidden}},
|
||||
P_{\text{output}} \} \). If there are \( N_{\text{hidden} } \) neurons in
|
||||
the hidden layer, then \( P_{\text{hidden}} \) is an \( N_{\text{hidden} }
|
||||
\times 2 \) matrix.
|
||||
|
||||
<p>
|
||||
The first column in \( P_{\text{hidden} } \) represents
|
||||
the bias for each neuron in the hidden layer and the second column
|
||||
represents the weigths for each neuron. If there are \( N_{\text{output}
|
||||
} \) neurons in the output layer, then \( P_{\text{output}} \) is a
|
||||
\( N_{\text{output} } \times (1 + N_{\text{hidden} }) \) matrix. Its first
|
||||
column represents the bias of each neuron and the remaining columns
|
||||
represents the weights to each neuron.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs088.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs081.html">82</a></li>
|
||||
<li><a href="._NeuralNet-bs082.html">83</a></li>
|
||||
<li><a href="._NeuralNet-bs083.html">84</a></li>
|
||||
<li><a href="._NeuralNet-bs084.html">85</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0090"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec89" class="anchor">More details </h2>
|
||||
|
||||
<p>
|
||||
We have \( g(0) = g_0 \). The trial solution must fulfill this
|
||||
condition to be a proper solution of <a href="._NeuralNet-bs088.html#mjx-eqn-16">(16)</a>.
|
||||
|
||||
<p>
|
||||
A possible way to
|
||||
ensure that \( g_t(0, P) = g_0 \), is to let \( F(N(x,P)) = x\cdot N(x,P) \)
|
||||
and \( A(x) = g_0 \). This gives the following trial solution:
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
g_t(x, P) = g_0 + x \cdot N(x, P).
|
||||
\tag{17}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs089.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs082.html">83</a></li>
|
||||
<li><a href="._NeuralNet-bs083.html">84</a></li>
|
||||
<li><a href="._NeuralNet-bs084.html">85</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0091"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec90" class="anchor">Reformulating the problem </h2>
|
||||
|
||||
<p>
|
||||
Often, the role of a neural network is to minimize its parameters with
|
||||
respect to some given error criteria. This criteria, the cost or loss
|
||||
function, is a measure of how much error the output of the network has
|
||||
compared to some given known answers. A reformulation of
|
||||
<a href="._NeuralNet-bs088.html#mjx-eqn-16">(16)</a> must therefore be done, such that it describes the
|
||||
problem a neural network can solve.
|
||||
|
||||
<p>
|
||||
The neural network must find the set of weigths and biases \( P \) such
|
||||
that the trial solution in satisfies
|
||||
<a href="._NeuralNet-bs088.html#mjx-eqn-16">(16)</a>. The trial solution has been chosen such that it
|
||||
already solves the condition \( g(0) = g_0 \). What remains, is to find
|
||||
\( P \) such that
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
g_t'(x, P) = - \gamma g_t(x, P)
|
||||
\tag{18}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
is fulfilled as <em>best as possible</em>.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs090.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs083.html">84</a></li>
|
||||
<li><a href="._NeuralNet-bs084.html">85</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0092"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec91" class="anchor">Estimating errors </h2>
|
||||
|
||||
<p>
|
||||
Having two sides of an equation as equal as
|
||||
possible, means that the absolute or squared difference between the
|
||||
sides must be as close to zero as small. In this case, the difference
|
||||
squared is an appropiate measurement of how errorneous the trial
|
||||
solution is with respect to \( P \) of the neural network. Therefore, the
|
||||
problem our network must solve, is
|
||||
|
||||
$$
|
||||
\min_{P}\Big\{ \big(g_t'(x, P) - ( -\gamma g_t(x, P) \big)^2 \Big\}
|
||||
$$
|
||||
|
||||
<p>
|
||||
or, in terms of weights and biases for each layer:
|
||||
|
||||
$$
|
||||
\min_{P_{\text{hidden} }, \ P_{\text{output} }}\Big\{ \big(g_t'(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) - ( -\gamma g_t(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) \big)^2 \Big\}
|
||||
$$
|
||||
|
||||
<p>
|
||||
for an input value \( x \).
|
||||
If the neural network evaluates \( g_t(x, P) \) at more avalues for \( x \), say \( N \) values \( x_i \) for \( i = 1, \dots, N \), then the <em>total</em> error to minimize is
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
\tag{19}
|
||||
\min_{P}\Big\{\sum_i \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2 \Big\}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
Letting \( c(x, P) = \sum_i \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P)
|
||||
\big)^2 \) denote the cost function, the minimization problem of which
|
||||
our network must solve, is
|
||||
|
||||
$$
|
||||
\min_{P} c(x, P)
|
||||
$$
|
||||
|
||||
<p>
|
||||
or in terms of \( P_{\text{hidden} } \) and \( P_{\text{output} } \)
|
||||
|
||||
$$
|
||||
\min_{P_{\text{hidden} }, \ P_{\text{output} }} c(x, \{P_{\text{hidden} }, P_{\text{output} }\})
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs091.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs084.html">85</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0093"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec92" class="anchor">Creating a simple Deep Neural Net </h2>
|
||||
|
||||
<p>
|
||||
The next step is to decide how the neural net \( N(x, P) \)
|
||||
should be. In this case, the neural network is made
|
||||
from scratch to understand better how a neural network works, gain
|
||||
more control over its architecture, and see how Autograd can be used
|
||||
to simplify the implementation.
|
||||
|
||||
<p>
|
||||
Since a deep neural network (DNN) is a neural network with more than
|
||||
one hidden layer, we can first look on how to implement a neural
|
||||
network. Having an implementation of a neural network at hand, an
|
||||
extension of it into a deep neural network would (hopefully) be
|
||||
painless.
|
||||
|
||||
<p>
|
||||
For simplicity, we assume that the input is an array \( \vec x =
|
||||
(x_1, \dots, x_N) \) with \( N \) elements. It is at these points the neural
|
||||
network should find \( P \) such that it fulfills <a href="._NeuralNet-bs092.html#mjx-eqn-19">(19)</a>.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs092.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs085.html">86</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0094"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec93" class="anchor">Feedforward </h2>
|
||||
|
||||
<p>
|
||||
First, a feedforward of the inputs must be done. This means that \( \vec
|
||||
x \) must be passed through an input layer, a hidden layer and a output
|
||||
layer. The input layer in this case, does not need to process the
|
||||
data any further. The input layer will consist of \( N_{\text{input} } \)
|
||||
neurons, passing its element to each neuron in the hidden layer. The
|
||||
number of neurons in the hidden layer will be \( N_{\text{hidden} } \).
|
||||
|
||||
<p>
|
||||
For the \( i \)-th in the hidden layer with weight \( w_i^{\text{hidden} } \)
|
||||
and bias \( b_i^{\text{hidden} } \), the weighting from the \( j \)-th neuron
|
||||
at the input layer is:
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
z_{i,j}^{\text{hidden}} &= b_i^{\text{hidden}} + w_i^{\text{hidden}}x_j \\
|
||||
&=
|
||||
\begin{pmatrix}
|
||||
b_i^{\text{hidden}} & w_i^{\text{hidden}}
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 \\
|
||||
x_j
|
||||
\end{pmatrix}
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs093.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs086.html">87</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0095"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec94" class="anchor">Result after weighting </h2>
|
||||
|
||||
<p>
|
||||
The result after weighting the input at the \( i \)-th hidden neuron can be written as a vector:
|
||||
$$
|
||||
\begin{aligned}
|
||||
\vec{z}_{i}^{\text{hidden}} &= \Big( b_i^{\text{hidden}} + w_i^{\text{hidden}}x_1 , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_2, \ \dots \, , \ b_i^{\text{hidden}} + w_i^{\text{hidden}} x_N\Big) \\
|
||||
&=
|
||||
\begin{pmatrix}
|
||||
b_i^{\text{hidden}} & w_i^{\text{hidden}}
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 & 1 & \dots & 1 \\
|
||||
x_1 & x_2 & \dots & x_N
|
||||
\end{pmatrix} \\
|
||||
&= \vec{p}_{i, \text{hidden}}^T X
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
It is the vector \( \vec{p}_{i, \text{hidden}}^T \) that defines each row
|
||||
in \( P_{\text{hidden} } \), which contains the weights for the neural
|
||||
network to minimize according to <a href="._NeuralNet-bs092.html#mjx-eqn-19">(19)</a>.
|
||||
|
||||
<p>
|
||||
After having found \( \vec{z}_{i}^{\text{hidden}} \) for every neuron \( i \)
|
||||
in the hidden layer, the vector will be sent to an activation function
|
||||
\( a_i(\vec{z}) \). In this example, the sigmoid function has been used:
|
||||
|
||||
$$
|
||||
f(z) = \frac{1}{1 + \exp{(-z)}}.
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs094.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs087.html">88</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0096"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec95" class="anchor">Output </h2>
|
||||
|
||||
<p>
|
||||
The output $\vec{x}_i^{\text{hidden} }$from each \( i \)-th hidden neuron is:
|
||||
|
||||
$$
|
||||
\vec{x}_i^{\text{hidden} } = f\big( \vec{z}_{i}^{\text{hidden}} \big).
|
||||
$$
|
||||
|
||||
<p>
|
||||
The outputs \( \vec{x}_i^{\text{hidden} } \) are then sent to the output layer.
|
||||
|
||||
<p>
|
||||
The output layer consist of one neuron in this case, and combines the
|
||||
output from each of the neurons in the hidden layers. The output layer
|
||||
combines the results from the hidden layer using some weights \(
|
||||
w_i^{\text{output}} \) and biases \( b_i^{\text{output}} \). In this case,
|
||||
it is assumes that the number of neurons in the output layer is one.
|
||||
|
||||
<p>
|
||||
The procedure of weigthing the output neuron \( j \) in the hidden layer
|
||||
to the \( i \)-th neuron in the output layer is similar as for the hidden
|
||||
layer described previously.
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
z_{1,j}^{\text{output}} & =
|
||||
\begin{pmatrix}
|
||||
b_1^{\text{output}} & \vec{w}_1^{\text{output}}
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 \\
|
||||
\vec{x}_j^{\text{hidden}}
|
||||
\end{pmatrix}
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
Expressing \( z_{1,j}^{\text{output}} \) as a vector gives the following procedure of weighting the inputs from the hidden layer:
|
||||
|
||||
$$
|
||||
\vec{z}_{1}^{\text{output}} =
|
||||
\begin{pmatrix}
|
||||
b_1^{\text{output}} & \vec{w}_1^{\text{output}}
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 & 1 & \dots & 1 \\
|
||||
\vec{x}_1^{\text{hidden}} & \vec{x}_2^{\text{hidden}} & \dots & \vec{x}_N^{\text{hidden}}
|
||||
\end{pmatrix}
|
||||
$$
|
||||
|
||||
<p>
|
||||
In this case we seek a continous range of values since we are
|
||||
approximating a function. This means that after computing
|
||||
\( \vec{z}_{1}^{\text{output}} \) the neural network has finished its
|
||||
feedforward step, and \( \vec{z}_{1}^{\text{output}} \) is the final
|
||||
output of the network.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs095.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs088.html">89</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0097"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec96" class="anchor">Setting up the code, feed forward part </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Note that we use the numpy wrapper for Autograd (see the gradient descent slides)</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad, elementwise_grad
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy.random</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">npr</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> pyplot <span style="color: #008000; font-weight: bold">as</span> plt
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(z):
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">neural_network</span>(params, x):
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the weights (including and biases) for the hidden and output layer.</span>
|
||||
<span style="color: #408080; font-style: italic"># Assume that params is a list of parameters for each layer. </span>
|
||||
<span style="color: #408080; font-style: italic"># The biases are the first element for each array in params, </span>
|
||||
<span style="color: #408080; font-style: italic"># and the weights are the remaning elements in each array in params. </span>
|
||||
|
||||
w_hidden <span style="color: #666666">=</span> params[<span style="color: #666666">0</span>]
|
||||
w_output <span style="color: #666666">=</span> params[<span style="color: #666666">1</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Assumes input x being an one-dimensional array</span>
|
||||
num_values <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(x)
|
||||
x <span style="color: #666666">=</span> x<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, num_values)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Assume that the input layer does nothing to the input x</span>
|
||||
x_input <span style="color: #666666">=</span> x
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Hidden layer:</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Add a row of ones to include bias</span>
|
||||
x_input <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_input ), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
||||
|
||||
z_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_hidden, x_input)
|
||||
x_hidden <span style="color: #666666">=</span> sigmoid(z_hidden)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Output layer:</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Include bias:</span>
|
||||
x_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_hidden ), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
||||
|
||||
z_output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_output, x_hidden)
|
||||
x_output <span style="color: #666666">=</span> z_output
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> x_output
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs096.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs089.html">90</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0098"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec97" class="anchor">Backpropagation </h2>
|
||||
|
||||
<p>
|
||||
Now that feedforward can be done, the next step is to decide how the
|
||||
parameters should change such that they minimize the cost function.
|
||||
|
||||
<p>
|
||||
Recall that the chosen cost function for this problem is
|
||||
|
||||
$$
|
||||
c(x, P) = \sum_i \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2
|
||||
$$
|
||||
|
||||
<p>
|
||||
In order to minimize it, an optimalization method must be chosen.
|
||||
|
||||
<p>
|
||||
Here, gradient descent with a constant step size has been chosen.
|
||||
|
||||
<p>
|
||||
Before looking at the gradient descent method, let us set up the cost
|
||||
function along with the right ride of the ODE and trial solution.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># The trial solution using the deep neural network:</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g_trial</span>(x,params, g0 <span style="color: #666666">=</span> <span style="color: #666666">10</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> g0 <span style="color: #666666">+</span> x<span style="color: #666666">*</span>neural_network(params,x)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The right side of the ODE:</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g</span>(x, g_trial, gamma <span style="color: #666666">=</span> <span style="color: #666666">2</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">-</span>gamma<span style="color: #666666">*</span>g_trial
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The cost function:</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cost_function</span>(P, x):
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Evaluate the trial function with the current parameters P</span>
|
||||
g_t <span style="color: #666666">=</span> g_trial(x,P)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the derivative w.r.t x of the neural network</span>
|
||||
d_net_out <span style="color: #666666">=</span> elementwise_grad(neural_network,<span style="color: #666666">1</span>)(P,x)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the derivative w.r.t x of the trial function</span>
|
||||
d_g_t <span style="color: #666666">=</span> elementwise_grad(g_trial,<span style="color: #666666">0</span>)(x,P)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The right side of the ODE </span>
|
||||
func <span style="color: #666666">=</span> g(x, g_t)
|
||||
|
||||
err_sqr <span style="color: #666666">=</span> (d_g_t <span style="color: #666666">-</span> func)<span style="color: #666666">**2</span>
|
||||
cost_sum <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(err_sqr)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> cost_sum
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs097.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs090.html">91</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0099"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec98" class="anchor">Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
The idea of the gradient descent algorithm is to update parameters in
|
||||
direction where the cost function decreases goes to a minimum.
|
||||
|
||||
<p>
|
||||
In general, the update of some parameters \( \vec \omega \) given a cost
|
||||
function defined by some weights \( \vec \omega \), \( c(x, \vec \omega) \),
|
||||
goes as follows:
|
||||
|
||||
$$
|
||||
\vec \omega_{\text{new} } = \vec \omega - \lambda \nabla_{\vec \omega} c(x, \vec \omega),
|
||||
$$
|
||||
|
||||
<p>
|
||||
for a number of iterations or until $ \big|\big| \vec
|
||||
\omega_{\text{new} } - \vec \omega \big|\big|$ is smaller than some
|
||||
given tolerance.
|
||||
|
||||
<p>
|
||||
The value of \( \lambda \) decides how large steps the algorithm must take
|
||||
in the direction of $ \nabla_{\vec \omega} c(x, \vec \omega)$. The
|
||||
notatation \( \nabla_{\vec \omega} \) denotes the gradient with respect to
|
||||
the elements in \( \vec \omega \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs098.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs091.html">92</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0100"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec99" class="anchor">More on GD and cost function </h2>
|
||||
|
||||
<p>
|
||||
In our case, we have to minimize the cost function \( c(x, P) \) with
|
||||
respect to the two sets of weights and bisases, that is for the hidden
|
||||
layer \( P_{\text{hidden} } \) and for the ouput layer \( P_{\text{output}
|
||||
} \) .
|
||||
|
||||
<p>
|
||||
This means that \( P_{\text{hidden} } \) and \( P_{\text{output} } \) is
|
||||
updated by
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
P_{\text{hidden},\text{new}} &= P_{\text{hidden}} - \lambda \nabla_{P_{\text{hidden}}} c(x, P) \\
|
||||
P_{\text{output},\text{new}} &= P_{\text{output}} - \lambda \nabla_{P_{\text{output}}} c(x, P)
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
This might look like a cumberstone to set up the correct expression
|
||||
for finding the gradients. Luckily, Autograd comes to the rescue.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">solve_ode_neural_network</span>(x, num_neurons_hidden, num_iter, lmb):
|
||||
<span style="color: #408080; font-style: italic">## Set up initial weigths and biases </span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># For the hidden layer</span>
|
||||
p0 <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(num_neurons_hidden, <span style="color: #666666">2</span> )
|
||||
|
||||
<span style="color: #408080; font-style: italic"># For the output layer</span>
|
||||
p1 <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(<span style="color: #666666">1</span>, num_neurons_hidden <span style="color: #666666">+</span> <span style="color: #666666">1</span> ) <span style="color: #408080; font-style: italic"># +1 since bias is included</span>
|
||||
|
||||
P <span style="color: #666666">=</span> [p0, p1]
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Initial cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function(P, x))
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Start finding the optimal weigths using gradient descent</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the Python function that represents the gradient of the cost function</span>
|
||||
<span style="color: #408080; font-style: italic"># w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer</span>
|
||||
cost_function_grad <span style="color: #666666">=</span> grad(cost_function,<span style="color: #666666">0</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Let the update be done num_iter times</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(num_iter):
|
||||
<span style="color: #408080; font-style: italic"># Evaluate the gradient at the current weights and biases in P. </span>
|
||||
<span style="color: #408080; font-style: italic"># The cost_grad consist now of two arrays; </span>
|
||||
<span style="color: #408080; font-style: italic"># one for the gradient w.r.t P_hidden and </span>
|
||||
<span style="color: #408080; font-style: italic"># one for the gradient w.r.t P_output</span>
|
||||
cost_grad <span style="color: #666666">=</span> cost_function_grad(P, x)
|
||||
|
||||
P[<span style="color: #666666">0</span>] <span style="color: #666666">=</span> P[<span style="color: #666666">0</span>] <span style="color: #666666">-</span> lmb <span style="color: #666666">*</span> cost_grad[<span style="color: #666666">0</span>]
|
||||
P[<span style="color: #666666">1</span>] <span style="color: #666666">=</span> P[<span style="color: #666666">1</span>] <span style="color: #666666">-</span> lmb <span style="color: #666666">*</span> cost_grad[<span style="color: #666666">1</span>]
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Final cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function(P, x))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> P
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs099.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs092.html">93</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0101"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec100" class="anchor">An implementation of a Deep Neural Network </h2>
|
||||
|
||||
<p>
|
||||
As previously stated, a Deep Neural Network (DNN) follows the same
|
||||
concept of a neural network, but having more than one hidden
|
||||
layer. Suppose that the network has \( N_{\text{hidden}} \) hidden layers
|
||||
where the \( l \)-th layer has \( N_{\text{hidden}}^{(l)} \) neurons. The
|
||||
input is still assumed to be an array of size \( 1 \times N \). The
|
||||
network must now try to optimalize its output with respect to the
|
||||
collection of weigths and biases \( P = \big\{P_{\text{input} }, \
|
||||
P_{\text{hidden} }^{(1)}, \ P_{\text{hidden} }^{(2)}, \ \dots , \
|
||||
P_{\text{hidden} }^{(N_{\text{hidden}})}, \ P_{\text{output} }\big\} \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs100.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs093.html">94</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0102"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec101" class="anchor">Feed forward again </h2>
|
||||
|
||||
<p>
|
||||
The feedforward step is similar to as for the neural netowork, but now considering more than one hidden layer.
|
||||
|
||||
<p>
|
||||
The \( i \)-th neuron at layer \( l \) recieves the result
|
||||
\( \vec{x}_j^{(l-1),\text{hidden} } \) from the \( j \)-th neuron at layer
|
||||
\( l-1 \). The \( i \)-th neuron at layer \( l \) weights all of the elements in
|
||||
\( \vec{x}_j^{(l-1),\text{hidden} } \) with a weight vector \( \vec
|
||||
w_{i,j}^{(l), \ \text{hidden} } \) with as many weigths as there are
|
||||
elements in$\vec{x}_j^{(l-1),\text{hidden} }$, and adds a bias
|
||||
\( b_i^{(l), \ \text{hidden} } \):
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
z_{i,j}^{(l),\ \text{hidden}} &= b_i^{(l), \ \text{hidden}} + \big(\vec{w}_{i}^{(l), \ \text{hidden}}\big)^T\vec{x}_j^{(l-1),\text{hidden} } \\
|
||||
&=
|
||||
\begin{pmatrix}
|
||||
b_i^{(l), \ \text{hidden}} & \big(\vec{w}_{i}^{(l), \ \text{hidden}}\big)^T
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 \\
|
||||
\vec{x}_j^{(l-1),\text{hidden} }
|
||||
\end{pmatrix}
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
The output from the \( i \)-th neuron at the hidden layer \( l \) becomes a vector \( \vec{z}_{i}^{(l),\ \text{hidden}} \):
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
\vec{z}_{i}^{(l),\ \text{hidden}} &= \Big( b_i^{(l), \ \text{hidden}} + \big(\vec{w}_{i}^{(l), \ \text{hidden}}\big)^T\vec{x}_1^{(l-1),\text{hidden} }, \ \dots \ , \ b_i^{(l), \ \text{hidden}} + \big(\vec{w}_{i}^{(l), \ \text{hidden}}\big)^T\vec{x}_{N_{hidden}^{(l-1)}}^{(l-1),\text{hidden} } \Big) \\
|
||||
&=
|
||||
\begin{pmatrix}
|
||||
b_i^{(l), \ \text{hidden}} & \big(\vec{w}_{i}^{(l), \ \text{hidden}}\big)^T
|
||||
\end{pmatrix}
|
||||
\begin{pmatrix}
|
||||
1 & 1 & \dots & 1 \\
|
||||
\vec{x}_{1}^{(l-1),\text{hidden} } & \vec{x}_{2}^{(l-1),\text{hidden} } & \dots & \vec{x}_{N_{hidden}^{(l-1)}}^{(l-1),\text{hidden} }
|
||||
\end{pmatrix}
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs101.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs094.html">95</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0103"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec102" class="anchor">The final parts of the code </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">deep_neural_network</span>(deep_params, x):
|
||||
<span style="color: #408080; font-style: italic"># N_hidden is the number of hidden layers </span>
|
||||
N_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(deep_params) <span style="color: #666666">-</span> <span style="color: #666666">1</span> <span style="color: #408080; font-style: italic"># -1 since params consist of parameters to all the hidden layers AND the output layer</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Assumes input x being an one-dimensional array</span>
|
||||
num_values <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(x)
|
||||
x <span style="color: #666666">=</span> x<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, num_values)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Assume that the input layer does nothing to the input x</span>
|
||||
x_input <span style="color: #666666">=</span> x
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Due to multiple hidden layers, define a variable referencing to the</span>
|
||||
<span style="color: #408080; font-style: italic"># output of the previous layer:</span>
|
||||
x_prev <span style="color: #666666">=</span> x_input
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Hidden layers:</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(N_hidden):
|
||||
<span style="color: #408080; font-style: italic"># From the list of parameters P; find the correct weigths and bias for this layer</span>
|
||||
w_hidden <span style="color: #666666">=</span> deep_params[l]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Add a row of ones to include bias</span>
|
||||
x_prev <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_prev ), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
||||
|
||||
z_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_hidden, x_prev)
|
||||
x_hidden <span style="color: #666666">=</span> sigmoid(z_hidden)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Update x_prev such that next layer can use the output from this layer</span>
|
||||
x_prev <span style="color: #666666">=</span> x_hidden
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Output layer:</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Get the weights and bias for this layer</span>
|
||||
w_output <span style="color: #666666">=</span> deep_params[<span style="color: #666666">-1</span>]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Include bias:</span>
|
||||
x_prev <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_prev), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
||||
|
||||
z_output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_output, x_prev)
|
||||
x_output <span style="color: #666666">=</span> z_output
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> x_output
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs102.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs095.html">96</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0104"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec103" class="anchor">And adding Back propagation </h2>
|
||||
|
||||
<p>
|
||||
This step is very similar for the neural network. The idea in this
|
||||
step is the same as for the neural network, but with more parameters
|
||||
to update for. Again there is no need for computing the gradients
|
||||
analytically since Autograd does the work for us.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># The trial solution using the deep neural network:</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g_trial_deep</span>(x,params, g0 <span style="color: #666666">=</span> <span style="color: #666666">10</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> g0 <span style="color: #666666">+</span> x<span style="color: #666666">*</span>deep_neural_network(params,x)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The same cost function as for the neural network, but calls deep_neural_network instead.</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cost_function_deep</span>(P, x):
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Evaluate the trial function with the current parameters P</span>
|
||||
g_t <span style="color: #666666">=</span> g_trial_deep(x,P)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the derivative w.r.t x of the neural network</span>
|
||||
d_net_out <span style="color: #666666">=</span> elementwise_grad(deep_neural_network,<span style="color: #666666">1</span>)(P,x)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the derivative w.r.t x of the trial function</span>
|
||||
d_g_t <span style="color: #666666">=</span> elementwise_grad(g_trial_deep,<span style="color: #666666">0</span>)(x,P)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The right side of the ODE </span>
|
||||
func <span style="color: #666666">=</span> g(x, g_t)
|
||||
|
||||
err_sqr <span style="color: #666666">=</span> (d_g_t <span style="color: #666666">-</span> func)<span style="color: #666666">**2</span>
|
||||
cost_sum <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(err_sqr)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> cost_sum
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">solve_ode_deep_neural_network</span>(x, num_neurons, num_iter, lmb):
|
||||
<span style="color: #408080; font-style: italic"># num_hidden_neurons is now a list of number of neurons within each hidden layer</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the number of hidden layers:</span>
|
||||
N_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(num_neurons)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Set up initial weigths and biases </span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Initialize the list of parameters:</span>
|
||||
P <span style="color: #666666">=</span> [<span style="color: #008000">None</span>]<span style="color: #666666">*</span>(N_hidden <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #408080; font-style: italic"># + 1 to include the output layer</span>
|
||||
|
||||
P[<span style="color: #666666">0</span>] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(num_neurons[<span style="color: #666666">0</span>], <span style="color: #666666">2</span> )
|
||||
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,N_hidden):
|
||||
P[l] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(num_neurons[l], num_neurons[l<span style="color: #666666">-1</span>] <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #408080; font-style: italic"># +1 to include bias </span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># For the output layer</span>
|
||||
P[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(<span style="color: #666666">1</span>, num_neurons[<span style="color: #666666">-1</span>] <span style="color: #666666">+</span> <span style="color: #666666">1</span> ) <span style="color: #408080; font-style: italic"># +1 since bias is included</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Initial cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function_deep(P, x))
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Start finding the optimal weigths using gradient descent</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Find the Python function that represents the gradient of the cost function</span>
|
||||
<span style="color: #408080; font-style: italic"># w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer</span>
|
||||
cost_function_deep_grad <span style="color: #666666">=</span> grad(cost_function_deep,<span style="color: #666666">0</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Let the update be done num_iter times</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(num_iter):
|
||||
<span style="color: #408080; font-style: italic"># Evaluate the gradient at the current weights and biases in P. </span>
|
||||
<span style="color: #408080; font-style: italic"># The cost_grad consist now of N_hidden + 1 arrays; the gradient w.r.t the weights and biases</span>
|
||||
<span style="color: #408080; font-style: italic"># in the hidden layers and output layers evaluated at x.</span>
|
||||
cost_deep_grad <span style="color: #666666">=</span> cost_function_deep_grad(P, x)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(N_hidden<span style="color: #666666">+1</span>):
|
||||
P[l] <span style="color: #666666">=</span> P[l] <span style="color: #666666">-</span> lmb <span style="color: #666666">*</span> cost_deep_grad[l]
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Final cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function_deep(P, x))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> P
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs103.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs096.html">97</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0105"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec104" class="anchor">Solving the ODE </h2>
|
||||
|
||||
<p>
|
||||
Finally, having set up the networks we are ready to use them to solve the ODE problem.
|
||||
We add the analytical solution
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g_analytic</span>(x, gamma <span style="color: #666666">=</span> <span style="color: #666666">2</span>, g0 <span style="color: #666666">=</span> <span style="color: #666666">10</span>):
|
||||
<span style="color: #008000; font-weight: bold">return</span> g0<span style="color: #666666">*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>gamma<span style="color: #666666">*</span>x)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs104.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs097.html">98</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0106"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec105" class="anchor">Using neural network </h2>
|
||||
|
||||
<p>
|
||||
The code below solves the ODE using a neural network. The number of
|
||||
values for the input \( \vec x \) is 10, number of hidden neurons in the
|
||||
hidden layer being 10 and th step size used in gradien descent
|
||||
\( \lambda = 0.001 \). The program updates the weights and biases in the
|
||||
network <em>num_iter</em> times. Finally, it plots the results from using the
|
||||
neural network along with the analytical solution.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>npr<span style="color: #666666">.</span>seed(<span style="color: #666666">15</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Decide the vales of arguments to the function to solve</span>
|
||||
N <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, N)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Set up the initial parameters</span>
|
||||
num_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
num_iter <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
|
||||
lmb <span style="color: #666666">=</span> <span style="color: #666666">0.001</span>
|
||||
|
||||
P <span style="color: #666666">=</span> solve_ode_neural_network(x, num_hidden_neurons, num_iter, lmb)
|
||||
|
||||
res <span style="color: #666666">=</span> g_trial(x,P)
|
||||
res_analytical <span style="color: #666666">=</span> g_analytic(x)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'Max absolute difference: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>np<span style="color: #666666">.</span>max(np<span style="color: #666666">.</span>abs(res <span style="color: #666666">-</span> res_analytical)))
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>,<span style="color: #666666">10</span>))
|
||||
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">'Performance of neural network solving an ODE compared to the analytical solution'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, res_analytical)
|
||||
plt<span style="color: #666666">.</span>plot(x, res[<span style="color: #666666">0</span>,:])
|
||||
plt<span style="color: #666666">.</span>legend([<span style="color: #BA2121">'analytical'</span>,<span style="color: #BA2121">'nn'</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'x'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'g(x)'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs105.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs098.html">99</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs108.html#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0107"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec106" class="anchor">Using a deep neural network </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>npr<span style="color: #666666">.</span>seed(<span style="color: #666666">15</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Decide the vales of arguments to the function to solve</span>
|
||||
N <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, N)
|
||||
|
||||
<span style="color: #408080; font-style: italic">## Set up the initial parameters</span>
|
||||
num_hidden_neurons <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([<span style="color: #666666">10</span>,<span style="color: #666666">10</span>])
|
||||
num_iter <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
|
||||
lmb <span style="color: #666666">=</span> <span style="color: #666666">0.001</span>
|
||||
|
||||
P <span style="color: #666666">=</span> solve_ode_deep_neural_network(x, num_hidden_neurons, num_iter, lmb)
|
||||
|
||||
res <span style="color: #666666">=</span> g_trial_deep(x,P)
|
||||
res_analytical <span style="color: #666666">=</span> g_analytic(x)
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>,<span style="color: #666666">10</span>))
|
||||
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">'Performance of a deep neural network solving an ODE compared to the analytical solution'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, res_analytical)
|
||||
plt<span style="color: #666666">.</span>plot(x, res[<span style="color: #666666">0</span>,:])
|
||||
plt<span style="color: #666666">.</span>legend([<span style="color: #BA2121">'analytical'</span>,<span style="color: #BA2121">'dnn'</span>])
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'g(x)'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs106.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs099.html">100</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
<li><a href="._NeuralNet-bs108.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
<!--
|
||||
Automatically generated HTML file from DocOnce source
|
||||
(https://github.com/hplgit/doconce/)
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||||
|
||||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
<!-- not necessary
|
||||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||||
-->
|
||||
|
||||
<style type="text/css">
|
||||
|
||||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||||
.dropdown-menu {
|
||||
height: auto;
|
||||
max-height: 400px;
|
||||
overflow-x: hidden;
|
||||
}
|
||||
|
||||
/* Adds an invisible element before each target to offset for the navigation
|
||||
bar */
|
||||
.anchor::before {
|
||||
content:"";
|
||||
display:block;
|
||||
height:50px; /* fixed header height for style bootstrap */
|
||||
margin:-50px 0 0; /* negative fixed header height */
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||||
('Artificial neurons', 2, None, '___sec1'),
|
||||
('Neural network types', 2, None, '___sec2'),
|
||||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||||
('Recurrent neural networks', 2, None, '___sec5'),
|
||||
('Other types of networks', 2, None, '___sec6'),
|
||||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||||
('Mathematical model', 2, None, '___sec9'),
|
||||
('Mathematical model', 2, None, '___sec10'),
|
||||
('Mathematical model', 2, None, '___sec11'),
|
||||
('Mathematical model', 2, None, '___sec12'),
|
||||
('Mathematical model', 2, None, '___sec13'),
|
||||
('Matrix-vector notation', 3, None, '___sec14'),
|
||||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||||
('Activation functions', 3, None, '___sec16'),
|
||||
('Activation functions, Logistic and Hyperbolic ones',
|
||||
3,
|
||||
None,
|
||||
'___sec17'),
|
||||
('Relevance', 3, None, '___sec18'),
|
||||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||||
('From one to many layers, the universal approximation theorem',
|
||||
2,
|
||||
None,
|
||||
'___sec20'),
|
||||
('Deriving the back propagation code for a multilayer perceptron '
|
||||
'model',
|
||||
2,
|
||||
None,
|
||||
'___sec21'),
|
||||
('Definitions', 2, None, '___sec22'),
|
||||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||||
('Derivative of the cost function', 2, None, '___sec24'),
|
||||
('Bringing it together, first back propagation equation',
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||||
('Bringing it together', 2, None, '___sec27'),
|
||||
('Final back propagating equation', 2, None, '___sec28'),
|
||||
('Setting up the Back propagation algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Setting up a Multi-layer perceptron model for classification',
|
||||
2,
|
||||
None,
|
||||
'___sec30'),
|
||||
('Defining the cost function', 2, None, '___sec31'),
|
||||
('Example: binary classification problem', 2, None, '___sec32'),
|
||||
('The Softmax function', 2, None, '___sec33'),
|
||||
('Developing a code for doing neural networks with back '
|
||||
'propagation',
|
||||
2,
|
||||
None,
|
||||
'___sec34'),
|
||||
('Collect and pre-process data', 2, None, '___sec35'),
|
||||
('Train and test datasets', 2, None, '___sec36'),
|
||||
('Define model and architecture', 2, None, '___sec37'),
|
||||
('Layers', 2, None, '___sec38'),
|
||||
('Weights and biases', 2, None, '___sec39'),
|
||||
('Feed-forward pass', 2, None, '___sec40'),
|
||||
('Matrix multiplications', 2, None, '___sec41'),
|
||||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||||
('Optimizing the cost function', 2, None, '___sec43'),
|
||||
('Regularization', 2, None, '___sec44'),
|
||||
('Matrix multiplication', 2, None, '___sec45'),
|
||||
('Improving performance', 2, None, '___sec46'),
|
||||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||||
('Visualization', 2, None, '___sec50'),
|
||||
('scikit-learn implementation', 2, None, '___sec51'),
|
||||
('Visualization', 2, None, '___sec52'),
|
||||
('Building neural networks in Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
('Tensorflow', 2, None, '___sec54'),
|
||||
('Collect and pre-process data', 2, None, '___sec55'),
|
||||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||||
('Using Keras', 2, None, '___sec58'),
|
||||
('Which activation function should I use?', 2, None, '___sec59'),
|
||||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||||
2,
|
||||
None,
|
||||
'___sec60'),
|
||||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||||
('The RELU function family', 2, None, '___sec62'),
|
||||
('Which activation function should we use?', 2, None, '___sec63'),
|
||||
('A top-down perspective on Neural networks',
|
||||
2,
|
||||
None,
|
||||
'___sec64'),
|
||||
('Limitations of supervised learning with deep networks',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'___sec67'),
|
||||
('3D volumes of neurons', 2, None, '___sec68'),
|
||||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||||
('Transforming images', 2, None, '___sec70'),
|
||||
('CNNs in brief', 2, None, '___sec71'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'___sec72'),
|
||||
('Setting it up', 2, None, '___sec73'),
|
||||
('The MNIST dataset again', 2, None, '___sec74'),
|
||||
('Strong correlations', 2, None, '___sec75'),
|
||||
('Layers of a CNN', 2, None, '___sec76'),
|
||||
('Systematic reduction', 2, None, '___sec77'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'___sec78'),
|
||||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||||
('Train the model', 2, None, '___sec81'),
|
||||
('Visualizing the results', 2, None, '___sec82'),
|
||||
('Running with Keras', 2, None, '___sec83'),
|
||||
('Final part', 2, None, '___sec84'),
|
||||
('Final visualization', 2, None, '___sec85'),
|
||||
('Fun links', 2, None, '___sec86'),
|
||||
('Applications: solving ordinary differential equations with '
|
||||
'Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec87'),
|
||||
('Trial solution', 2, None, '___sec88'),
|
||||
('More details', 2, None, '___sec89'),
|
||||
('Reformulating the problem', 2, None, '___sec90'),
|
||||
('Estimating errors', 2, None, '___sec91'),
|
||||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||||
('Feedforward', 2, None, '___sec93'),
|
||||
('Result after weighting', 2, None, '___sec94'),
|
||||
('Output', 2, None, '___sec95'),
|
||||
('Setting up the code, feed forward part', 2, None, '___sec96'),
|
||||
('Backpropagation', 2, None, '___sec97'),
|
||||
('Gradient Descent', 2, None, '___sec98'),
|
||||
('More on GD and cost function', 2, None, '___sec99'),
|
||||
('An implementation of a Deep Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec100'),
|
||||
('Feed forward again', 2, None, '___sec101'),
|
||||
('The final parts of the code', 2, None, '___sec102'),
|
||||
('And adding Back propagation', 2, None, '___sec103'),
|
||||
('Solving the ODE', 2, None, '___sec104'),
|
||||
('Using neural network', 2, None, '___sec105'),
|
||||
('Using a deep neural network', 2, None, '___sec106'),
|
||||
('Wrapping it up', 2, None, '___sec107')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
<ul class="nav navbar-nav navbar-right">
|
||||
<li class="dropdown">
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs066.html#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Feedforward</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Result after weighting</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Output</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Feed forward again</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs105.html#___sec104" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs106.html#___sec105" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs107.html#___sec106" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec107" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div> <!-- end of navigation bar -->
|
||||
|
||||
<div class="container">
|
||||
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0108"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec107" class="anchor">Wrapping it up </h2>
|
||||
|
||||
<p>
|
||||
By rewriting the ODE as a minimization problem, it was possible to
|
||||
solve equation using either a neural network (one hidden layer) or a
|
||||
deep neural network (more than one hidden layers). How well the
|
||||
network performed is measured by a specified cost function, which is
|
||||
the function the network tries to minimize. Using a trial solution
|
||||
which satisfies the additional condition and being defined by using
|
||||
the output from the network in some way, the minimization problem
|
||||
could be explicitly defined for out network to solve. The proposed
|
||||
solution from the network is then the trial solution with parameters,
|
||||
that is weights and biases within each layer in the network, such that
|
||||
the solution minimizes the cost function.
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
<li><a href="._NeuralNet-bs107.html">«</a></li>
|
||||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._NeuralNet-bs100.html">101</a></li>
|
||||
<li><a href="._NeuralNet-bs101.html">102</a></li>
|
||||
<li><a href="._NeuralNet-bs102.html">103</a></li>
|
||||
<li><a href="._NeuralNet-bs103.html">104</a></li>
|
||||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||||
<li><a href="._NeuralNet-bs105.html">106</a></li>
|
||||
<li><a href="._NeuralNet-bs106.html">107</a></li>
|
||||
<li><a href="._NeuralNet-bs107.html">108</a></li>
|
||||
<li class="active"><a href="._NeuralNet-bs108.html">109</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
</div> <!-- end container -->
|
||||
<!-- include javascript, jQuery *first* -->
|
||||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||||
|
||||
<!-- Bootstrap footer
|
||||
<footer>
|
||||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||||
</footer>
|
||||
-->
|
||||
|
||||
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright only on the titlepage -->
|
||||
</center>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user