mhjensen
434 lines
29 KiB
HTML
434 lines
29 KiB
HTML
<!--
|
||
Automatically generated HTML file from DocOnce source
|
||
(https://github.com/hplgit/doconce/)
|
||
-->
|
||
<html>
|
||
<head>
|
||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<meta name="description" content="Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks">
|
||
|
||
<title>Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</title>
|
||
|
||
<!-- Bootstrap style: bootstrap -->
|
||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||
<!-- not necessary
|
||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||
-->
|
||
|
||
<style type="text/css">
|
||
|
||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||
.dropdown-menu {
|
||
height: auto;
|
||
max-height: 400px;
|
||
overflow-x: hidden;
|
||
}
|
||
|
||
/* Adds an invisible element before each target to offset for the navigation
|
||
bar */
|
||
.anchor::before {
|
||
content:"";
|
||
display:block;
|
||
height:50px; /* fixed header height for style bootstrap */
|
||
margin:-50px 0 0; /* negative fixed header height */
|
||
}
|
||
</style>
|
||
|
||
|
||
</head>
|
||
|
||
<!-- tocinfo
|
||
{'highest level': 2,
|
||
'sections': [('Neural networks', 2, None, '___sec0'),
|
||
('Artificial neurons', 2, None, '___sec1'),
|
||
('Neural network types', 2, None, '___sec2'),
|
||
('Feed-forward neural networks', 2, None, '___sec3'),
|
||
('Convolutional Neural Network', 2, None, '___sec4'),
|
||
('Recurrent neural networks', 2, None, '___sec5'),
|
||
('Other types of networks', 2, None, '___sec6'),
|
||
('Multilayer perceptrons', 2, None, '___sec7'),
|
||
('Why multilayer perceptrons?', 2, None, '___sec8'),
|
||
('Mathematical model', 2, None, '___sec9'),
|
||
('Mathematical model', 2, None, '___sec10'),
|
||
('Mathematical model', 2, None, '___sec11'),
|
||
('Mathematical model', 2, None, '___sec12'),
|
||
('Mathematical model', 2, None, '___sec13'),
|
||
('Matrix-vector notation', 3, None, '___sec14'),
|
||
('Matrix-vector notation and activation', 3, None, '___sec15'),
|
||
('Activation functions', 3, None, '___sec16'),
|
||
('Activation functions, Logistic and Hyperbolic ones',
|
||
3,
|
||
None,
|
||
'___sec17'),
|
||
('Relevance', 3, None, '___sec18'),
|
||
('The multilayer perceptron (MLP)', 2, None, '___sec19'),
|
||
('From one to many layers, the universal approximation theorem',
|
||
2,
|
||
None,
|
||
'___sec20'),
|
||
('Deriving the back propagation code for a multilayer perceptron '
|
||
'model',
|
||
2,
|
||
None,
|
||
'___sec21'),
|
||
('Definitions', 2, None, '___sec22'),
|
||
('Derivatives and the chain rule', 2, None, '___sec23'),
|
||
('Derivative of the cost function', 2, None, '___sec24'),
|
||
('Bringing it together, first back propagation equation',
|
||
2,
|
||
None,
|
||
'___sec25'),
|
||
('Derivatives in terms of $z_j^L$', 2, None, '___sec26'),
|
||
('Bringing it together', 2, None, '___sec27'),
|
||
('Final back propagating equation', 2, None, '___sec28'),
|
||
('Setting up the Back propagation algorithm',
|
||
2,
|
||
None,
|
||
'___sec29'),
|
||
('Setting up a Multi-layer perceptron model for classification',
|
||
2,
|
||
None,
|
||
'___sec30'),
|
||
('Defining the cost function', 2, None, '___sec31'),
|
||
('Example: binary classification problem', 2, None, '___sec32'),
|
||
('The Softmax function', 2, None, '___sec33'),
|
||
('Developing a code for doing neural networks with back '
|
||
'propagation',
|
||
2,
|
||
None,
|
||
'___sec34'),
|
||
('Collect and pre-process data', 2, None, '___sec35'),
|
||
('Train and test datasets', 2, None, '___sec36'),
|
||
('Define model and architecture', 2, None, '___sec37'),
|
||
('Layers', 2, None, '___sec38'),
|
||
('Weights and biases', 2, None, '___sec39'),
|
||
('Feed-forward pass', 2, None, '___sec40'),
|
||
('Matrix multiplications', 2, None, '___sec41'),
|
||
('Choose cost function and optimizer', 2, None, '___sec42'),
|
||
('Optimizing the cost function', 2, None, '___sec43'),
|
||
('Regularization', 2, None, '___sec44'),
|
||
('Matrix multiplication', 2, None, '___sec45'),
|
||
('Improving performance', 2, None, '___sec46'),
|
||
('Full object-oriented implementation', 2, None, '___sec47'),
|
||
('Evaluate model performance on test data', 2, None, '___sec48'),
|
||
('Adjust hyperparameters', 2, None, '___sec49'),
|
||
('Visualization', 2, None, '___sec50'),
|
||
('scikit-learn implementation', 2, None, '___sec51'),
|
||
('Visualization', 2, None, '___sec52'),
|
||
('Building neural networks in Tensorflow and Keras',
|
||
2,
|
||
None,
|
||
'___sec53'),
|
||
('Tensorflow', 2, None, '___sec54'),
|
||
('Collect and pre-process data', 2, None, '___sec55'),
|
||
('Using TensorFlow backend', 2, None, '___sec56'),
|
||
('Optimizing and using gradient descent', 2, None, '___sec57'),
|
||
('Using Keras', 2, None, '___sec58'),
|
||
('Which activation function should I use?', 2, None, '___sec59'),
|
||
('Is the Logistic activation function (Sigmoid) our choice?',
|
||
2,
|
||
None,
|
||
'___sec60'),
|
||
('The derivative of the Logistic funtion', 2, None, '___sec61'),
|
||
('The RELU function family', 2, None, '___sec62'),
|
||
('Which activation function should we use?', 2, None, '___sec63'),
|
||
('A top-down perspective on Neural networks',
|
||
2,
|
||
None,
|
||
'___sec64'),
|
||
('Limitations of supervised learning with deep networks',
|
||
2,
|
||
None,
|
||
'___sec65'),
|
||
('Convolutional Neural Networks (recognizing images)',
|
||
2,
|
||
None,
|
||
'___sec66'),
|
||
('Regular NNs don’t scale well to full images',
|
||
2,
|
||
None,
|
||
'___sec67'),
|
||
('3D volumes of neurons', 2, None, '___sec68'),
|
||
('Layers used to build CNNs', 2, None, '___sec69'),
|
||
('Transforming images', 2, None, '___sec70'),
|
||
('CNNs in brief', 2, None, '___sec71'),
|
||
('CNNs in more detail, building convolutional neural networks in '
|
||
'Tensorflow and Keras',
|
||
2,
|
||
None,
|
||
'___sec72'),
|
||
('Setting it up', 2, None, '___sec73'),
|
||
('The MNIST dataset again', 2, None, '___sec74'),
|
||
('Strong correlations', 2, None, '___sec75'),
|
||
('Layers of a CNN', 2, None, '___sec76'),
|
||
('Systematic reduction', 2, None, '___sec77'),
|
||
('Prerequisites: Collect and pre-process data',
|
||
2,
|
||
None,
|
||
'___sec78'),
|
||
('Importing Keras and Tensorflow', 2, None, '___sec79'),
|
||
('Using TensorFlow backend', 2, None, '___sec80'),
|
||
('Train the model', 2, None, '___sec81'),
|
||
('Visualizing the results', 2, None, '___sec82'),
|
||
('Running with Keras', 2, None, '___sec83'),
|
||
('Final part', 2, None, '___sec84'),
|
||
('Final visualization', 2, None, '___sec85'),
|
||
('Fun links', 2, None, '___sec86'),
|
||
('Applications: solving ordinary differential equations with '
|
||
'Neural Networks',
|
||
2,
|
||
None,
|
||
'___sec87'),
|
||
('Trial solution', 2, None, '___sec88'),
|
||
('More details', 2, None, '___sec89'),
|
||
('Reformulating the problem', 2, None, '___sec90'),
|
||
('Estimating errors', 2, None, '___sec91'),
|
||
('Creating a simple Deep Neural Net', 2, None, '___sec92'),
|
||
('Setting up the code, feed forward part', 2, None, '___sec93'),
|
||
('Backpropagation', 2, None, '___sec94'),
|
||
('Gradient Descent', 2, None, '___sec95'),
|
||
('More on GD and cost function', 2, None, '___sec96'),
|
||
('An implementation of a Deep Neural Network',
|
||
2,
|
||
None,
|
||
'___sec97'),
|
||
('The final parts of the code', 2, None, '___sec98'),
|
||
('And adding Back propagation', 2, None, '___sec99'),
|
||
('Solving the ODE', 2, None, '___sec100'),
|
||
('Using neural network', 2, None, '___sec101'),
|
||
('Using a deep neural network', 2, None, '___sec102'),
|
||
('Wrapping it up', 2, None, '___sec103')]}
|
||
end of tocinfo -->
|
||
|
||
<body>
|
||
|
||
|
||
|
||
<script type="text/x-mathjax-config">
|
||
MathJax.Hub.Config({
|
||
TeX: {
|
||
equationNumbers: { autoNumber: "none" },
|
||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||
}
|
||
});
|
||
</script>
|
||
<script type="text/javascript" async
|
||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||
</script>
|
||
|
||
|
||
|
||
|
||
<!-- Bootstrap navigation bar -->
|
||
<div class="navbar navbar-default navbar-fixed-top">
|
||
<div class="navbar-header">
|
||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
<a class="navbar-brand" href="NeuralNet-bs.html">Data Analysis and Machine Learning: Neural networks, from the simple perceptron to deep learning and convolutional networks</a>
|
||
</div>
|
||
|
||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||
<ul class="nav navbar-nav navbar-right">
|
||
<li class="dropdown">
|
||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||
<ul class="dropdown-menu">
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs001.html#___sec0" style="font-size: 80%;"><b>Neural networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs002.html#___sec1" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs003.html#___sec2" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs004.html#___sec3" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs005.html#___sec4" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs006.html#___sec5" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs007.html#___sec6" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs008.html#___sec7" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs009.html#___sec8" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs010.html#___sec9" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs011.html#___sec10" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs012.html#___sec11" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs013.html#___sec12" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs014.html#___sec13" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs015.html#___sec14" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs016.html#___sec15" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs017.html#___sec16" style="font-size: 80%;"> Activation functions</a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs018.html#___sec17" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs019.html#___sec18" style="font-size: 80%;"> Relevance</a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs020.html#___sec19" style="font-size: 80%;"><b>The multilayer perceptron (MLP)</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs021.html#___sec20" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs022.html#___sec21" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs023.html#___sec22" style="font-size: 80%;"><b>Definitions</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs024.html#___sec23" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs025.html#___sec24" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs026.html#___sec25" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs027.html#___sec26" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs028.html#___sec27" style="font-size: 80%;"><b>Bringing it together</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs029.html#___sec28" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs030.html#___sec29" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs031.html#___sec30" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs032.html#___sec31" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs033.html#___sec32" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs034.html#___sec33" style="font-size: 80%;"><b>The Softmax function</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs035.html#___sec34" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs036.html#___sec35" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs037.html#___sec36" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs038.html#___sec37" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs039.html#___sec38" style="font-size: 80%;"><b>Layers</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs040.html#___sec39" style="font-size: 80%;"><b>Weights and biases</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs041.html#___sec40" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs042.html#___sec41" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs043.html#___sec42" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs044.html#___sec43" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs045.html#___sec44" style="font-size: 80%;"><b>Regularization</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs046.html#___sec45" style="font-size: 80%;"><b>Matrix multiplication</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs047.html#___sec46" style="font-size: 80%;"><b>Improving performance</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs048.html#___sec47" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs049.html#___sec48" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs050.html#___sec49" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs051.html#___sec50" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs052.html#___sec51" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs053.html#___sec52" style="font-size: 80%;"><b>Visualization</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs054.html#___sec53" style="font-size: 80%;"><b>Building neural networks in Tensorflow and Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs055.html#___sec54" style="font-size: 80%;"><b>Tensorflow</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs056.html#___sec55" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs057.html#___sec56" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs058.html#___sec57" style="font-size: 80%;"><b>Optimizing and using gradient descent</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs059.html#___sec58" style="font-size: 80%;"><b>Using Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs060.html#___sec59" style="font-size: 80%;"><b>Which activation function should I use?</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs061.html#___sec60" style="font-size: 80%;"><b>Is the Logistic activation function (Sigmoid) our choice?</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs062.html#___sec61" style="font-size: 80%;"><b>The derivative of the Logistic funtion</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs063.html#___sec62" style="font-size: 80%;"><b>The RELU function family</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs064.html#___sec63" style="font-size: 80%;"><b>Which activation function should we use?</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs065.html#___sec64" style="font-size: 80%;"><b>A top-down perspective on Neural networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="#___sec65" style="font-size: 80%;"><b>Limitations of supervised learning with deep networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs067.html#___sec66" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs068.html#___sec67" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs069.html#___sec68" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs070.html#___sec69" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs071.html#___sec70" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs072.html#___sec71" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs073.html#___sec72" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs074.html#___sec73" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs075.html#___sec74" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs076.html#___sec75" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs077.html#___sec76" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs078.html#___sec77" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs079.html#___sec78" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs080.html#___sec79" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs081.html#___sec80" style="font-size: 80%;"><b>Using TensorFlow backend</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs082.html#___sec81" style="font-size: 80%;"><b>Train the model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs083.html#___sec82" style="font-size: 80%;"><b>Visualizing the results</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs084.html#___sec83" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs085.html#___sec84" style="font-size: 80%;"><b>Final part</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs086.html#___sec85" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs087.html#___sec86" style="font-size: 80%;"><b>Fun links</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs088.html#___sec87" style="font-size: 80%;"><b>Applications: solving ordinary differential equations with Neural Networks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs089.html#___sec88" style="font-size: 80%;"><b>Trial solution</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs090.html#___sec89" style="font-size: 80%;"><b>More details</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs091.html#___sec90" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs092.html#___sec91" style="font-size: 80%;"><b>Estimating errors</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs093.html#___sec92" style="font-size: 80%;"><b>Creating a simple Deep Neural Net</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs094.html#___sec93" style="font-size: 80%;"><b>Setting up the code, feed forward part</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs095.html#___sec94" style="font-size: 80%;"><b>Backpropagation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs096.html#___sec95" style="font-size: 80%;"><b>Gradient Descent</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs097.html#___sec96" style="font-size: 80%;"><b>More on GD and cost function</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs098.html#___sec97" style="font-size: 80%;"><b>An implementation of a Deep Neural Network</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs099.html#___sec98" style="font-size: 80%;"><b>The final parts of the code</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs100.html#___sec99" style="font-size: 80%;"><b>And adding Back propagation</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs101.html#___sec100" style="font-size: 80%;"><b>Solving the ODE</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs102.html#___sec101" style="font-size: 80%;"><b>Using neural network</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs103.html#___sec102" style="font-size: 80%;"><b>Using a deep neural network</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._NeuralNet-bs104.html#___sec103" style="font-size: 80%;"><b>Wrapping it up</b></a></li>
|
||
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div> <!-- end of navigation bar -->
|
||
|
||
<div class="container">
|
||
|
||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||
|
||
<a name="part0066"></a>
|
||
<!-- !split -->
|
||
|
||
<h2 id="___sec65" class="anchor">Limitations of supervised learning with deep networks </h2>
|
||
|
||
<p>
|
||
Like all statistical methods, supervised learning using neural
|
||
networks has important limitations. This is especially important when
|
||
one seeks to apply these methods, especially to physics problems. Like
|
||
all tools, DNNs are not a universal solution. Often, the same or
|
||
better performance on a task can be achieved by using a few
|
||
hand-engineered features (or even a collection of random
|
||
features).
|
||
|
||
<p>
|
||
Here we list some of the important limitations of supervised neural network based models.
|
||
|
||
<ul>
|
||
<li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
|
||
<li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
|
||
<li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e. some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
|
||
<li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
|
||
</ul>
|
||
|
||
Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.
|
||
|
||
<p>
|
||
<p>
|
||
<!-- navigation buttons at the bottom of the page -->
|
||
<ul class="pagination">
|
||
<li><a href="._NeuralNet-bs065.html">«</a></li>
|
||
<li><a href="._NeuralNet-bs000.html">1</a></li>
|
||
<li><a href="">...</a></li>
|
||
<li><a href="._NeuralNet-bs058.html">59</a></li>
|
||
<li><a href="._NeuralNet-bs059.html">60</a></li>
|
||
<li><a href="._NeuralNet-bs060.html">61</a></li>
|
||
<li><a href="._NeuralNet-bs061.html">62</a></li>
|
||
<li><a href="._NeuralNet-bs062.html">63</a></li>
|
||
<li><a href="._NeuralNet-bs063.html">64</a></li>
|
||
<li><a href="._NeuralNet-bs064.html">65</a></li>
|
||
<li><a href="._NeuralNet-bs065.html">66</a></li>
|
||
<li class="active"><a href="._NeuralNet-bs066.html">67</a></li>
|
||
<li><a href="._NeuralNet-bs067.html">68</a></li>
|
||
<li><a href="._NeuralNet-bs068.html">69</a></li>
|
||
<li><a href="._NeuralNet-bs069.html">70</a></li>
|
||
<li><a href="._NeuralNet-bs070.html">71</a></li>
|
||
<li><a href="._NeuralNet-bs071.html">72</a></li>
|
||
<li><a href="._NeuralNet-bs072.html">73</a></li>
|
||
<li><a href="._NeuralNet-bs073.html">74</a></li>
|
||
<li><a href="._NeuralNet-bs074.html">75</a></li>
|
||
<li><a href="._NeuralNet-bs075.html">76</a></li>
|
||
<li><a href="">...</a></li>
|
||
<li><a href="._NeuralNet-bs104.html">105</a></li>
|
||
<li><a href="._NeuralNet-bs067.html">»</a></li>
|
||
</ul>
|
||
<!-- ------------------- end of main content --------------- -->
|
||
|
||
</div> <!-- end container -->
|
||
<!-- include javascript, jQuery *first* -->
|
||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||
|
||
<!-- Bootstrap footer
|
||
<footer>
|
||
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
||
</footer>
|
||
-->
|
||
|
||
|
||
<center style="font-size:80%">
|
||
<!-- copyright only on the titlepage -->
|
||
</center>
|
||
|
||
|
||
</body>
|
||
</html>
|
||
|
||
|