Files
FYS-STK4155/doc/pub/week41/html/week41-bs.html
T
Morten Hjorth-Jensen c5759046bd typo correction
2021-10-15 15:41:10 +02:00

423 lines
24 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!--
Automatically generated HTML file from DocOnce source
(https://github.com/doconce/doconce/)
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
<!-- Bootstrap style: bootstrap -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
('Videos on Neural Networks',
2,
None,
'videos-on-neural-networks'),
('Setting up the Back propagation algorithm',
2,
None,
'setting-up-the-back-propagation-algorithm'),
('Setting up a Multi-layer perceptron model for classification',
2,
None,
'setting-up-a-multi-layer-perceptron-model-for-classification'),
('Defining the cost function',
2,
None,
'defining-the-cost-function'),
('Example: binary classification problem',
2,
None,
'example-binary-classification-problem'),
('The Softmax function', 2, None, 'the-softmax-function'),
('Developing a code for doing neural networks with back '
'propagation',
2,
None,
'developing-a-code-for-doing-neural-networks-with-back-propagation'),
('Collect and pre-process data',
2,
None,
'collect-and-pre-process-data'),
('Train and test datasets', 2, None, 'train-and-test-datasets'),
('Define model and architecture',
2,
None,
'define-model-and-architecture'),
('Layers', 2, None, 'layers'),
('Weights and biases', 2, None, 'weights-and-biases'),
('Feed-forward pass', 2, None, 'feed-forward-pass'),
('Matrix multiplications', 2, None, 'matrix-multiplications'),
('Choose cost function and optimizer',
2,
None,
'choose-cost-function-and-optimizer'),
('Optimizing the cost function',
2,
None,
'optimizing-the-cost-function'),
('Regularization', 2, None, 'regularization'),
('Matrix multiplication', 2, None, 'matrix-multiplication'),
('Improving performance', 2, None, 'improving-performance'),
('Full object-oriented implementation',
2,
None,
'full-object-oriented-implementation'),
('Evaluate model performance on test data',
2,
None,
'evaluate-model-performance-on-test-data'),
('Adjust hyperparameters', 2, None, 'adjust-hyperparameters'),
('Visualization', 2, None, 'visualization'),
('scikit-learn implementation',
2,
None,
'scikit-learn-implementation'),
('Visualization', 2, None, 'visualization'),
('Testing our code for the XOR, OR and AND gates',
2,
None,
'testing-our-code-for-the-xor-or-and-and-gates'),
('The AND and XOR Gates', 2, None, 'the-and-and-xor-gates'),
('Representing the Data Sets',
2,
None,
'representing-the-data-sets'),
('Setting up the Neural Network',
2,
None,
'setting-up-the-neural-network'),
('The Code using Scikit-Learn',
2,
None,
'the-code-using-scikit-learn'),
('Building neural networks in Tensorflow and Keras',
2,
None,
'building-neural-networks-in-tensorflow-and-keras'),
('Tensorflow', 2, None, 'tensorflow'),
('Using Keras', 2, None, 'using-keras'),
('Collect and pre-process data',
2,
None,
'collect-and-pre-process-data'),
('The Breast Cancer Data, now with Keras',
2,
None,
'the-breast-cancer-data-now-with-keras'),
('The Mathematics of Neural Networks',
2,
None,
'the-mathematics-of-neural-networks'),
('Fine-tuning neural network hyperparameters',
2,
None,
'fine-tuning-neural-network-hyperparameters'),
('Hidden layers', 2, None, 'hidden-layers'),
('Which activation function should I use?',
2,
None,
'which-activation-function-should-i-use'),
('Is the Logistic activation function (Sigmoid) our choice?',
2,
None,
'is-the-logistic-activation-function-sigmoid-our-choice'),
('The derivative of the Logistic funtion',
2,
None,
'the-derivative-of-the-logistic-funtion'),
('The RELU function family', 2, None, 'the-relu-function-family'),
('Which activation function should we use?',
2,
None,
'which-activation-function-should-we-use'),
('More on activation functions, output layers',
2,
None,
'more-on-activation-functions-output-layers'),
('Batch Normalization', 2, None, 'batch-normalization'),
('Dropout', 2, None, 'dropout'),
('Gradient Clipping', 2, None, 'gradient-clipping'),
('A very nice website on Neural Networks',
2,
None,
'a-very-nice-website-on-neural-networks'),
('A top-down perspective on Neural networks',
2,
None,
'a-top-down-perspective-on-neural-networks'),
('Limitations of supervised learning with deep networks',
2,
None,
'limitations-of-supervised-learning-with-deep-networks'),
('Overarching Views, a personal note',
2,
None,
'overarching-views-a-personal-note'),
('From a Spherical Cow to a real one',
2,
None,
'from-a-spherical-cow-to-a-real-one'),
('Convolutional Neural Networks (recognizing images)',
2,
None,
'convolutional-neural-networks-recognizing-images'),
('Regular NNs dont scale well to full images',
2,
None,
'regular-nns-don-t-scale-well-to-full-images'),
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
('Layers used to build CNNs',
2,
None,
'layers-used-to-build-cnns'),
('Transforming images', 2, None, 'transforming-images'),
('CNNs in brief', 2, None, 'cnns-in-brief'),
('CNNs in more detail, building convolutional neural networks in '
'Tensorflow and Keras',
2,
None,
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
('Setting it up', 2, None, 'setting-it-up'),
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
('Strong correlations', 2, None, 'strong-correlations'),
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
('Systematic reduction', 2, None, 'systematic-reduction'),
('Prerequisites: Collect and pre-process data',
2,
None,
'prerequisites-collect-and-pre-process-data'),
('Importing Keras and Tensorflow',
2,
None,
'importing-keras-and-tensorflow'),
('Running with Keras', 2, None, 'running-with-keras'),
('Final part', 2, None, 'final-part'),
('Final visualization', 2, None, 'final-visualization'),
('Fun links', 2, None, 'fun-links')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs003.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs005.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs006.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs007.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs008.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs035.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs010.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs011.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs012.html#layers" style="font-size: 80%;">Layers</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs013.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs014.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs015.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs016.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs017.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs018.html#regularization" style="font-size: 80%;">Regularization</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs019.html#matrix-multiplication" style="font-size: 80%;">Matrix multiplication</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs020.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs021.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs022.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs023.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs026.html#visualization" style="font-size: 80%;">Visualization</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs025.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs026.html#visualization" style="font-size: 80%;">Visualization</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs027.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs028.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs029.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs030.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs031.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs032.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs033.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs034.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs035.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs036.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs037.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid) our choice?</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs052.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal note</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs053.html#from-a-spherical-cow-to-a-real-one" style="font-size: 80%;">From a Spherical Cow to a real one</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs054.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs055.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs dont scale well to full images</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs056.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs057.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs058.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs059.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs060.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs061.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs062.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs063.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs064.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs065.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs066.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs067.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs068.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs069.html#final-part" style="font-size: 80%;">Final part</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs070.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
<!-- navigation toc: --> <li><a href="._week41-bs071.html#fun-links" style="font-size: 80%;">Fun links</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0000"></a>
<!-- ------------------- main content ---------------------- -->
<div class="jumbotron">
<center><h1>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1></center> <!-- document title -->
<p>
<!-- author(s): Morten Hjorth-Jensen -->
<center>
<b>Morten Hjorth-Jensen</b> [1, 2]
</center>
<p>
<!-- institution(s) -->
<center>[1] <b>Department of Physics, University of Oslo</b></center>
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Oct 15, 2021</h4></center> <!-- date -->
<br>
<p>
<p><a href="._week41-bs001.html" class="btn btn-primary btn-lg">Read &raquo;</a></p>
</div> <!-- end jumbotron -->
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li class="active"><a href="._week41-bs000.html">1</a></li>
<li><a href="._week41-bs001.html">2</a></li>
<li><a href="._week41-bs002.html">3</a></li>
<li><a href="._week41-bs003.html">4</a></li>
<li><a href="._week41-bs004.html">5</a></li>
<li><a href="._week41-bs005.html">6</a></li>
<li><a href="._week41-bs006.html">7</a></li>
<li><a href="._week41-bs007.html">8</a></li>
<li><a href="._week41-bs008.html">9</a></li>
<li><a href="._week41-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._week41-bs071.html">72</a></li>
<li><a href="._week41-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
</body>
</html>