From f055133e352a5625dbe8b5cdcf8d10a69fd32eb7 Mon Sep 17 00:00:00 2001
From: Morten Hjorth-Jensen <morten.hjorth-jensen@fys.uio.no>
Date: Mon, 9 Oct 2023 06:47:15 +0200
Subject: [PATCH] update week 41

---
 doc/pub/week41/html/._week41-bs000.html      |  285 +-
 doc/pub/week41/html/._week41-bs001.html      |  320 +-
 doc/pub/week41/html/._week41-bs002.html      |  291 +-
 doc/pub/week41/html/._week41-bs003.html      |  296 +-
 doc/pub/week41/html/._week41-bs004.html      |  391 ++-
 doc/pub/week41/html/._week41-bs005.html      |  331 +-
 doc/pub/week41/html/._week41-bs006.html      |  339 +-
 doc/pub/week41/html/._week41-bs007.html      |  341 +-
 doc/pub/week41/html/._week41-bs008.html      |  312 +-
 doc/pub/week41/html/._week41-bs009.html      |  307 +-
 doc/pub/week41/html/._week41-bs010.html      |  396 +--
 doc/pub/week41/html/._week41-bs011.html      |  353 +-
 doc/pub/week41/html/._week41-bs012.html      |  330 +-
 doc/pub/week41/html/._week41-bs013.html      |  368 +-
 doc/pub/week41/html/._week41-bs014.html      |  355 +-
 doc/pub/week41/html/._week41-bs015.html      |  328 +-
 doc/pub/week41/html/._week41-bs016.html      |  383 +--
 doc/pub/week41/html/._week41-bs017.html      |  337 +-
 doc/pub/week41/html/._week41-bs018.html      |  331 +-
 doc/pub/week41/html/._week41-bs019.html      |  319 +-
 doc/pub/week41/html/._week41-bs020.html      |  419 +--
 doc/pub/week41/html/._week41-bs021.html      |  323 +-
 doc/pub/week41/html/._week41-bs022.html      |  424 +--
 doc/pub/week41/html/._week41-bs023.html      |  341 +-
 doc/pub/week41/html/._week41-bs024.html      |  341 +-
 doc/pub/week41/html/._week41-bs025.html      |  382 ++-
 doc/pub/week41/html/._week41-bs026.html      |  357 +-
 doc/pub/week41/html/._week41-bs027.html      |  358 +-
 doc/pub/week41/html/._week41-bs028.html      |  327 +-
 doc/pub/week41/html/._week41-bs029.html      |  347 +-
 doc/pub/week41/html/._week41-bs030.html      |  304 +-
 doc/pub/week41/html/._week41-bs031.html      |  383 +--
 doc/pub/week41/html/._week41-bs032.html      |  392 +--
 doc/pub/week41/html/._week41-bs033.html      |  303 +-
 doc/pub/week41/html/._week41-bs034.html      |  431 +--
 doc/pub/week41/html/._week41-bs035.html      |  333 +-
 doc/pub/week41/html/._week41-bs036.html      |  566 ++--
 doc/pub/week41/html/._week41-bs037.html      |  530 ++-
 doc/pub/week41/html/._week41-bs038.html      |  369 +-
 doc/pub/week41/html/._week41-bs039.html      |  328 +-
 doc/pub/week41/html/._week41-bs040.html      |  348 +-
 doc/pub/week41/html/._week41-bs041.html      |  347 +-
 doc/pub/week41/html/._week41-bs042.html      |  327 +-
 doc/pub/week41/html/._week41-bs043.html      |  319 +-
 doc/pub/week41/html/._week41-bs044.html      |  403 ++-
 doc/pub/week41/html/._week41-bs045.html      |  353 +-
 doc/pub/week41/html/._week41-bs046.html      |  334 +-
 doc/pub/week41/html/._week41-bs047.html      |  332 +-
 doc/pub/week41/html/._week41-bs048.html      |  345 +-
 doc/pub/week41/html/._week41-bs049.html      |  312 +-
 doc/pub/week41/html/._week41-bs050.html      |  407 ++-
 doc/pub/week41/html/._week41-bs051.html      |  331 +-
 doc/pub/week41/html/._week41-bs052.html      |  513 ++-
 doc/pub/week41/html/._week41-bs053.html      |  510 ++-
 doc/pub/week41/html/._week41-bs054.html      |  594 ++--
 doc/pub/week41/html/._week41-bs055.html      |  501 +--
 doc/pub/week41/html/._week41-bs056.html      |  611 ++--
 doc/pub/week41/html/._week41-bs057.html      |  533 ++-
 doc/pub/week41/html/._week41-bs058.html      |  532 ++-
 doc/pub/week41/html/._week41-bs059.html      |  532 ++-
 doc/pub/week41/html/._week41-bs060.html      |  535 ++-
 doc/pub/week41/html/._week41-bs061.html      |  534 ++-
 doc/pub/week41/html/._week41-bs062.html      |  500 ++-
 doc/pub/week41/html/._week41-bs063.html      |  523 ++-
 doc/pub/week41/html/._week41-bs064.html      |  502 +--
 doc/pub/week41/html/._week41-bs065.html      |  576 ++--
 doc/pub/week41/html/._week41-bs066.html      |  569 ++--
 doc/pub/week41/html/week41-bs.html           |  285 +-
 doc/pub/week41/html/week41-reveal.html       | 2224 +++++++-----
 doc/pub/week41/html/week41-solarized.html    | 2230 ++++++------
 doc/pub/week41/html/week41.html              | 2230 ++++++------
 doc/pub/week41/ipynb/ipynb-week41-src.tar.gz |  Bin 191 -> 34323 bytes
 doc/pub/week41/ipynb/week41.ipynb            | 3174 ++++++++++++------
 doc/src/week41/figures/nns.png               |  Bin 0 -> 33457 bytes
 doc/src/week41/week41.do.txt                 | 1765 ++++++----
 75 files changed, 20866 insertions(+), 17536 deletions(-)
 create mode 100644 doc/src/week41/figures/nns.png

diff --git a/doc/pub/week41/html/._week41-bs000.html b/doc/pub/week41/html/._week41-bs000.html
index 8b355d07a..9aad39fba 100644
--- a/doc/pub/week41/html/._week41-bs000.html
+++ b/doc/pub/week41/html/._week41-bs000.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -276,7 +329,7 @@ MathJax.Hub.Config({
 <!-- ------------------- main content ---------------------- -->
 <div class="jumbotron">
 <center>
-<h1>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1>
+<h1>Week 41 Neural networks and constructing a neural network code</h1>
 </center>  <!-- document title -->
 
 <!-- author(s): Morten Hjorth-Jensen -->
@@ -317,7 +370,7 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs008.html">9</a></li>
   <li><a href="._week41-bs009.html">10</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs001.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs001.html b/doc/pub/week41/html/._week41-bs001.html
index 18cb326c8..9c331f568 100644
--- a/doc/pub/week41/html/._week41-bs001.html
+++ b/doc/pub/week41/html/._week41-bs001.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -276,14 +329,39 @@ MathJax.Hub.Config({
 <!-- !split -->
 <h2 id="plan-for-week-41" class="anchor">Plan for week 41 </h2>
 
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
 <ul>
-<li> Building our own Feed-forward Neural Network and discussion of project 2.</li>
-<li> Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks.</li> 
+  <li> Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation</li>
+  <li> One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code</li>
+  <li> Discussion of project 2</li>
+  <li> Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent</li>
 </ul>
-<p>"
-Reading suggestions: These notes, <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_self">Aurelien Geron's chapters 10-11</a>.
-For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read.
-</p>
+</div>
+</div>
+  
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<ul>
+  <li> Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.</li>
+  <li> Building our own Feed-forward Neural Network</li>
+  <li> Readings and Videos:</li>
+<ul>
+    <li> These lecture notes</li>
+    <li> For neural networks we recommend Goodfellow et al chapter 6.</li>
+    <li> <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_self">Neural Networks demystified</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex" target="_self">Building Neural Networks from scratch</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_self">Video on Neural Networks</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_self">Video on the back propagation algorithm</a></li>
+</ul>
+</ul>
+<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_self"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+</div>
+</div>
+
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -301,7 +379,7 @@ For a more in depth discussion on  neural networks we recommend Goodfellow et al
   <li><a href="._week41-bs009.html">10</a></li>
   <li><a href="._week41-bs010.html">11</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs002.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs002.html b/doc/pub/week41/html/._week41-bs002.html
index 278ccf88b..7eba29031 100644
--- a/doc/pub/week41/html/._week41-bs002.html
+++ b/doc/pub/week41/html/._week41-bs002.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,13 +327,7 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0002"></a>
 <!-- !split -->
-<h2 id="videos-on-neural-networks" class="anchor">Videos on Neural Networks </h2>
-
-<ul>
-<li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_self">Video on Neural Networks</a></li>
-<li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_self">Video on the back propagation algorithm</a></li>
-</ul>
-<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_self"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+<h2 id="lecture-thursday-october-12" class="anchor">Lecture Thursday October 12 </h2>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -299,7 +346,7 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs010.html">11</a></li>
   <li><a href="._week41-bs011.html">12</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs003.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs003.html b/doc/pub/week41/html/._week41-bs003.html
index 4f78321bc..fcd766f06 100644
--- a/doc/pub/week41/html/._week41-bs003.html
+++ b/doc/pub/week41/html/._week41-bs003.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,12 +327,15 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0003"></a>
 <!-- !split -->
-<h2 id="review-of-the-back-propagation-algorithm" class="anchor">Review of the back propagation algorithm </h2>
+<h2 id="introduction-to-neural-networks" class="anchor">Introduction to Neural networks </h2>
 
-<p>During the last lecture we discussed in detail the back propagation
-algorithm. This algorithm is based on a repeated application of the
-chain rule. Let us bring back the basic equation and at the same time
-link this with the basic mathematics of automatic differentiation.
+<p>Artificial neural networks are computational systems that can learn to
+perform tasks by considering examples, generally without being
+programmed with any task-specific rules. It is supposed to mimic a
+biological system, wherein neurons interact by sending signals in the
+form of mathematical functions between layers. All layers can contain
+an arbitrary number of neurons, and each connection is represented by
+a weight variable.
 </p>
 
 <p>
@@ -300,7 +356,7 @@ link this with the basic mathematics of automatic differentiation.
   <li><a href="._week41-bs011.html">12</a></li>
   <li><a href="._week41-bs012.html">13</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs004.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs004.html b/doc/pub/week41/html/._week41-bs004.html
index a61e3e056..39e92afcd 100644
--- a/doc/pub/week41/html/._week41-bs004.html
+++ b/doc/pub/week41/html/._week41-bs004.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,73 +327,63 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0004"></a>
 <!-- !split -->
-<h2 id="setting-up-the-back-propagation-algorithm" class="anchor">Setting up the Back propagation algorithm </h2>
+<h2 id="artificial-neurons" class="anchor">Artificial neurons  </h2>
 
-<p>The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
-
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
-\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
-the pertinent outputs \( \boldsymbol{a}^1 \).
+<p>The field of artificial neural networks has a long history of
+development, and is closely connected with the advancement of computer
+science and computers in general. A model of artificial neurons was
+first developed by McCulloch and Pitts in 1943 to study signal
+processing in the brain and has later been refined by others. The
+general idea is to mimic neural networks in the human brain, which is
+composed of billions of neurons that communicate with each other by
+sending electrical signals.  Each neuron accumulates its incoming
+signals, which must exceed an activation threshold to yield an
+output. If the threshold is not overcome, the neuron remains inactive,
+i.e. has zero output.
 </p>
-</div>
-</div>
 
+<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
 
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>Secondly, we perform then the feed forward till we reach the output
-layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
-activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
-\( l=2,3,\dots,L \).
+$$
+\begin{equation}
+ y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
+\tag{1}
+\end{equation}
+$$
+
+<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
+a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
 </p>
-</div>
-</div>
 
+<p>Conceptually, it is helpful to divide neural networks into four
+categories:
+</p>
+<ol>
+<li> general purpose neural networks for supervised learning,</li>
+<li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
+<li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
+<li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
+</ol>
+<p>In natural science, DNNs and CNNs have already found numerous
+applications. In statistical physics, they have been applied to detect
+phase transitions in 2D Ising and Potts models, lattice gauge
+theories, and different phases of polymers, or solving the
+Navier-Stokes equation in weather forecasting.  Deep learning has also
+found interesting applications in quantum physics. Various quantum
+phase transitions can be detected and studied using DNNs and CNNs,
+topological phases, and even non-equilibrium many-body
+localization. Representing quantum states as DNNs quantum state
+tomography are among some of the impressive achievements to reveal the
+potential of DNNs to facilitate the study of quantum systems.
+</p>
 
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
-$$
-\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
-$$
-</div>
-</div>
+<p>In quantum information theory, it has been shown that one can perform
+gate decompositions with the help of neural. 
+</p>
 
-
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
-$$
-\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
-$$
-</div>
-</div>
-
-
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
-$$
-w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
-$$
-
-
-$$
-b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
-$$
-</div>
-</div>
-
-
-<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
-Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+<p>The applications are not limited to the natural sciences. There is a
+plethora of applications in essentially all disciplines, from the
+humanities to life science and medicine.
 </p>
 
 <p>
@@ -362,7 +405,7 @@ Here it is convenient to use stochastic gradient descent (see the examples below
   <li><a href="._week41-bs012.html">13</a></li>
   <li><a href="._week41-bs013.html">14</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs005.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs005.html b/doc/pub/week41/html/._week41-bs005.html
index b3e17459d..fe52b55c9 100644
--- a/doc/pub/week41/html/._week41-bs005.html
+++ b/doc/pub/week41/html/._week41-bs005.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,42 +326,30 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0005"></a>
-<!-- !split  -->
-<h2 id="setting-up-a-multi-layer-perceptron-model-for-classification" class="anchor">Setting up a Multi-layer perceptron model for classification  </h2>
+<!-- !split -->
+<h2 id="neural-network-types" class="anchor">Neural network types </h2>
 
-<p>We are now gong to develop an example based on the MNIST data
-base. This is a classification problem and we need to use our
-cross-entropy function we discussed in connection with logistic
-regression. The cross-entropy defines our cost function for the
-classificaton problems with neural networks.
+<p>An artificial neural network (ANN), is a computational model that
+consists of layers of connected neurons, or nodes or units.  We will
+refer to these interchangeably as units or nodes, and sometimes as
+neurons.
 </p>
 
-<p>In binary classification with two classes \( (0, 1) \) we define the
-logistic/sigmoid function as the probability that a particular input
-is in class \( 0 \) or \( 1 \).  This is possible because the logistic
-function takes any input from the real numbers and inputs a number
-between 0 and 1, and can therefore be interpreted as a probability. It
-also has other nice properties, such as a derivative that is simple to
-calculate.
+<p>It is supposed to mimic a biological nervous system by letting each
+neuron interact with other neurons by sending signals in the form of
+mathematical functions between layers.  A wide variety of different
+ANNs have been developed, but most of them consist of an input layer,
+an output layer and eventual layers in-between, called <em>hidden
+layers</em>. All layers can contain an arbitrary number of nodes, and each
+connection between two nodes is associated with a weight variable.
 </p>
 
-<p>For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
-is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
-represents our activation values \( z \). We have
+<p>Neural networks (also called neural nets) are neural-inspired
+nonlinear models for supervised learning.  As we will see, neural nets
+can be viewed as natural, more powerful extensions of supervised
+learning methods such as linear and logistic regression and soft-max
+methods we discussed earlier.
 </p>
-$$
-P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp{(- \boldsymbol{x}})} ,
-$$
-
-<p>and</p>
-$$
-P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) ,
-$$
-
-<p>where \( y \in \{0, 1\} \)  and \( \boldsymbol{\theta} \) represents the weights and biases
-of our network.
-</p>
-
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -330,7 +371,7 @@ of our network.
   <li><a href="._week41-bs013.html">14</a></li>
   <li><a href="._week41-bs014.html">15</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs006.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs006.html b/doc/pub/week41/html/._week41-bs006.html
index 74775058b..20e02b27d 100644
--- a/doc/pub/week41/html/._week41-bs006.html
+++ b/doc/pub/week41/html/._week41-bs006.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,57 +327,21 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0006"></a>
 <!-- !split -->
-<h2 id="defining-the-cost-function" class="anchor">Defining the cost function </h2>
+<h2 id="feed-forward-neural-networks" class="anchor">Feed-forward neural networks </h2>
 
-<p>Our cost function is given as (see the Logistic regression lectures)</p>
-$$
-\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n
-y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) .
-$$
-
-<p>This last equality means that we can interpret our <em>cost</em> function as a sum over the <em>loss</em> function
-for each point in the dataset \( \mathcal{L}_i(\boldsymbol{\theta}) \).  
-The negative sign is just so that we can think about our algorithm as minimizing a positive number, rather
-than maximizing a negative number.  
+<p>The feed-forward neural network (FFNN) was the first and simplest type
+of ANNs that were devised. In this network, the information moves in
+only one direction: forward through the layers.
 </p>
 
-<p>In <em>multiclass</em> classification it is common to treat each integer label as a so called <em>one-hot</em> vector:  </p>
-
-<p>\( y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) , \) and</p>
-
-\( y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) , \)
-
-<p>i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset (numbers from \( 0 \) to \( 9 \))..  </p>
-
-<p>If \( \boldsymbol{x}_i \) is the \( i \)-th input (image), \( y_{ic} \) refers to the \( c \)-th component of the \( i \)-th
-output vector \( \boldsymbol{y}_i \).  
-The probability of \( \boldsymbol{x}_i \) being in class \( c \) will be given by the softmax function:  
+<p>Nodes are represented by circles, while the arrows display the
+connections between the nodes, including the direction of information
+flow. Additionally, each arrow corresponds to a weight variable
+(figure to come).  We observe that each node in a layer is connected
+to <em>all</em> nodes in the subsequent layer, making this a so-called
+<em>fully-connected</em> FFNN.
 </p>
 
-$$
-P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}}
-{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} ,
-$$
-
-<p>which reduces to the logistic function in the binary case.  
-The likelihood of this \( C \)-class classifier
-is now given as:  
-</p>
-
-$$
-P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} .
-$$
-
-<p>Again we take the negative log-likelihood to define our cost function:  </p>
-
-$$
-\mathcal{C}(\boldsymbol{\theta}) = - \log{P(\mathcal{D} \mid \boldsymbol{\theta})}.
-$$
-
-<p>See the logistic regression lectures for a full definition of the cost function.</p>
-
-<p>The back propagation equations need now only a small change, namely the definition of a new cost function. We are thus ready to use the same equations as before!</p>
-
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -346,7 +363,7 @@ $$
   <li><a href="._week41-bs014.html">15</a></li>
   <li><a href="._week41-bs015.html">16</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs007.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs007.html b/doc/pub/week41/html/._week41-bs007.html
index 1f5028703..72fe5ce66 100644
--- a/doc/pub/week41/html/._week41-bs007.html
+++ b/doc/pub/week41/html/._week41-bs007.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,50 +327,28 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0007"></a>
 <!-- !split -->
-<h2 id="example-binary-classification-problem" class="anchor">Example: binary classification problem </h2>
+<h2 id="convolutional-neural-network" class="anchor">Convolutional Neural Network </h2>
 
-<p>As an example of the above, relevant for project 2 as well, let us consider a binary class. As discussed in our logistic regression lectures, we defined a cost function in terms of the parameters \( \beta \) as</p>
-$$
-\mathcal{C}(\boldsymbol{\beta}) = - \sum_{i=1}^n \left(y_i\log{p(y_i \vert x_i,\boldsymbol{\beta})}+(1-y_i)\log{1-p(y_i \vert x_i,\boldsymbol{\beta})}\right),
-$$
-
-<p>where we had defined the logistic (sigmoid) function</p>
-$$
-p(y_i =1\vert x_i,\boldsymbol{\beta})=\frac{\exp{(\beta_0+\beta_1 x_i)}}{1+\exp{(\beta_0+\beta_1 x_i)}},
-$$
-
-<p>and</p>
-$$
-p(y_i =0\vert x_i,\boldsymbol{\beta})=1-p(y_i =1\vert x_i,\boldsymbol{\beta}).
-$$
-
-<p>The parameters \( \boldsymbol{\beta} \) were defined using a minimization method like gradient descent or Newton-Raphson's method. </p>
-
-<p>Now we replace \( x_i \) with the activation \( z_i^l \) for a given layer \( l \) and the outputs as \( y_i=a_i^l=f(z_i^l) \), with \( z_i^l \) now being a function of the weights \( w_{ij}^l \) and biases \( b_i^l \). 
-We have then
+<p>A different variant of FFNNs are <em>convolutional neural networks</em>
+(CNNs), which have a connectivity pattern inspired by the animal
+visual cortex. Individual neurons in the visual cortex only respond to
+stimuli from small sub-regions of the visual field, called a receptive
+field. This makes the neurons well-suited to exploit the strong
+spatially local correlation present in natural images. The response of
+each neuron can be approximated mathematically as a convolution
+operation.  (figure to come)
 </p>
-$$
-a_i^l = y_i = \frac{\exp{(z_i^l)}}{1+\exp{(z_i^l)}},
-$$
 
-<p>with </p>
-$$
-z_i^l = \sum_{j}w_{ij}^l a_j^{l-1}+b_i^l,
-$$
-
-<p>where the superscript \( l-1 \) indicates that these are the outputs from layer \( l-1 \).
-Our cost function at the final layer \( l=L \) is now
+<p>Convolutional neural networks emulate the behaviour of neurons in the
+visual cortex by enforcing a <em>local</em> connectivity pattern between
+nodes of adjacent layers: Each node in a convolutional layer is
+connected only to a subset of the nodes in the previous layer, in
+contrast to the fully-connected FFNN.  Often, CNNs consist of several
+convolutional layers that learn local features of the input, with a
+fully-connected layer at the end, which gathers all the local data and
+produces the outputs. They have wide applications in image and video
+recognition.
 </p>
-$$
-\mathcal{C}(\boldsymbol{W}) = - \sum_{i=1}^n \left(t_i\log{a_i^L}+(1-t_i)\log{(1-a_i^L)}\right),
-$$
-
-<p>where we have defined the targets \( t_i \). The derivatives of the cost function with respect to the output \( a_i^L \) are then easily calculated and we get</p>
-$$
-\frac{\partial \mathcal{C}(\boldsymbol{W})}{\partial a_i^L} = \frac{a_i^L-t_i}{a_i^L(1-a_i^L)}. 
-$$
-
-<p>In case we use another activation function than the logistic one, we need to evaluate other derivatives. </p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -341,7 +372,7 @@ $$
   <li><a href="._week41-bs015.html">16</a></li>
   <li><a href="._week41-bs016.html">17</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs008.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs008.html b/doc/pub/week41/html/._week41-bs008.html
index b6444da91..177c8c233 100644
--- a/doc/pub/week41/html/._week41-bs008.html
+++ b/doc/pub/week41/html/._week41-bs008.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,24 +327,19 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0008"></a>
 <!-- !split -->
-<h2 id="the-softmax-function" class="anchor">The Softmax function </h2>
-<p>In case we employ the more general case given by the Softmax equation, we need to evaluate the derivative of the activation function with respect to the activation \( z_i^l \), that is we need</p>
-$$
-\frac{\partial f(z_i^l)}{\partial w_{jk}^l} =
-\frac{\partial f(z_i^l)}{\partial z_j^l} \frac{\partial z_j^l}{\partial w_{jk}^l}= \frac{\partial f(z_i^l)}{\partial z_j^l}a_k^{l-1}.
-$$
+<h2 id="recurrent-neural-networks" class="anchor">Recurrent neural networks </h2>
 
-<p>For the Softmax function we have</p>
-$$
-f(z_i^l) = \frac{\exp{(z_i^l)}}{\sum_{m=1}^K\exp{(z_m^l)}}.
-$$
-
-<p>Its derivative with respect to \( z_j^l \) gives </p>
-$$
-\frac{\partial f(z_i^l)}{\partial z_j^l}= f(z_i^l)\left(\delta_{ij}-f(z_j^l)\right), 
-$$
-
-<p>which in case of the simply binary model reduces to  having \( i=j \). </p>
+<p>So far we have only mentioned ANNs where information flows in one
+direction: forward. <em>Recurrent neural networks</em> on the other hand,
+have connections between nodes that form directed <em>cycles</em>. This
+creates a form of internal memory which are able to capture
+information on what has been calculated before; the output is
+dependent on the previous computations. Recurrent NNs make use of
+sequential information by performing the same task for every element
+in a sequence, where each element depends on previous elements. An
+example of such information is sentences, making recurrent NNs
+especially well-suited for handwriting and speech recognition.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -316,7 +364,7 @@ $$
   <li><a href="._week41-bs016.html">17</a></li>
   <li><a href="._week41-bs017.html">18</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs009.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs009.html b/doc/pub/week41/html/._week41-bs009.html
index ef053f774..000140aad 100644
--- a/doc/pub/week41/html/._week41-bs009.html
+++ b/doc/pub/week41/html/._week41-bs009.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,19 +326,21 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0009"></a>
-<!-- !split  -->
-<h2 id="developing-a-code-for-doing-neural-networks-with-back-propagation" class="anchor">Developing a code for doing neural networks with back propagation </h2>
+<!-- !split -->
+<h2 id="other-types-of-networks" class="anchor">Other types of networks </h2>
 
-<p>One can identify a set of key steps when using neural networks to solve supervised learning problems:  </p>
+<p>There are many other kinds of ANNs that have been developed. One type
+that is specifically designed for interpolation in multidimensional
+space is the radial basis function (RBF) network. RBFs are typically
+made up of three layers: an input layer, a hidden layer with
+non-linear radial symmetric activation functions and a linear output
+layer (''linear'' here means that each node in the output layer has a
+linear activation function). The layers are normally fully-connected
+and there are no cycles, thus RBFs can be viewed as a type of
+fully-connected FFNN. They are however usually treated as a separate
+type of NN due the unusual activation functions.
+</p>
 
-<ol>
-<li> Collect and pre-process data</li>  
-<li> Define model and architecture</li>  
-<li> Choose cost function and optimizer</li>  
-<li> Train the model</li>  
-<li> Evaluate model performance on test data</li>  
-<li> Adjust hyperparameters (if necessary, network architecture)</li>
-</ol>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -310,7 +365,7 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs017.html">18</a></li>
   <li><a href="._week41-bs018.html">19</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs010.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs010.html b/doc/pub/week41/html/._week41-bs010.html
index d7cd7daa0..181cd480f 100644
--- a/doc/pub/week41/html/._week41-bs010.html
+++ b/doc/pub/week41/html/._week41-bs010.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,116 +327,15 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0010"></a>
 <!-- !split -->
-<h2 id="collect-and-pre-process-data" class="anchor">Collect and pre-process data </h2>
+<h2 id="multilayer-perceptrons" class="anchor">Multilayer perceptrons  </h2>
 
-<p>Here we will be using the MNIST dataset, which is readily available through the <b>scikit-learn</b>
-package. You may also find it for example <a href="http://yann.lecun.com/exdb/mnist/" target="_self">here</a>.  
-The <em>MNIST</em> (Modified National Institute of Standards and Technology) database is a large database
-of handwritten digits that is commonly used for training various image processing systems.  
-The MNIST dataset consists of 70 000 images of size \( 28\times 28 \) pixels, each labeled from 0 to 9.  
-The scikit-learn dataset we will use consists of a selection of 1797 images of size \( 8\times 8 \) collected and processed from this database.  
+<p>One uses often so-called fully-connected feed-forward neural networks
+with three or more layers (an input layer, one or more hidden layers
+and an output layer) consisting of neurons that have non-linear
+activation functions.
 </p>
 
-<p>To feed data into a feed-forward neural network we need to represent
-the inputs as a design/feature matrix \( X = (n_{inputs}, n_{features}) \).  Each
-row represents an <em>input</em>, in this case a handwritten digit, and
-each column represents a <em>feature</em>, in this case a pixel.  The
-correct answers, also known as <em>labels</em> or <em>targets</em> are
-represented as a 1D array of integers 
-\( Y = (n_{inputs}) = (5, 3, 1, 8,...) \).
-</p>
-
-<p>As an example, say we want to build a neural network using supervised learning to predict Body-Mass Index (BMI) from
-measurements of height (in m)  
-and weight (in kg). If we have measurements of 5 people the design/feature matrix could be for example:  
-</p>
-
-<p>$$ X = \begin{bmatrix}
-1.85 &amp; 81\\
-1.71 &amp; 65\\
-1.95 &amp; 103\\
-1.55 &amp; 42\\
-1.63 &amp; 56
-\end{bmatrix} ,$$  
-</p>
-
-<p>and the targets would be:  </p>
-
-<p>$$ Y = (23.7, 22.2, 27.1, 17.5, 21.1) $$  </p>
-
-<p>Since each input image is a 2D matrix, we need to flatten the image
-(i.e. "unravel" the 2D matrix into a 1D array) to turn the data into a
-design/feature matrix. This means we lose all spatial information in the
-image, such as locality and translational invariance. More complicated
-architectures such as Convolutional Neural Networks can take advantage
-of such information, and are most commonly applied when analyzing
-images.
-</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
-
-
-<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
-
-<span style="color: #408080; font-style: italic"># display images in notebook</span>
-<span style="color: #666666">%</span>matplotlib inline
-plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;figure.figsize&#39;</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
-
-
-<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
-digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
-
-<span style="color: #408080; font-style: italic"># define inputs and labels</span>
-inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
-labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># flatten the image</span>
-<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
-n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
-inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;X = (n_inputs, n_features) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># choose some random images to display</span>
-indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
-random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
-
-<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
-    plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
-    plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">&#39;off&#39;</span>)
-    plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>)
-    plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
+<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -410,7 +362,7 @@ plt<span style="color: #666666">.</span>show()
   <li><a href="._week41-bs018.html">19</a></li>
   <li><a href="._week41-bs019.html">20</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs011.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs011.html b/doc/pub/week41/html/._week41-bs011.html
index 4c24560be..a9f0ad57a 100644
--- a/doc/pub/week41/html/._week41-bs011.html
+++ b/doc/pub/week41/html/._week41-bs011.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,68 +327,20 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0011"></a>
 <!-- !split -->
-<h2 id="train-and-test-datasets" class="anchor">Train and test datasets </h2>
+<h2 id="why-multilayer-perceptrons" class="anchor">Why multilayer perceptrons?  </h2>
 
-<p>Performing analysis before partitioning the dataset is a major error, that can lead to incorrect conclusions.  </p>
-
-<p>We will reserve \( 80 \% \) of our dataset for training and \( 20 \% \) for testing.  </p>
-
-<p>It is important that the train and test datasets are drawn randomly from our dataset, to ensure
-no bias in the sampling.  
-Say you are taking measurements of weather data to predict the weather in the coming 5 days.
-You don't want to train your model on measurements taken from the hours 00.00 to 12.00, and then test it on data
-collected from 12.00 to 24.00.
+<p>According to the <em>Universal approximation theorem</em>, a feed-forward
+neural network with just a single hidden layer containing a finite
+number of neurons can approximate a continuous multidimensional
+function to arbitrary accuracy, assuming the activation function for
+the hidden layer is a <b>non-constant, bounded and
+monotonically-increasing continuous function</b>.
 </p>
 
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
-
-<span style="color: #408080; font-style: italic"># one-liner from scikit-learn library</span>
-train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
-test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
-X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
-                                                    test_size<span style="color: #666666">=</span>test_size)
-
-<span style="color: #408080; font-style: italic"># equivalently in numpy</span>
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train_test_split_numpy</span>(inputs, labels, train_size, test_size):
-    n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
-    inputs_shuffled <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>copy()
-    labels_shuffled <span style="color: #666666">=</span> labels<span style="color: #666666">.</span>copy()
-    
-    np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>shuffle(inputs_shuffled)
-    np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>shuffle(labels_shuffled)
-    
-    train_end <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n_inputs<span style="color: #666666">*</span>train_size)
-    X_train, X_test <span style="color: #666666">=</span> inputs_shuffled[:train_end], inputs_shuffled[train_end:]
-    Y_train, Y_test <span style="color: #666666">=</span> labels_shuffled[:train_end], labels_shuffled[train_end:]
-    
-    <span style="color: #008000; font-weight: bold">return</span> X_train, X_test, Y_train, Y_test
-
-<span style="color: #408080; font-style: italic">#X_train, X_test, Y_train, Y_test = train_test_split_numpy(inputs, labels, train_size, test_size)</span>
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Number of training images: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(<span style="color: #008000">len</span>(X_train)))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Number of test images: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(<span style="color: #008000">len</span>(X_test)))
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
+<p>Note that the requirements on the activation function only applies to
+the hidden layer, the output nodes are always assumed to be linear, so
+as to not restrict the range of output values.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -362,7 +367,7 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
   <li><a href="._week41-bs019.html">20</a></li>
   <li><a href="._week41-bs020.html">21</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs012.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs012.html b/doc/pub/week41/html/._week41-bs012.html
index 7fb1d241d..62f7cac35 100644
--- a/doc/pub/week41/html/._week41-bs012.html
+++ b/doc/pub/week41/html/._week41-bs012.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,46 +327,15 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0012"></a>
 <!-- !split -->
-<h2 id="define-model-and-architecture" class="anchor">Define model and architecture </h2>
+<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" class="anchor">Illustration of a single perceptron model and a multi-perceptron model </h2>
 
-<p>Our simple feed-forward neural network will consist of an <em>input</em> layer, a single <em>hidden</em> layer and an <em>output</em> layer. The activation \( y \) of each neuron is a weighted sum of inputs, passed through an activation function. In case of the simple perceptron model we have </p>
-
-<p>$$ z = \sum_{i=1}^n w_i a_i ,$$</p>
-
-<p>$$ y = f(z) ,$$</p>
-
-<p>where \( f \) is the activation function, \( a_i \) represents input from neuron \( i \) in the preceding layer
-and \( w_i \) is the weight to input \( i \).  
-The activation of the neurons in the input layer is just the features (e.g. a pixel value).  
-</p>
-
-<p>The simplest activation function for a neuron is the <em>Heaviside</em> function:</p>
-
-<p>$$ f(z) = 
-\begin{cases}
-1,  &  z > 0\\
-0,  & \text{otherwise}
-\end{cases}
-$$
-</p>
-
-<p>A feed-forward neural network with this activation is known as a <em>perceptron</em>.  
-For a binary classifier (i.e. two classes, 0 or 1, dog or not-dog) we can also use this in our output layer.  
-This activation can be generalized to \( k \) classes (using e.g. the <em>one-against-all</em> strategy), 
-and we call these architectures <em>multiclass perceptrons</em>.  
-</p>
-
-<p>However, it is now common to use the terms Single Layer Perceptron (SLP) (1 hidden layer) and  
-Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to feed-forward neural networks with any activation function.  
-</p>
-
-<p>Typical choices for activation functions include the sigmoid function, hyperbolic tangent, and Rectified Linear Unit (ReLU).  
-We will be using the sigmoid function \( \sigma(x) \):  
-</p>
-
-<p>$$ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,$$</p>
-
-<p>which is inspired by probability theory (see logistic regression) and was most commonly used until about 2011. See the discussion below concerning other activation functions.</p>
+<center>  <!-- FIGURE -->
+<hr class="figure">
+<center>
+<p class="caption">Figure 1:  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer. </p>
+</center>
+<p><img src="figures/nns.png" width="600" align="bottom"></p>
+</center>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -340,7 +362,7 @@ We will be using the sigmoid function \( \sigma(x) \):
   <li><a href="._week41-bs020.html">21</a></li>
   <li><a href="._week41-bs021.html">22</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs013.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs013.html b/doc/pub/week41/html/._week41-bs013.html
index 46d73c875..debc3cf17 100644
--- a/doc/pub/week41/html/._week41-bs013.html
+++ b/doc/pub/week41/html/._week41-bs013.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,46 +326,69 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0013"></a>
-<!-- !split   -->
-<h2 id="layers" class="anchor">Layers </h2>
+<!-- !split -->
+<h2 id="examples-of-xor-or-and-and-gates" class="anchor">Examples of XOR, OR and AND gates </h2>
 
-<ul>
-<li> Input</li> 
-</ul>
-<p>Since each input image has 8x8 = 64 pixels or features, we have an input layer of 64 neurons.  </p>
-
-<ul>
-<li> Hidden layer</li>
-</ul>
-<p>We will use 50 neurons in the hidden layer receiving input from the neurons in the input layer.  
-Since each neuron in the hidden layer is connected to the 64 inputs we have 64x50 = 3200 weights to the hidden layer.  
+<p>Let us first try to fit various gates using standard linear
+regression. The gates we are thinking of are the classical XOR, OR and
+AND gates, well-known elements in computer science. The tables here
+show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
+specific target \( y_i \).
 </p>
 
-<ul>
-<li> Output</li>
-</ul>
-<p>If we were building a binary classifier, it would be sufficient with a single neuron in the output layer,
-which could output 0 or 1 according to the Heaviside function. This would be an example of a <em>hard</em> classifier, meaning it outputs the class of the input directly. However, if we are dealing with noisy data it is often beneficial to use a <em>soft</em> classifier, which outputs the probability of being in class 0 or 1.  
-</p>
 
-<p>For a soft binary classifier, we could use a single neuron and interpret the output as either being the probability of being in class 0 or the probability of being in class 1. Alternatively we could use 2 neurons, and interpret each neuron as the probability of being in each class.  </p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<span style="color: #BA2121; font-style: italic">Simple code that tests XOR, OR and AND gates with linear regression</span>
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
 
-<p>Since we are doing multiclass classification, with 10 categories, it is natural to use 10 neurons in the output layer. We number the neurons \( j = 0,1,...,9 \). The activation of each output neuron \( j \) will be according to the <em>softmax</em> function:  </p>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The invers of X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
 
-<p>$$ P(\text{class \( j \)} \mid \text{input \( \boldsymbol{a} \)}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}}
-{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,$$  
-</p>
+<span style="color: #408080; font-style: italic"># The XOR gate </span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+ThetaXOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
 
-<p>i.e. each neuron \( j \) outputs the probability of being in class \( j \) given an input from the hidden layer \( \boldsymbol{a} \), with \( \boldsymbol{w}_j \) the weights of neuron \( j \) to the inputs.  
-The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.  
-The exponent is just the weighted sum of inputs as before:  
-</p>
 
-<p>$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$  </p>
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+ThetaOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
 
-<p>Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
-weights to the output layer.
-</p>
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+ThetaAND  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>What is happening here?</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -339,7 +415,7 @@ weights to the output layer.
   <li><a href="._week41-bs021.html">22</a></li>
   <li><a href="._week41-bs022.html">23</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs014.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs014.html b/doc/pub/week41/html/._week41-bs014.html
index 66c54d540..2580923a4 100644
--- a/doc/pub/week41/html/._week41-bs014.html
+++ b/doc/pub/week41/html/._week41-bs014.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,20 +326,9 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0014"></a>
-<!-- !split   -->
-<h2 id="weights-and-biases" class="anchor">Weights and biases </h2>
+<!-- !split -->
+<h2 id="does-logistic-regression-do-a-better-job" class="anchor">Does Logistic Regression do a better Job? </h2>
 
-<p>Typically weights are initialized with small values distributed around zero, drawn from a uniform
-or normal distribution. Setting all weights to zero means all neurons give the same output, making the network useless.  
-</p>
-
-<p>Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
-of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):  
-</p>
-
-<p>$$ z_j = \sum_{i=1}^n w_ {ij} a_i + b_j.$$  </p>
-
-<p>The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.</p>
 
 <!-- code=python (!bc pycod) typeset with pygments style "default" -->
 <div class="cell border-box-sizing code_cell rendered">
@@ -294,21 +336,55 @@ of values. Without it, any input with the value 0 will be mapped to zero (before
     <div class="inner_cell">
       <div class="input_area">
         <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># building our neural network</span>
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<span style="color: #BA2121; font-style: italic">Simple code that tests XOR and OR gates with linear regression</span>
+<span style="color: #BA2121; font-style: italic">and logistic regression</span>
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
 
-n_inputs, n_features <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape
-n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">50</span>
-n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
 
-<span style="color: #408080; font-style: italic"># we make the weights normally distributed using numpy.random.randn</span>
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The invers of X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
 
-<span style="color: #408080; font-style: italic"># weights and bias in the hidden layer</span>
-hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_features, n_hidden_neurons)
-hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+<span style="color: #408080; font-style: italic"># The XOR gate </span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+ThetaXOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
 
-<span style="color: #408080; font-style: italic"># weights and bias in the output layer</span>
-output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_hidden_neurons, n_categories)
-output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+ThetaOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+ThetaAND  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+<span style="color: #408080; font-style: italic"># Now we change to logistic regression</span>
+
+
+<span style="color: #408080; font-style: italic"># Logistic Regression</span>
+logreg <span style="color: #666666">=</span> LogisticRegression()
+logreg<span style="color: #666666">.</span>fit(X, yOR)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for OR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yOR)))
+
+logreg<span style="color: #666666">.</span>fit(X, yXOR)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for XOR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yXOR)))
+
+
+logreg<span style="color: #666666">.</span>fit(X, yAND)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for AND gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yAND)))
 </pre>
 </div>
       </div>
@@ -324,6 +400,7 @@ output_bias <span style="color: #666666">=</span> np<span style="color: #666666"
   </div>
 </div>
 
+<p>Not exactly impressive, but somewhat better.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -350,7 +427,7 @@ output_bias <span style="color: #666666">=</span> np<span style="color: #666666"
   <li><a href="._week41-bs022.html">23</a></li>
   <li><a href="._week41-bs023.html">24</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs015.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs015.html b/doc/pub/week41/html/._week41-bs015.html
index 1bf9722db..918bced40 100644
--- a/doc/pub/week41/html/._week41-bs015.html
+++ b/doc/pub/week41/html/._week41-bs015.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,27 +327,38 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0015"></a>
 <!-- !split -->
-<h2 id="feed-forward-pass" class="anchor">Feed-forward pass </h2>
+<h2 id="adding-neural-networks" class="anchor">Adding Neural Networks </h2>
 
-<p>Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.  
-For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):  
-</p>
 
-<p>$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$</p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># and now neural networks with Scikit-Learn and the XOR</span>
 
-<p>this is then passed through our activation function  </p>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_classification
+X, yXOR <span style="color: #666666">=</span> make_classification(n_samples<span style="color: #666666">=100</span>, random_state<span style="color: #666666">=1</span>)
+FFNN <span style="color: #666666">=</span> MLPClassifier(random_state<span style="color: #666666">=1</span>, max_iter<span style="color: #666666">=300</span>)<span style="color: #666666">.</span>fit(X, yXOR)
+FFNN<span style="color: #666666">.</span>predict_proba(X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;Test set accuracy with Feed Forward Neural Network  for XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>FFNN<span style="color: #666666">.</span>score(X, yXOR)<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>$$ a_{j}^{l} = f(z_{j}^{l}) .$$  </p>
-
-<p>We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:  </p>
-
-<p>$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$  </p>
-
-<p>Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:  </p>
-
-<p>$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
-{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$  
-</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -321,7 +385,7 @@ For each input image we calculate a weighted sum of input features (pixel values
   <li><a href="._week41-bs023.html">24</a></li>
   <li><a href="._week41-bs024.html">25</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs016.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs016.html b/doc/pub/week41/html/._week41-bs016.html
index aace9ce5a..1581a1234 100644
--- a/doc/pub/week41/html/._week41-bs016.html
+++ b/doc/pub/week41/html/._week41-bs016.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,96 +326,22 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0016"></a>
-<!-- !split    -->
-<h2 id="matrix-multiplications" class="anchor">Matrix multiplications </h2>
+<!-- !split -->
+<h2 id="mathematical-model" class="anchor">Mathematical model  </h2>
 
-<p>Since our data has the dimensions \( X = (n_{inputs}, n_{features}) \) and our weights to the hidden
-layer have the dimensions  
-\( W_{hidden} = (n_{features}, n_{hidden}) \),
-we can easily feed the network all our training data in one go by taking the matrix product  
+<p>The output \( y \) is produced via the activation function \( f \)</p>
+$$
+ y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
+$$
+
+<p>This function receives \( x_i \) as inputs.
+Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \). 
+In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
+the neurons in the preceding layer. Furthermore, an MLP is
+fully-connected, which means that each neuron receives a weighted sum
+of the outputs of <em>all</em> neurons in the previous layer.
 </p>
 
-<p>$$ X W^{h} = (n_{inputs}, n_{hidden}),$$ </p>
-
-<p>and obtain a matrix that holds the weighted sum of inputs to the hidden layer
-for each input image and each hidden neuron.    
-We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):  
-</p>
-
-<p>$$ \boldsymbol{z}^{l} = \boldsymbol{X} \boldsymbol{W}^{l} + \boldsymbol{b}^{l} ,$$</p>
-
-<p>meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.  
-This is then passed through the activation:  
-</p>
-
-<p>$$ \boldsymbol{a}^{l} = f(\boldsymbol{z}^l) .$$  </p>
-
-<p>This is fed to the output layer:  </p>
-
-<p>$$ \boldsymbol{z}^{L} = \boldsymbol{a}^{L} \boldsymbol{W}^{L} + \boldsymbol{b}^{L} .$$</p>
-
-<p>Finally we receive our output values for each image and each category by passing it through the softmax function:  </p>
-
-<p>$$ output = softmax (\boldsymbol{z}^{L}) = (n_{inputs}, n_{categories}) .$$</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># setup the feed-forward pass, subscript h = hidden layer</span>
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
-    <span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(X):
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
-    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
-    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
-    a_h <span style="color: #666666">=</span> sigmoid(z_h)
-    
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
-    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
-    <span style="color: #408080; font-style: italic"># softmax output</span>
-    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
-    exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
-    probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
-    
-    <span style="color: #008000; font-weight: bold">return</span> probabilities
-
-probabilities <span style="color: #666666">=</span> feed_forward(X_train)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probabilities = (n_inputs, n_categories) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probability that image 0 is in category 0,1,2,...,9 = </span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities[<span style="color: #666666">0</span>]))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probabilities sum up to: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>sum()))
-<span style="color: #008000">print</span>()
-
-<span style="color: #408080; font-style: italic"># we obtain a prediction by taking the class with the highest likelihood</span>
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(X):
-    probabilities <span style="color: #666666">=</span> feed_forward(X)
-    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
-
-predictions <span style="color: #666666">=</span> predict(X_train)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;predictions = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(predictions<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;prediction for image 0: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(predictions[<span style="color: #666666">0</span>]))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;correct label for image 0: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(Y_train[<span style="color: #666666">0</span>]))
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -388,7 +367,7 @@ predictions <span style="color: #666666">=</span> predict(X_train)
   <li><a href="._week41-bs024.html">25</a></li>
   <li><a href="._week41-bs025.html">26</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs017.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs017.html b/doc/pub/week41/html/._week41-bs017.html
index 73a73e222..d2bb8b9e0 100644
--- a/doc/pub/week41/html/._week41-bs017.html
+++ b/doc/pub/week41/html/._week41-bs017.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,32 +327,48 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0017"></a>
 <!-- !split -->
-<h2 id="choose-cost-function-and-optimizer" class="anchor">Choose cost function and optimizer </h2>
+<h2 id="mathematical-model" class="anchor">Mathematical model  </h2>
 
-<p>To measure how well our neural network is doing we need to introduce a cost function.  
-We will call the function that gives the error of a single sample output the <em>loss</em> function, and the function
-that gives the total error of our network across all samples the <em>cost</em> function.
-A typical choice for multiclass classification is the <em>cross-entropy</em> loss, also known as the negative log likelihood.  
+<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
+
+$$
+\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
+\tag{2}
+\end{equation}
+$$
+
+<p>Here \( b_i \) is the so-called bias which is normally needed in
+case of zero activation weights or inputs. How to fix the biases and
+the weights will be discussed below.  The value of \( z_i^1 \) is the
+argument to the activation function \( f_i \) of each node \( i \), The
+variable \( M \) stands for all possible inputs to a given node \( i \) in the
+first layer.  We define  the output \( y_i^1 \) of all neurons in layer 1 as
 </p>
 
-<p>In <em>multiclass</em> classification it is common to treat each integer label as a so called <em>one-hot</em> vector:  </p>
+$$
+\begin{equation}
+ y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
+\tag{3}
+\end{equation}
+$$
 
-<p>$$ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,$$  </p>
-
-<p>$$ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,$$  </p>
-
-<p>i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset.  </p>
-
-<p>Let \( y_{ic} \) denote the \( c \)-th component of the \( i \)-th one-hot vector.  
-We define the cost function \( \mathcal{C} \) as a sum over the cross-entropy loss for each point \( \boldsymbol{x}_i \) in the dataset.
+<p>where we assume that all nodes in the same layer have identical
+activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
+In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
 </p>
 
-<p>In the one-hot representation only one of the terms in the loss function is non-zero, namely the
-probability of the correct category \( c' \)  
-(i.e. the category \( c' \) such that \( y_{ic'} = 1 \)). This means that the cross entropy loss only punishes you for how wrong
-you got the correct label. The probability of category \( c \) is given by the softmax function. The vector \( \boldsymbol{\theta} \) represents the parameters of our network, i.e. all the weights and biases.  
-</p>
+$$
+\begin{equation}
+ y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
+\tag{4}
+\end{equation}
+$$
 
+<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
+all the nodes in the first hidden layer are computed, the values of
+the subsequent layer can be calculated and so forth until the output
+is obtained.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -326,7 +395,7 @@ you got the correct label. The probability of category \( c \) is given by the s
   <li><a href="._week41-bs025.html">26</a></li>
   <li><a href="._week41-bs026.html">27</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs018.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs018.html b/doc/pub/week41/html/._week41-bs018.html
index b47ac487d..670e48622 100644
--- a/doc/pub/week41/html/._week41-bs018.html
+++ b/doc/pub/week41/html/._week41-bs018.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,39 +327,31 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0018"></a>
 <!-- !split -->
-<h2 id="optimizing-the-cost-function" class="anchor">Optimizing the cost function </h2>
+<h2 id="mathematical-model" class="anchor">Mathematical model  </h2>
 
-<p>The network is trained by finding the weights and biases that minimize the cost function. One of the most widely used classes of methods is <em>gradient descent</em> and its generalizations. The idea behind gradient descent
-is simply to adjust the weights in the direction where the gradient of the cost function is large and negative. This ensures we flow toward a <em>local</em> minimum of the cost function.  
-Each parameter \( \theta \) is iteratively adjusted according to the rule  
-</p>
+<p>The output of neuron \( i \) in layer 2 is thus,</p>
 
-<p>$$ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,$$</p>
+$$
+\begin{align}
+ y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) 
+\tag{5}\\
+ &= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
+\tag{6}
+\end{align}
+$$
 
-<p>where \( \eta \) is known as the <em>learning rate</em>, which controls how big a step we take towards the minimum.  
-This update can be repeated for any number of iterations, or until we are satisfied with the result.  
-</p>
+<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
 
-<p>A simple and effective improvement is a variant called <em>Batch Gradient Descent</em>.  
-Instead of calculating the gradient on the whole dataset, we calculate an approximation of the gradient
-on a subset of the data called a <em>minibatch</em>.  
-If there are \( N \) data points and we have a minibatch size of \( M \), the total number of batches
-is \( N/M \).  
-We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient then becomes:  
-</p>
+$$
+\begin{align}
+ y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) 
+\tag{7}\\
+ &= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+  + b_1^3\right]
+\tag{8}
+\end{align}
+$$
 
-<p>$$ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
-\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,$$
-</p>
-
-<p>i.e. instead of averaging the loss over the entire dataset, we average over a minibatch.  </p>
-
-<p>This has two important benefits:  </p>
-<ol>
-<li> Introducing stochasticity decreases the chance that the algorithm becomes stuck in a local minima.</li>  
-<li> It significantly speeds up the calculation, since we do not have to use the entire dataset to calculate the gradient.</li>  
-</ol>
-<p>The various optmization  methods, with codes and algorithms,  are discussed in our lectures on <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_self">Gradient descent approaches</a>.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -333,7 +378,7 @@ We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient th
   <li><a href="._week41-bs026.html">27</a></li>
   <li><a href="._week41-bs027.html">28</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs019.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs019.html b/doc/pub/week41/html/._week41-bs019.html
index 47378b137..9f0dba901 100644
--- a/doc/pub/week41/html/._week41-bs019.html
+++ b/doc/pub/week41/html/._week41-bs019.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,34 +326,22 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0019"></a>
-<!-- !split   -->
-<h2 id="regularization" class="anchor">Regularization </h2>
+<!-- !split -->
+<h2 id="mathematical-model" class="anchor">Mathematical model  </h2>
 
-<p>It is common to add an extra term to the cost function, proportional
-to the size of the weights.  This is equivalent to constraining the
-size of the weights, so that they do not grow out of control.
-Constraining the size of the weights means that the weights cannot
-grow arbitrarily large to fit the training data, and in this way
-reduces <em>overfitting</em>.
+<p>We can generalize this expression to an MLP with \( l \) hidden
+layers. The complete functional form is,
 </p>
 
-<p>We will measure the size of the weights using the so called <em>L2-norm</em>, meaning our cost function becomes:  </p>
+$$
+\begin{align}
+&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
+\tag{9}
+\end{align}
+$$
 
-<p>$$  \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) \quad \rightarrow \quad
-\frac{1}{N} \sum_{i=1}^N  \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2 
-= \frac{1}{N} \sum_{i=1}^N  \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,$$  
-</p>
-
-<p>i.e. we sum up all the weights squared. The factor \( \lambda \) is known as a regularization parameter.</p>
-
-<p>In order to train the model, we need to calculate the derivative of
-the cost function with respect to every bias and weight in the
-network.  In total our network has \( (64 + 1)\times 50=3250 \) weights in
-the hidden layer and \( (50 + 1)\times 10=510 \) weights to the output
-layer (\( +1 \) for the bias), and the gradient must be calculated for
-every parameter.  We use the <em>backpropagation</em> algorithm discussed
-above. This is a clever use of the chain rule that allows us to
-calculate the gradient efficently. 
+<p>which illustrates a basic property of MLPs: The only independent
+variables are the input values \( x_n \).
 </p>
 
 <p>
@@ -328,7 +369,7 @@ calculate the gradient efficently.
   <li><a href="._week41-bs027.html">28</a></li>
   <li><a href="._week41-bs028.html">29</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs020.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs020.html b/doc/pub/week41/html/._week41-bs020.html
index 54ab35197..62dc35b12 100644
--- a/doc/pub/week41/html/._week41-bs020.html
+++ b/doc/pub/week41/html/._week41-bs020.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,132 +327,32 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0020"></a>
 <!-- !split -->
-<h2 id="matrix-multiplication" class="anchor">Matrix  multiplication </h2>
+<h2 id="mathematical-model" class="anchor">Mathematical model  </h2>
 
-<p>To more efficently train our network these equations are implemented using matrix operations.  
-The error in the output layer is calculated simply as, with \( \boldsymbol{t} \) being our targets,  
+<p>This confirms that an MLP, despite its quite convoluted mathematical
+form, is nothing more than an analytic function, specifically a
+mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
+\hat{y} \in \mathbb{R}^m \).
 </p>
 
-<p>$$ \delta_L = \boldsymbol{t} - \boldsymbol{y} = (n_{inputs}, n_{categories}) .$$  </p>
-
-<p>The gradient for the output weights is calculated as  </p>
-
-<p>$$ \nabla W_{L} = \boldsymbol{a}^T \delta_L   = (n_{hidden}, n_{categories}) ,$$</p>
-
-<p>where \( \boldsymbol{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.  
-Since we are going backwards we have to transpose the activation matrix.  
+<p>Furthermore, the flexibility and universality of an MLP can be
+illustrated by realizing that the expression is essentially a nested
+sum of scaled activation functions of the form
 </p>
 
-<p>The gradient with respect to the output bias is then  </p>
+$$
+\begin{equation}
+ f(x) = c_1 f(c_2 x + c_3) + c_4
+\tag{10}
+\end{equation}
+$$
 
-<p>$$ \nabla \boldsymbol{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$  </p>
-
-<p>The error in the hidden layer is  </p>
-
-<p>$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$  </p>
-
-<p>where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
-that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
-the <em>Hadamard product</em>, meaning element-wise multiplication.  
+<p>where the parameters \( c_i \) are weights and biases. By adjusting these
+parameters, the activation functions can be shifted up and down or
+left and right, change slope or be rescaled which is the key to the
+flexibility of a neural network.
 </p>
 
-<p>This again gives us the gradients in the hidden layer:  </p>
-
-<p>$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$  </p>
-
-<p>$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># to categorical turns our integer vector into a onehot representation</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
-
-<span style="color: #408080; font-style: italic"># one-hot in numpy</span>
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">to_categorical_numpy</span>(integer_vector):
-    n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(integer_vector)
-    n_categories <span style="color: #666666">=</span> np<span style="color: #666666">.</span>max(integer_vector) <span style="color: #666666">+</span> <span style="color: #666666">1</span>
-    onehot_vector <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_inputs, n_categories))
-    onehot_vector[<span style="color: #008000">range</span>(n_inputs), integer_vector] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
-    
-    <span style="color: #008000; font-weight: bold">return</span> onehot_vector
-
-<span style="color: #408080; font-style: italic">#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)</span>
-Y_train_onehot, Y_test_onehot <span style="color: #666666">=</span> to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward_train</span>(X):
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
-    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
-    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
-    a_h <span style="color: #666666">=</span> sigmoid(z_h)
-    
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
-    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
-    <span style="color: #408080; font-style: italic"># softmax output</span>
-    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
-    exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
-    probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
-    
-    <span style="color: #408080; font-style: italic"># for backpropagation need activations in hidden and output layers</span>
-    <span style="color: #008000; font-weight: bold">return</span> a_h, probabilities
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backpropagation</span>(X, Y):
-    a_h, probabilities <span style="color: #666666">=</span> feed_forward_train(X)
-    
-    <span style="color: #408080; font-style: italic"># error in the output layer</span>
-    error_output <span style="color: #666666">=</span> probabilities <span style="color: #666666">-</span> Y
-    <span style="color: #408080; font-style: italic"># error in the hidden layer</span>
-    error_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(error_output, output_weights<span style="color: #666666">.</span>T) <span style="color: #666666">*</span> a_h <span style="color: #666666">*</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> a_h)
-    
-    <span style="color: #408080; font-style: italic"># gradients for the output layer</span>
-    output_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h<span style="color: #666666">.</span>T, error_output)
-    output_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_output, axis<span style="color: #666666">=0</span>)
-    
-    <span style="color: #408080; font-style: italic"># gradient for the hidden layer</span>
-    hidden_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X<span style="color: #666666">.</span>T, error_hidden)
-    hidden_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_hidden, axis<span style="color: #666666">=0</span>)
-
-    <span style="color: #008000; font-weight: bold">return</span> output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Old accuracy on training data: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(accuracy_score(predict(X_train), Y_train)))
-
-eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
-lmbd <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1000</span>):
-    <span style="color: #408080; font-style: italic"># calculate gradients</span>
-    dWo, dBo, dWh, dBh <span style="color: #666666">=</span> backpropagation(X_train, Y_train_onehot)
-    
-    <span style="color: #408080; font-style: italic"># regularization term gradients</span>
-    dWo <span style="color: #666666">+=</span> lmbd <span style="color: #666666">*</span> output_weights
-    dWh <span style="color: #666666">+=</span> lmbd <span style="color: #666666">*</span> hidden_weights
-    
-    <span style="color: #408080; font-style: italic"># update weights and biases</span>
-    output_weights <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dWo
-    output_bias <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dBo
-    hidden_weights <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dWh
-    hidden_bias <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dBh
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;New accuracy on training data: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(accuracy_score(predict(X_train), Y_train)))
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -425,7 +378,7 @@ lmbd <span style="color: #666666">=</span> <span style="color: #666666">0.01</sp
   <li><a href="._week41-bs028.html">29</a></li>
   <li><a href="._week41-bs029.html">30</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs021.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs021.html b/doc/pub/week41/html/._week41-bs021.html
index 6b8826a36..bf66b7ea6 100644
--- a/doc/pub/week41/html/._week41-bs021.html
+++ b/doc/pub/week41/html/._week41-bs021.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,21 +327,41 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0021"></a>
 <!-- !split -->
-<h2 id="improving-performance" class="anchor">Improving performance </h2>
+<h3 id="matrix-vector-notation" class="anchor">Matrix-vector notation </h3>
 
-<p>As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image.  
-In order to obtain a network that does something useful, we will have to do a bit more work.  
+<p>We can introduce a more convenient notation for the activations in an A NN. </p>
+
+<p>Additionally, we can represent the biases and activations
+as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector 
+is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively. 
 </p>
 
-<p>The choice of <em>hyperparameters</em> such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a <em>grid-search</em> is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates \( \eta = 10^{-6}, 10^{-5},...,10^{-1} \) with different regularization parameters \( \lambda = 10^{-6},...,10^{-0} \).  </p>
-
-<p>Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an <em>iteration</em>, and a full training period
-going through the entire dataset (\( n/M \) batches) an <em>epoch</em>.
+<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors. 
+With this notation, the sum becomes a matrix-vector multiplication, and we can write
+the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
 </p>
+$$
+\begin{equation}
+ \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
+ f_2\left(\left[\begin{array}{ccc}
+    w^2_{11} &w^2_{12} &w^2_{13} \\
+    w^2_{21} &w^2_{22} &w^2_{23} \\
+    w^2_{31} &w^2_{32} &w^2_{33} \\
+    \end{array} \right] \cdot
+    \left[\begin{array}{c}
+           y^1_1 \\
+           y^1_2 \\
+           y^1_3 \\
+          \end{array}\right] + 
+    \left[\begin{array}{c}
+           b^2_1 \\
+           b^2_2 \\
+           b^2_3 \\
+          \end{array}\right]\right).
+\tag{11}
+\end{equation}
+$$
 
-<p>If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers.  
-Andrew Ng goes through some of these considerations in this <a href="https://youtu.be/F1ka6a13S9I" target="_self">video</a>. You can find a summary of the video <a href="https://kevinzakka.github.io/2016/09/26/applying-deep-learning/" target="_self">here</a>.  
-</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -315,7 +388,7 @@ Andrew Ng goes through some of these considerations in this <a href="https://you
   <li><a href="._week41-bs029.html">30</a></li>
   <li><a href="._week41-bs030.html">31</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs022.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs022.html b/doc/pub/week41/html/._week41-bs022.html
index 4bf7c8453..63923d2e8 100644
--- a/doc/pub/week41/html/._week41-bs022.html
+++ b/doc/pub/week41/html/._week41-bs022.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,134 +327,25 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0022"></a>
 <!-- !split -->
-<h2 id="full-object-oriented-implementation" class="anchor">Full object-oriented implementation </h2>
+<h3 id="matrix-vector-notation-and-activation" class="anchor">Matrix-vector notation  and activation </h3>
 
-<p>It is very natural to think of the network as an object, with specific instances of the network
-being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below.
+<p>The activation of node \( i \) in layer 2 is</p>
+
+$$
+\begin{equation}
+ y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
+ f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
+\tag{12}
+\end{equation}
+$$
+
+<p>This is not just a convenient and compact notation, but also a useful
+and intuitive way to think about MLPs: The output is calculated by a
+series of matrix-vector multiplications and vector additions that are
+used as input to the activation functions. For each operation
+\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
 </p>
 
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">NeuralNetwork</span>:
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
-            <span style="color: #008000">self</span>,
-            X_data,
-            Y_data,
-            n_hidden_neurons<span style="color: #666666">=50</span>,
-            n_categories<span style="color: #666666">=10</span>,
-            epochs<span style="color: #666666">=10</span>,
-            batch_size<span style="color: #666666">=100</span>,
-            eta<span style="color: #666666">=0.1</span>,
-            lmbd<span style="color: #666666">=0.0</span>):
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data_full <span style="color: #666666">=</span> X_data
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data_full <span style="color: #666666">=</span> Y_data
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">=</span> X_data<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_features <span style="color: #666666">=</span> X_data<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons <span style="color: #666666">=</span> n_hidden_neurons
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories <span style="color: #666666">=</span> n_categories
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>epochs <span style="color: #666666">=</span> epochs
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size <span style="color: #666666">=</span> batch_size
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>iterations <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">=</span> eta
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">=</span> lmbd
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>create_biases_and_weights()
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_biases_and_weights</span>(<span style="color: #008000">self</span>):
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons)
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories)
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(<span style="color: #008000">self</span>):
-        <span style="color: #408080; font-style: italic"># feed-forward for training</span>
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_data, <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h <span style="color: #666666">=</span> sigmoid(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_h)
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>a_h, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias
-
-        exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_o)
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward_out</span>(<span style="color: #008000">self</span>, X):
-        <span style="color: #408080; font-style: italic"># feed-forward for output</span>
-        z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias
-        a_h <span style="color: #666666">=</span> sigmoid(z_h)
-
-        z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias
-        
-        exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
-        probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
-        <span style="color: #008000; font-weight: bold">return</span> probabilities
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backpropagation</span>(<span style="color: #008000">self</span>):
-        error_output <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>probabilities <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data
-        error_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(error_output, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights<span style="color: #666666">.</span>T) <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h <span style="color: #666666">*</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h)
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>a_h<span style="color: #666666">.</span>T, error_output)
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_output, axis<span style="color: #666666">=0</span>)
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_data<span style="color: #666666">.</span>T, error_hidden)
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_hidden, axis<span style="color: #666666">=0</span>)
-
-        <span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">&gt;</span> <span style="color: #666666">0.0</span>:
-            <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights
-            <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights
-
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias_gradient
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient
-        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias_gradient
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(<span style="color: #008000">self</span>, X):
-        probabilities <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward_out(X)
-        <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict_probabilities</span>(<span style="color: #008000">self</span>, X):
-        probabilities <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward_out(X)
-        <span style="color: #008000; font-weight: bold">return</span> probabilities
-
-    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train</span>(<span style="color: #008000">self</span>):
-        data_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs)
-
-        <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
-            <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations):
-                <span style="color: #408080; font-style: italic"># pick datapoints with replacement</span>
-                chosen_datapoints <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(
-                    data_indices, size<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size, replace<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>
-                )
-
-                <span style="color: #408080; font-style: italic"># minibatch training data</span>
-                <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data_full[chosen_datapoints]
-                <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data_full[chosen_datapoints]
-
-                <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward()
-                <span style="color: #008000">self</span><span style="color: #666666">.</span>backpropagation()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -427,7 +371,7 @@ being realizations of this object with different hyperparameters. An implementat
   <li><a href="._week41-bs030.html">31</a></li>
   <li><a href="._week41-bs031.html">32</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs023.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs023.html b/doc/pub/week41/html/._week41-bs023.html
index 0e2869ac0..b905a80d8 100644
--- a/doc/pub/week41/html/._week41-bs023.html
+++ b/doc/pub/week41/html/._week41-bs023.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,56 +327,20 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0023"></a>
 <!-- !split -->
-<h2 id="evaluate-model-performance-on-test-data" class="anchor">Evaluate model performance on test data </h2>
+<h3 id="activation-functions" class="anchor">Activation functions  </h3>
 
-<p>To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data.  
-We measure the performance of the network using the <em>accuracy</em> score.  
-The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \).  
+<p>A property that characterizes a neural network, other than its
+connectivity, is the choice of activation function(s).  As described
+in, the following restrictions are imposed on an activation function
+for a FFNN to fulfill the universal approximation theorem
 </p>
 
-<p>$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\tilde{y}_i = y_i)}{n} ,$$  </p>
-
-<p>where \( I \) is the indicator function, \( 1 \) if \( \tilde{y}_i = y_i \) and \( 0 \) otherwise.</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-
-dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
-                    n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
-dnn<span style="color: #666666">.</span>train()
-test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
-
-<span style="color: #408080; font-style: italic"># accuracy score from scikit library</span>
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
-
-<span style="color: #408080; font-style: italic"># equivalent in numpy</span>
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_score_numpy</span>(Y_test, Y_pred):
-    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum(Y_test <span style="color: #666666">==</span> Y_pred) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(Y_test)
-
-<span style="color: #408080; font-style: italic">#print(&quot;Accuracy score on test set: &quot;, accuracy_score_numpy(Y_test, test_predict))</span>
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
+<ul>
+  <li> Non-constant</li>
+  <li> Bounded</li>
+  <li> Monotonically-increasing</li>
+  <li> Continuous</li>
+</ul>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -349,7 +366,7 @@ test_predict <span style="color: #666666">=</span> dnn<span style="color: #66666
   <li><a href="._week41-bs031.html">32</a></li>
   <li><a href="._week41-bs032.html">33</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs024.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs024.html b/doc/pub/week41/html/._week41-bs024.html
index 6d8eb8cdb..d62d40f21 100644
--- a/doc/pub/week41/html/._week41-bs024.html
+++ b/doc/pub/week41/html/._week41-bs024.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,53 +327,27 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0024"></a>
 <!-- !split -->
-<h2 id="adjust-hyperparameters" class="anchor">Adjust hyperparameters </h2>
+<h3 id="activation-functions-logistic-and-hyperbolic-ones" class="anchor">Activation functions, Logistic and Hyperbolic ones  </h3>
 
-<p>We now perform a grid search to find the optimal hyperparameters for the network.  
-Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
+<p>The second requirement excludes all linear functions. Furthermore, in
+a MLP with only linear activation functions, each layer simply
+performs a linear transformation of its inputs.
 </p>
 
+<p>Regardless of the number of layers, the output of the NN will be
+nothing but a linear function of the inputs. Thus we need to introduce
+some kind of non-linearity to the NN to be able to fit non-linear
+functions Typical examples are the logistic <em>Sigmoid</em>
+</p>
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-<span style="color: #408080; font-style: italic"># store the models for later use</span>
-DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
+$$
+ f(x) = \frac{1}{1 + e^{-x}},
+$$
 
-<span style="color: #408080; font-style: italic"># grid search</span>
-<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
-    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
-        dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
-                            n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
-        dnn<span style="color: #666666">.</span>train()
-        
-        DNN_numpy[i][j] <span style="color: #666666">=</span> dnn
-        
-        test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
-        
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
-        <span style="color: #008000">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
+<p>and the <em>hyperbolic tangent</em> function</p>
+$$
+ f(x) = \tanh(x)
+$$
 
 
 <p>
@@ -348,7 +375,7 @@ DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.
   <li><a href="._week41-bs032.html">33</a></li>
   <li><a href="._week41-bs033.html">34</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs025.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs025.html b/doc/pub/week41/html/._week41-bs025.html
index edc60cd85..8a7752865 100644
--- a/doc/pub/week41/html/._week41-bs025.html
+++ b/doc/pub/week41/html/._week41-bs025.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,7 +327,14 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0025"></a>
 <!-- !split -->
-<h2 id="visualization" class="anchor">Visualization </h2>
+<h3 id="relevance" class="anchor">Relevance </h3>
+
+<p>The <em>sigmoid</em> function are more biologically plausible because the
+output of inactive neurons are zero. Such activation function are
+called <em>one-sided</em>. However, it has been shown that the hyperbolic
+tangent performs better than the sigmoid for training MLPs.  has
+become the most popular for <em>deep neural networks</em>
+</p>
 
 
 <!-- code=python (!bc pycod) typeset with pygments style "default" -->
@@ -283,38 +343,76 @@ MathJax.Hub.Config({
     <div class="inner_cell">
       <div class="input_area">
         <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># visual representation of grid search</span>
-<span style="color: #408080; font-style: italic"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;The sigmoid function (or the logistic curve) is a </span>
+<span style="color: #BA2121; font-style: italic">function that takes any real number, z, and outputs a number (0,1).</span>
+<span style="color: #BA2121; font-style: italic">It is useful in neural networks for assigning weights on a relative scale.</span>
+<span style="color: #BA2121; font-style: italic">The value z is the weighted sum of parameters involved in the learning algorithm.&quot;&quot;&quot;</span>
 
-sns<span style="color: #666666">.</span>set()
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mt</span>
 
-train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.1</span>)
+sigma_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1/</span>(<span style="color: #666666">1+</span>numpy<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z)))
+sigma <span style="color: #666666">=</span> sigma_fn(z)
 
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
-        dnn <span style="color: #666666">=</span> DNN_numpy[i][j]
-        
-        train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train) 
-        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, sigma)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.1</span>, <span style="color: #666666">1.1</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;sigmoid function&#39;</span>)
 
-        train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
-        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
-
-        
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
 plt<span style="color: #666666">.</span>show()
 
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Step Function&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.02</span>)
+step_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1.0</span> <span style="color: #008000; font-weight: bold">if</span> z <span style="color: #666666">&gt;=</span> <span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">else</span> <span style="color: #666666">0.0</span>)
+step <span style="color: #666666">=</span> step_fn(z)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, step)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.5</span>, <span style="color: #666666">1.5</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;step function&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Sine Function&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">0.1</span>)
+t <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>sin(z)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, t)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi,<span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;sine function&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Plots a graph of the squashing function used by a rectified linear</span>
+<span style="color: #BA2121; font-style: italic">unit&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2</span>, <span style="color: #666666">2</span>, <span style="color: #666666">.1</span>)
+zero <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>zeros(<span style="color: #008000">len</span>(z))
+y <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>max([zero, z], axis<span style="color: #666666">=0</span>)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, y)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;Rectified linear unit&#39;</span>)
+
 plt<span style="color: #666666">.</span>show()
 </pre>
 </div>
@@ -357,7 +455,7 @@ plt<span style="color: #666666">.</span>show()
   <li><a href="._week41-bs033.html">34</a></li>
   <li><a href="._week41-bs034.html">35</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs026.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs026.html b/doc/pub/week41/html/._week41-bs026.html
index c506265e9..ce15773a9 100644
--- a/doc/pub/week41/html/._week41-bs026.html
+++ b/doc/pub/week41/html/._week41-bs026.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,60 +327,38 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0026"></a>
 <!-- !split -->
-<h2 id="scikit-learn-implementation" class="anchor">scikit-learn implementation </h2>
+<h2 id="the-multilayer-perceptron-mlp" class="anchor">The multilayer  perceptron (MLP) </h2>
 
-<p><b>scikit-learn</b> focuses more
-on traditional machine learning methods, such as regression,
-clustering, decision trees, etc. As such, it has only two types of
-neural networks: Multi Layer Perceptron outputting continuous values,
-<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
-<em>MLPClassifier</em>. We will see how simple it is to use these classes.
+<p>The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of</p>
+<ol>
+<li> A neural network with one or more layers of nodes between the input and the output nodes.</li>
+<li> The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.</li>
+<li> The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.</li>
+</ol>
+<p>As a convention it is normal to call  a  network with one layer of input units, one layer of hidden
+units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
 </p>
 
-<p><b>scikit-learn</b> implements a few improvements from our neural network,
-such as early stopping, a varying learning rate, different
-optimization methods, etc. We would therefore expect a better
-performance overall.
+<p>For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.
+Hereafter we will call the various entities of a layer for nodes.
+There are also no connections within a single layer.
 </p>
 
+<p>The number of input nodes does not need to equal the number of output
+nodes. This applies also to the hidden layers. Each layer may have its
+own number of nodes and activation functions.
+</p>
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
-<span style="color: #408080; font-style: italic"># store models for later use</span>
-DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
-
-<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
-    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
-        dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;logistic&#39;</span>,
-                            alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
-        dnn<span style="color: #666666">.</span>fit(X_train, Y_train)
-        
-        DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
-        
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, dnn<span style="color: #666666">.</span>score(X_test, Y_test))
-        <span style="color: #008000">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
+<p>The hidden layers have their name from the fact that they are not
+linked to observables and as we will see below when we define the
+so-called activation \( \hat{z} \), we can think of this as a basis
+expansion of the original inputs \( \hat{x} \). The difference however
+between neural networks and say linear regression is that now these
+basis functions (which will correspond to the weights in the network)
+are learned from data. This results in  an important difference between
+neural networks and deep learning approaches on one side and methods
+like logistic regression or linear regression and their modifications on the other side.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -354,7 +385,7 @@ DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">
   <li><a href="._week41-bs034.html">35</a></li>
   <li><a href="._week41-bs035.html">36</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs027.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs027.html b/doc/pub/week41/html/._week41-bs027.html
index ae90e77c7..3afbb3fa7 100644
--- a/doc/pub/week41/html/._week41-bs027.html
+++ b/doc/pub/week41/html/._week41-bs027.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,63 +327,30 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0027"></a>
 <!-- !split -->
-<h2 id="visualization" class="anchor">Visualization </h2>
+<h2 id="from-one-to-many-layers-the-universal-approximation-theorem" class="anchor">From one to many layers, the universal approximation theorem </h2>
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># optional</span>
-<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
-<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
+<p>A neural network with only one layer, what we called the simple
+perceptron, is best suited if we have a standard binary model with
+clear (linear) boundaries between the outcomes. As such it could
+equally well be replaced by standard linear regression or logistic
+regression. Networks with one or more hidden layers approximate
+systems with more complex boundaries.
+</p>
 
-sns<span style="color: #666666">.</span>set()
-
-train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
-        dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
-        
-        train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train) 
-        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
-
-        train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
-        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
-
-        
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
+<p>As stated earlier, 
+an important theorem in studies of neural networks, restated without
+proof here, is the <a href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf" target="_self">universal approximation
+theorem</a>.
+</p>
 
+<p>It states that a feed-forward network with a single hidden layer
+containing a finite number of neurons can approximate continuous
+functions on compact subsets of real functions. The theorem thus
+states that simple neural networks can represent a wide variety of
+interesting functions when given appropriate parameters. It is the
+multilayer feedforward architecture itself which gives neural networks
+the potential of being universal approximators.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -357,7 +377,7 @@ plt<span style="color: #666666">.</span>show()
   <li><a href="._week41-bs035.html">36</a></li>
   <li><a href="._week41-bs036.html">37</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs028.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs028.html b/doc/pub/week41/html/._week41-bs028.html
index 03bd5903e..d87130a0b 100644
--- a/doc/pub/week41/html/._week41-bs028.html
+++ b/doc/pub/week41/html/._week41-bs028.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,29 +327,33 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0028"></a>
 <!-- !split -->
-<h2 id="testing-our-code-for-the-xor-or-and-and-gates" class="anchor">Testing our code for the XOR, OR and AND gates </h2>
+<h2 id="deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" class="anchor">Deriving the back propagation code for a multilayer perceptron model </h2>
 
-<p>Last week we discussed three different types of gates, the so-called
-XOR, the OR and the AND gates.  Their inputs and outputs can be
-summarized using the following tables, first for the OR gate with
-inputs \( x_1 \) and \( x_2 \) and outputs \( y \):
+<p>As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.
+The unknowwn quantities are our weights \( w_{ij} \) and we need to find an algorithm for changing them so that our errors are as small as possible.
+This leads us to the famous <a href="https://www.nature.com/articles/323533a0" target="_self">back propagation algorithm</a>.
 </p>
 
-<div class="row">
-  <div class="col-xs-3">
-    <table class="dotable table-striped table-hover table-condensed">
-<thead>
-<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
-</thead>
-<tbody>
-<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
-<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   1          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
-</tbody>
-    </table>
-  </div> <!-- col-xs-3 -->
-</div> <!-- cell row -->
+<p>The questions we want to ask are how do changes in the biases and the
+weights in our network change the cost function and how can we use the
+final output to modify the weights?
+</p>
+
+<p>To derive these equations let us start with a plain regression problem
+and define our cost function as
+</p>
+
+$$
+{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2, 
+$$
+
+<p>where the $t_i$s are our \( n \) targets (the values we want to
+reproduce), while the outputs of the network after having propagated
+all inputs \( \hat{x} \) are given by \( y_i \).  Below we will demonstrate
+how the basic equations arising from the back propagation algorithm
+can be modified in order to study classification problems with \( K \)
+classes.
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -323,7 +380,7 @@ inputs \( x_1 \) and \( x_2 \) and outputs \( y \):
   <li><a href="._week41-bs036.html">37</a></li>
   <li><a href="._week41-bs037.html">38</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs029.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs029.html b/doc/pub/week41/html/._week41-bs029.html
index a9caf511f..5af8fe923 100644
--- a/doc/pub/week41/html/._week41-bs029.html
+++ b/doc/pub/week41/html/._week41-bs029.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,43 +327,41 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0029"></a>
 <!-- !split -->
-<h2 id="the-and-and-xor-gates" class="anchor">The AND and XOR Gates </h2>
+<h2 id="definitions" class="anchor">Definitions </h2>
 
-<p>The AND gate is defined as</p>
+<p>With our definition of the targets \( \hat{t} \), the outputs of the
+network \( \hat{y} \) and the inputs \( \hat{x} \) we
+define now the activation \( z_j^l \) of node/neuron/unit \( j \) of the
+\( l \)-th layer as a function of the bias, the weights which add up from
+the previous layer \( l-1 \) and the forward passes/outputs
+\( \hat{a}^{l-1} \) from the previous layer as
+</p>
 
-<div class="row">
-  <div class="col-xs-3">
-    <table class="dotable table-striped table-hover table-condensed">
-<thead>
-<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
-</thead>
-<tbody>
-<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
-<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   0          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
-</tbody>
-    </table>
-  </div> <!-- col-xs-3 -->
-</div> <!-- cell row -->
+$$
+z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,
+$$
 
-<p>And finally we have the XOR gate</p>
+<p>where \( b_k^l \) are the biases from layer \( l \).  Here \( M_{l-1} \)
+represents the total number of nodes/neurons/units of layer \( l-1 \). The
+figure here illustrates this equation.  We can rewrite this in a more
+compact form as the matrix-vector products we discussed earlier,
+</p>
+
+$$
+\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
+$$
+
+<p>With the activation values \( \hat{z}^l \) we can in turn define the
+output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
+activation function. In the examples here we will use the sigmoid
+function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
+and their nodes.  It means we have
+</p>
+
+$$
+a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
+$$
 
-<div class="row">
-  <div class="col-xs-3">
-    <table class="dotable table-striped table-hover table-condensed">
-<thead>
-<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
-</thead>
-<tbody>
-<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
-<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   1          </td> </tr>
-<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   0          </td> </tr>
-</tbody>
-    </table>
-  </div> <!-- col-xs-3 -->
-</div> <!-- cell row -->
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -337,7 +388,7 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs037.html">38</a></li>
   <li><a href="._week41-bs038.html">39</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs030.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs030.html b/doc/pub/week41/html/._week41-bs030.html
index ce3228bf9..a37692b2e 100644
--- a/doc/pub/week41/html/._week41-bs030.html
+++ b/doc/pub/week41/html/._week41-bs030.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,18 +327,23 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0030"></a>
 <!-- !split -->
-<h2 id="representing-the-data-sets" class="anchor">Representing the Data Sets </h2>
-
-<p>Our design matrix is defined by the input values \( x_1 \) and \( x_2 \). Since we have four possible outputs, our design matrix reads</p>
+<h2 id="derivatives-and-the-chain-rule" class="anchor">Derivatives and the chain rule </h2>
 
+<p>From the definition of the activation \( z_j^l \) we have</p>
 $$
-\boldsymbol{X}=\begin{bmatrix} 0 & 0 \\
-                       0 & 1 \\
-		       1 & 0 \\
-		       1 & 1 \end{bmatrix},
+\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
+$$
+
+<p>and</p>
+$$
+\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. 
+$$
+
+<p>With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))</p>
+$$
+\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)). 
 $$
 
-<p>while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate, \( \boldsymbol{y}^T=[0,0,0,1] \) for the AND gate and \( \boldsymbol{y}^T=[0,1,1,1] \) for the OR gate.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -312,7 +370,7 @@ $$
   <li><a href="._week41-bs038.html">39</a></li>
   <li><a href="._week41-bs039.html">40</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs031.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs031.html b/doc/pub/week41/html/._week41-bs031.html
index a941b3e6f..f3867e85b 100644
--- a/doc/pub/week41/html/._week41-bs031.html
+++ b/doc/pub/week41/html/._week41-bs031.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,98 +327,26 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0031"></a>
 <!-- !split -->
-<h2 id="setting-up-the-neural-network" class="anchor">Setting up the Neural Network </h2>
+<h2 id="derivative-of-the-cost-function" class="anchor">Derivative of the cost function </h2>
 
-<p>We define first our design matrix and the various output vectors for the different gates.</p>
+<p>With these definitions we can now compute the derivative of the cost function in terms of the weights.</p>
 
+<p>Let us specialize to the output layer \( l=L \). Our cost function is</p>
+$$
+{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2, 
+$$
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
-<span style="color: #BA2121; font-style: italic">Simple code that tests XOR, OR and AND gates with linear regression</span>
-<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<p>The derivative of this function with respect to the weights is</p>
 
-<span style="color: #408080; font-style: italic"># import necessary packages</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}}, 
+$$
 
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
-    <span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
+<p>The last partial derivative can easily be computed and reads (by applying the chain rule)</p>
+$$
+\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},  
+$$
 
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(X):
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
-    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
-    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
-    a_h <span style="color: #666666">=</span> sigmoid(z_h)
-    
-    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
-    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
-    <span style="color: #408080; font-style: italic"># softmax output</span>
-    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
-    probabilities <span style="color: #666666">=</span> sigmoid(z_o)
-    <span style="color: #008000; font-weight: bold">return</span> probabilities
-
-<span style="color: #408080; font-style: italic"># we obtain a prediction by taking the class with the highest likelihood</span>
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(X):
-    probabilities <span style="color: #666666">=</span> feed_forward(X)
-    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
-
-<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
-
-<span style="color: #408080; font-style: italic"># Design matrix</span>
-X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
-
-<span style="color: #408080; font-style: italic"># The XOR gate</span>
-yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
-<span style="color: #408080; font-style: italic"># The OR gate</span>
-yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
-<span style="color: #408080; font-style: italic"># The AND gate</span>
-yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
-
-<span style="color: #408080; font-style: italic"># Defining the neural network</span>
-n_inputs, n_features <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape
-n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">2</span>
-n_categories <span style="color: #666666">=</span> <span style="color: #666666">2</span>
-n_features <span style="color: #666666">=</span> <span style="color: #666666">2</span>
-
-<span style="color: #408080; font-style: italic"># we make the weights normally distributed using numpy.random.randn</span>
-
-<span style="color: #408080; font-style: italic"># weights and bias in the hidden layer</span>
-hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_features, n_hidden_neurons)
-hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
-
-<span style="color: #408080; font-style: italic"># weights and bias in the output layer</span>
-output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_hidden_neurons, n_categories)
-output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
-
-probabilities <span style="color: #666666">=</span> feed_forward(X)
-<span style="color: #008000">print</span>(probabilities)
-
-
-predictions <span style="color: #666666">=</span> predict(X)
-<span style="color: #008000">print</span>(predictions)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -392,7 +373,7 @@ predictions <span style="color: #666666">=</span> predict(X)
   <li><a href="._week41-bs039.html">40</a></li>
   <li><a href="._week41-bs040.html">41</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs032.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs032.html b/doc/pub/week41/html/._week41-bs032.html
index bcad4dc8b..57640b4f1 100644
--- a/doc/pub/week41/html/._week41-bs032.html
+++ b/doc/pub/week41/html/._week41-bs032.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,86 +327,49 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0032"></a>
 <!-- !split -->
-<h2 id="the-code-using-scikit-learn" class="anchor">The Code using Scikit-Learn </h2>
+<h2 id="bringing-it-together-first-back-propagation-equation" class="anchor">Bringing it together, first back propagation equation </h2>
 
+<p>We have thus</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1}, 
+$$
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
+<p>Defining</p>
+$$
+\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+$$
 
-<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
+<p>and using the Hadamard product of two vectors we can write this as</p>
+$$
+\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
+$$
 
-<span style="color: #408080; font-style: italic"># Design matrix</span>
-X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+<p>This is an important expression. The second term on the right handside
+measures how fast the cost function is changing as a function of the $j$th
+output activation.  If, for example, the cost function doesn't depend
+much on a particular output node \( j \), then \( \delta_j^L \) will be small,
+which is what we would expect. The first term on the right, measures
+how fast the activation function \( f \) is changing at a given activation
+value \( z_j^L \).
+</p>
 
-<span style="color: #408080; font-style: italic"># The XOR gate</span>
-yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
-<span style="color: #408080; font-style: italic"># The OR gate</span>
-yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
-<span style="color: #408080; font-style: italic"># The AND gate</span>
-yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+<p>Notice that everything in the above equations is easily computed.  In
+particular, we compute \( z_j^L \) while computing the behaviour of the
+network, and it is only a small additional overhead to compute
+\( f'(z^L_j) \).  The exact form of the derivative with respect to the
+output depends on the form of the cost function.
+However, provided the cost function is known there should be little
+trouble in calculating
+</p>
 
-<span style="color: #408080; font-style: italic"># Defining the neural network</span>
-n_inputs, n_features <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape
-n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">2</span>
-n_categories <span style="color: #666666">=</span> <span style="color: #666666">2</span>
-n_features <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+$$
+\frac{\partial {\cal C}}{\partial (a_j^L)}
+$$
 
-eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-<span style="color: #408080; font-style: italic"># store models for later use</span>
-DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
-epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-
-<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
-    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
-        dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;logistic&#39;</span>,
-                            alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
-        dnn<span style="color: #666666">.</span>fit(X, yXOR)
-        DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on data set: &quot;</span>, dnn<span style="color: #666666">.</span>score(X, yXOR))
-        <span style="color: #008000">print</span>()
-
-sns<span style="color: #666666">.</span>set()
-test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
-        dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
-        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X)
-        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(yXOR, test_pred)
-
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
+<p>With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
+$$
 
 
 <p>
@@ -381,7 +397,7 @@ plt<span style="color: #666666">.</span>show()
   <li><a href="._week41-bs040.html">41</a></li>
   <li><a href="._week41-bs041.html">42</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs033.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs033.html b/doc/pub/week41/html/._week41-bs033.html
index e51a2987d..e4634a0ad 100644
--- a/doc/pub/week41/html/._week41-bs033.html
+++ b/doc/pub/week41/html/._week41-bs033.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,18 +327,20 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0033"></a>
 <!-- !split -->
-<h2 id="building-neural-networks-in-tensorflow-and-keras" class="anchor">Building neural networks in Tensorflow and Keras </h2>
+<h2 id="derivatives-in-terms-of-z-j-l" class="anchor">Derivatives in terms of \( z_j^L \) </h2>
 
-<p>Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn
-and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
-and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  
-</p>
+<p>It is also easy to see that our previous equation can be written as</p>
 
-<p>In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
-clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
-NumPy arrays.
-</p>
+$$
+\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
+$$
 
+<p>which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely</p>
+$$
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
+$$
+
+<p>That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias. </p>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -311,7 +366,7 @@ NumPy arrays.
   <li><a href="._week41-bs041.html">42</a></li>
   <li><a href="._week41-bs042.html">43</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs034.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs034.html b/doc/pub/week41/html/._week41-bs034.html
index 5651b65b1..da391efc5 100644
--- a/doc/pub/week41/html/._week41-bs034.html
+++ b/doc/pub/week41/html/._week41-bs034.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,111 +327,63 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0034"></a>
 <!-- !split -->
-<h2 id="tensorflow" class="anchor">Tensorflow </h2>
+<h2 id="bringing-it-together" class="anchor">Bringing it together </h2>
 
-<p>Tensorflow is an open source library machine learning library
-developed by the Google Brain team for internal use. It was released
-under the Apache 2.0 open source license in November 9, 2015.
+<p>We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are</p>
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+
+$$
+\begin{equation}
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
+\tag{13}
+\end{equation}
+$$
+
+<p>and</p>
+$$
+\begin{equation}
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\tag{14}
+\end{equation}
+$$
+
+<p>and</p>
+
+$$
+\begin{equation}
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
+\tag{15}
+\end{equation}
+$$
+</div>
+</div>
+
+
+<p>An interesting consequence of the above equations is that when the
+activation \( a_k^{L-1} \) is small, the gradient term, that is the
+derivative of the cost function with respect to the weights, will also
+tend to be small. We say then that the weight learns slowly, meaning
+that it changes slowly when we minimize the weights via say gradient
+descent. In this case we say the system learns slowly.
 </p>
 
-<p>Tensorflow is a computational framework that allows you to construct
-machine learning models at different levels of abstraction, from
-high-level, object-oriented APIs like Keras, down to the C++ kernels
-that Tensorflow is built upon. The higher levels of abstraction are
-simpler to use, but less flexible, and our choice of implementation
-should reflect the problems we are trying to solve.
+<p>Another interesting feature is that is when the activation function,
+represented by the sigmoid function here, is rather flat when we move towards
+its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
+cases, the derivatives of the activation function will also be close
+to zero, meaning again that the gradients will be small and the
+network learns slowly again.
 </p>
 
-<p><a href="https://www.tensorflow.org/guide/graphs" target="_self">Tensorflow uses</a> so-called graphs to represent your computation
-in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
-to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
+<p>We need a fourth equation and we are set. We are going to propagate
+backwards in order to the determine the weights and biases. In order
+to do so we need to represent the error in the layer before the final
+one \( L-1 \) in terms of the errors in the final output layer.
 </p>
 
-<p>In this guide we will analyze the same data as we did in our NumPy and
-scikit-learn tutorial, gathered from the MNIST database of images. We
-will give an introduction to the lower level Python Application
-Program Interfaces (APIs), and see how we use them to build our graph.
-Then we will build (effectively) the same graph in Keras, to see just
-how simple solving a machine learning problem can be.
-</p>
-
-<p>To install tensorflow on Unix/Linux systems, use pip as</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">pip3 install tensorflow
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
-(current release of CPU-only TensorFlow)
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda create <span style="color: #666666">-</span>n tf tensorflow
-conda activate tf
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>To install the current release of GPU TensorFlow</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda create <span style="color: #666666">-</span>n tf<span style="color: #666666">-</span>gpu tensorflow<span style="color: #666666">-</span>gpu
-conda activate tf<span style="color: #666666">-</span>gpu
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -404,7 +409,7 @@ conda activate tf<span style="color: #666666">-</span>gpu
   <li><a href="._week41-bs042.html">43</a></li>
   <li><a href="._week41-bs043.html">44</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs035.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs035.html b/doc/pub/week41/html/._week41-bs035.html
index d88baaafe..abd1f23ef 100644
--- a/doc/pub/week41/html/._week41-bs035.html
+++ b/doc/pub/week41/html/._week41-bs035.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,38 +327,32 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0035"></a>
 <!-- !split -->
-<h2 id="using-keras" class="anchor">Using Keras </h2>
+<h2 id="final-back-propagating-equation" class="anchor">Final back propagating equation </h2>
 
-<p>Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_self">neural network</a>
-that supports Tensorflow, CTNK and Theano as backends.  
-If you have Anaconda installed you may run the following command
-</p>
+<p>We have that (replacing \( L \) with a general layer \( l \))</p>
+$$
+\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
+$$
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda install keras
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
+<p>We want to express this in terms of the equations for layer \( l+1 \). Using the chain rule and summing over all \( k \) entries we have</p>
 
-<p>You can look up the <a href="https://keras.io/" target="_self">instructions here</a> for more information.</p>
+$$
+\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
+$$
 
-<p>We will to a large extent use <b>keras</b> in this course. </p>
+<p>and recalling that</p>
+$$
+z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
+$$
+
+<p>with \( M_l \) being the number of nodes in layer \( l \), we obtain</p>
+$$
+\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
+$$
+
+<p>This is our final equation.</p>
+
+<p>We are now ready to set up the algorithm for back propagation and learning the weights and biases.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -332,7 +379,7 @@ If you have Anaconda installed you may run the following command
   <li><a href="._week41-bs043.html">44</a></li>
   <li><a href="._week41-bs044.html">45</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs036.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs036.html b/doc/pub/week41/html/._week41-bs036.html
index 572027dff..71ccd62c7 100644
--- a/doc/pub/week41/html/._week41-bs036.html
+++ b/doc/pub/week41/html/._week41-bs036.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,232 +327,75 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0036"></a>
 <!-- !split -->
-<h2 id="collect-and-pre-process-data" class="anchor">Collect and pre-process data </h2>
+<h2 id="setting-up-the-back-propagation-algorithm" class="anchor">Setting up the Back propagation algorithm </h2>
 
-<p>Let us look again at the MINST data set.</p>
+<p>The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
 
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
-
-
-<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
-
-<span style="color: #408080; font-style: italic"># display images in notebook</span>
-<span style="color: #666666">%</span>matplotlib inline
-plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;figure.figsize&#39;</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
-
-
-<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
-digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
-
-<span style="color: #408080; font-style: italic"># define inputs and labels</span>
-inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
-labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># flatten the image</span>
-<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
-n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
-inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;X = (n_inputs, n_features) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># choose some random images to display</span>
-indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
-random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
-
-<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
-    plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
-    plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">&#39;off&#39;</span>)
-    plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>)
-    plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
-plt<span style="color: #666666">.</span>show()
-</pre>
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>First, we set up the input data \( \hat{x} \) and the activations
+\( \hat{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \hat{a}^1 \).
+</p>
 </div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential      <span style="color: #408080; font-style: italic">#This allows appending layers to existing models</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense           <span style="color: #408080; font-style: italic">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> optimizers             <span style="color: #408080; font-style: italic">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers           <span style="color: #408080; font-style: italic">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical   <span style="color: #408080; font-style: italic">#This allows using categorical cross entropy as the cost function</span>
-
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
-
-<span style="color: #408080; font-style: italic"># one-hot representation of labels</span>
-labels <span style="color: #666666">=</span> to_categorical(labels)
-
-<span style="color: #408080; font-style: italic"># split into train and test data</span>
-train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
-test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
-X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
-                                                    test_size<span style="color: #666666">=</span>test_size)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-n_neurons_layer1 <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-n_neurons_layer2 <span style="color: #666666">=</span> <span style="color: #666666">50</span>
-n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
-eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_neural_network_keras</span>(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
-    model <span style="color: #666666">=</span> Sequential()
-    model<span style="color: #666666">.</span>add(Dense(n_neurons_layer1, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;sigmoid&#39;</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
-    model<span style="color: #666666">.</span>add(Dense(n_neurons_layer2, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;sigmoid&#39;</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
-    model<span style="color: #666666">.</span>add(Dense(n_categories, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;softmax&#39;</span>))
-    
-    sgd <span style="color: #666666">=</span> optimizers<span style="color: #666666">.</span>SGD(lr<span style="color: #666666">=</span>eta)
-    model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">&#39;categorical_crossentropy&#39;</span>, optimizer<span style="color: #666666">=</span>sgd, metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">&#39;accuracy&#39;</span>])
-    
-    <span style="color: #008000; font-weight: bold">return</span> model
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">DNN_keras <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
-        
-<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
-    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
-        DNN <span style="color: #666666">=</span> create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
-                                         eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd)
-        DNN<span style="color: #666666">.</span>fit(X_train, Y_train, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size, verbose<span style="color: #666666">=0</span>)
-        scores <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)
-        
-        DNN_keras[i][j] <span style="color: #666666">=</span> DNN
-        
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate = &quot;</span>, eta)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> scores[<span style="color: #666666">1</span>])
-        <span style="color: #008000">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># optional</span>
-<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
-<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
-
-sns<span style="color: #666666">.</span>set()
-
-train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
-        DNN <span style="color: #666666">=</span> DNN_keras[i][j]
-
-        train_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_train, Y_train)[<span style="color: #666666">1</span>]
-        test_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)[<span style="color: #666666">1</span>]
-
-        
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
 </div>
 
 
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \hat{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \hat{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Thereafter we compute the ouput error \( \hat{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+</p>
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -525,7 +421,7 @@ plt<span style="color: #666666">.</span>show()
   <li><a href="._week41-bs044.html">45</a></li>
   <li><a href="._week41-bs045.html">46</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs037.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs037.html b/doc/pub/week41/html/._week41-bs037.html
index d3903df5d..e25c36ca7 100644
--- a/doc/pub/week41/html/._week41-bs037.html
+++ b/doc/pub/week41/html/._week41-bs037.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,196 +327,75 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0037"></a>
 <!-- !split -->
-<h2 id="the-breast-cancer-data-now-with-keras" class="anchor">The Breast Cancer Data, now with Keras </h2>
+<h2 id="setting-up-the-back-propagation-algorithm" class="anchor">Setting up the Back propagation algorithm </h2>
 
+<p>The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
 
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential      <span style="color: #408080; font-style: italic">#This allows appending layers to existing models</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense           <span style="color: #408080; font-style: italic">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> optimizers             <span style="color: #408080; font-style: italic">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers           <span style="color: #408080; font-style: italic">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical   <span style="color: #408080; font-style: italic">#This allows using categorical cross entropy as the cost function</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split <span style="color: #008000; font-weight: bold">as</span> splitter
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pickle</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span> 
-
-
-<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Load breast cancer dataset&quot;&quot;&quot;</span>
-
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)        <span style="color: #408080; font-style: italic">#create same seed for random number every time</span>
-
-cancer<span style="color: #666666">=</span>load_breast_cancer()      <span style="color: #408080; font-style: italic">#Download breast cancer dataset</span>
-
-inputs<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>data                     <span style="color: #408080; font-style: italic">#Feature matrix of 569 rows (samples) and 30 columns (parameters)</span>
-outputs<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>target                  <span style="color: #408080; font-style: italic">#Label array of 569 rows (0 for benign and 1 for malignant)</span>
-labels<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names[<span style="color: #666666">0</span>:<span style="color: #666666">30</span>]
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;The content of the breast cancer dataset is:&#39;</span>)      <span style="color: #408080; font-style: italic">#Print information about the datasets</span>
-<span style="color: #008000">print</span>(labels)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;-------------------------&#39;</span>)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs =  &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;outputs =  &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(outputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels =  &quot;</span><span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
-
-x<span style="color: #666666">=</span>inputs      <span style="color: #408080; font-style: italic">#Reassign the Feature and Label matrices to other variables</span>
-y<span style="color: #666666">=</span>outputs
-
-<span style="color: #408080; font-style: italic">#%% </span>
-
-<span style="color: #408080; font-style: italic"># Visualisation of dataset (for correlation analysis)</span>
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">0</span>],x[:,<span style="color: #666666">2</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean radius&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean perimeter&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">5</span>],x[:,<span style="color: #666666">6</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean compactness&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean concavity&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">0</span>],x[:,<span style="color: #666666">1</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean radius&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean texture&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">2</span>],x[:,<span style="color: #666666">1</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean perimeter&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean compactness&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-
-<span style="color: #408080; font-style: italic"># Generate training and testing datasets</span>
-
-<span style="color: #408080; font-style: italic">#Select features relevant to classification (texture,perimeter,compactness and symmetery) </span>
-<span style="color: #408080; font-style: italic">#and add to input matrix</span>
-
-temp1<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">1</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">1</span>]),<span style="color: #666666">1</span>))
-temp2<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">2</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">2</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((temp1,temp2))      
-temp<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">5</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">5</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((X,temp))       
-temp<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">8</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">8</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((X,temp))       
-
-X_train,X_test,y_train,y_test<span style="color: #666666">=</span>splitter(X,y,test_size<span style="color: #666666">=0.1</span>)   <span style="color: #408080; font-style: italic">#Split datasets into training and testing</span>
-
-y_train<span style="color: #666666">=</span>to_categorical(y_train)     <span style="color: #408080; font-style: italic">#Convert labels to categorical when using categorical cross entropy</span>
-y_test<span style="color: #666666">=</span>to_categorical(y_test)
-
-<span style="color: #008000; font-weight: bold">del</span> temp1,temp2,temp
-
-<span style="color: #408080; font-style: italic"># %%</span>
-
-<span style="color: #408080; font-style: italic"># Define tunable parameters&quot;</span>
-
-eta<span style="color: #666666">=</span>np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-3</span>,<span style="color: #666666">-1</span>,<span style="color: #666666">3</span>)                    <span style="color: #408080; font-style: italic">#Define vector of learning rates (parameter to SGD optimiser)</span>
-lamda<span style="color: #666666">=0.01</span>                                  <span style="color: #408080; font-style: italic">#Define hyperparameter</span>
-n_layers<span style="color: #666666">=2</span>                                  <span style="color: #408080; font-style: italic">#Define number of hidden layers in the model</span>
-n_neuron<span style="color: #666666">=</span>np<span style="color: #666666">.</span>logspace(<span style="color: #666666">0</span>,<span style="color: #666666">3</span>,<span style="color: #666666">4</span>,dtype<span style="color: #666666">=</span><span style="color: #008000">int</span>)       <span style="color: #408080; font-style: italic">#Define number of neurons per layer</span>
-epochs<span style="color: #666666">=100</span>                                   <span style="color: #408080; font-style: italic">#Number of reiterations over the input data</span>
-batch_size<span style="color: #666666">=100</span>                              <span style="color: #408080; font-style: italic">#Number of samples per gradient update</span>
-
-<span style="color: #408080; font-style: italic"># %%</span>
-
-<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Define function to return Deep Neural Network model&quot;&quot;&quot;</span>
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">NN_model</span>(inputsize,n_layers,n_neuron,eta,lamda):
-    model<span style="color: #666666">=</span>Sequential()      
-    <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_layers):       <span style="color: #408080; font-style: italic">#Run loop to add hidden layers to the model</span>
-        <span style="color: #008000; font-weight: bold">if</span> (i<span style="color: #666666">==0</span>):                  <span style="color: #408080; font-style: italic">#First layer requires input dimensions</span>
-            model<span style="color: #666666">.</span>add(Dense(n_neuron,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;relu&#39;</span>,kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lamda),input_dim<span style="color: #666666">=</span>inputsize))
-        <span style="color: #008000; font-weight: bold">else</span>:                       <span style="color: #408080; font-style: italic">#Subsequent layers are capable of automatic shape inferencing</span>
-            model<span style="color: #666666">.</span>add(Dense(n_neuron,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;relu&#39;</span>,kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lamda)))
-    model<span style="color: #666666">.</span>add(Dense(<span style="color: #666666">2</span>,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;softmax&#39;</span>))  <span style="color: #408080; font-style: italic">#2 outputs - ordered and disordered (softmax for prob)</span>
-    sgd<span style="color: #666666">=</span>optimizers<span style="color: #666666">.</span>SGD(lr<span style="color: #666666">=</span>eta)
-    model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">&#39;categorical_crossentropy&#39;</span>,optimizer<span style="color: #666666">=</span>sgd,metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">&#39;accuracy&#39;</span>])
-    <span style="color: #008000; font-weight: bold">return</span> model
-
-    
-Train_accuracy<span style="color: #666666">=</span>np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(n_neuron),<span style="color: #008000">len</span>(eta)))      <span style="color: #408080; font-style: italic">#Define matrices to store accuracy scores as a function</span>
-Test_accuracy<span style="color: #666666">=</span>np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(n_neuron),<span style="color: #008000">len</span>(eta)))       <span style="color: #408080; font-style: italic">#of learning rate and number of hidden neurons for </span>
-
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(n_neuron)):     <span style="color: #408080; font-style: italic">#run loops over hidden neurons and learning rates to calculate </span>
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta)):      <span style="color: #408080; font-style: italic">#accuracy scores </span>
-        DNN_model<span style="color: #666666">=</span>NN_model(X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>],n_layers,n_neuron[i],eta[j],lamda)
-        DNN_model<span style="color: #666666">.</span>fit(X_train,y_train,epochs<span style="color: #666666">=</span>epochs,batch_size<span style="color: #666666">=</span>batch_size,verbose<span style="color: #666666">=1</span>)
-        Train_accuracy[i,j]<span style="color: #666666">=</span>DNN_model<span style="color: #666666">.</span>evaluate(X_train,y_train)[<span style="color: #666666">1</span>]
-        Test_accuracy[i,j]<span style="color: #666666">=</span>DNN_model<span style="color: #666666">.</span>evaluate(X_test,y_test)[<span style="color: #666666">1</span>]
-               
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_data</span>(x,y,data,title<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>):
-
-    <span style="color: #408080; font-style: italic"># plot results</span>
-    fontsize<span style="color: #666666">=16</span>
-
-
-    fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
-    ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
-    cax <span style="color: #666666">=</span> ax<span style="color: #666666">.</span>matshow(data, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=1</span>)
-    
-    cbar<span style="color: #666666">=</span>fig<span style="color: #666666">.</span>colorbar(cax)
-    cbar<span style="color: #666666">.</span>ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&#39;accuracy (%)&#39;</span>,rotation<span style="color: #666666">=90</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    cbar<span style="color: #666666">.</span>set_ticks([<span style="color: #666666">0</span>,<span style="color: #666666">.2</span>,<span style="color: #666666">.4</span>,<span style="color: #666666">0.6</span>,<span style="color: #666666">0.8</span>,<span style="color: #666666">1.0</span>])
-    cbar<span style="color: #666666">.</span>set_ticklabels([<span style="color: #BA2121">&#39;0%&#39;</span>,<span style="color: #BA2121">&#39;20%&#39;</span>,<span style="color: #BA2121">&#39;40%&#39;</span>,<span style="color: #BA2121">&#39;60%&#39;</span>,<span style="color: #BA2121">&#39;80%&#39;</span>,<span style="color: #BA2121">&#39;100%&#39;</span>])
-
-    <span style="color: #408080; font-style: italic"># put text on matrix elements</span>
-    <span style="color: #008000; font-weight: bold">for</span> i, x_val <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(np<span style="color: #666666">.</span>arange(<span style="color: #008000">len</span>(x))):
-        <span style="color: #008000; font-weight: bold">for</span> j, y_val <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(np<span style="color: #666666">.</span>arange(<span style="color: #008000">len</span>(y))):
-            c <span style="color: #666666">=</span> <span style="color: #BA2121">&quot;$</span><span style="color: #BB6688; font-weight: bold">{0:.1f}</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">%$&quot;</span><span style="color: #666666">.</span>format( <span style="color: #666666">100*</span>data[j,i])  
-            ax<span style="color: #666666">.</span>text(x_val, y_val, c, va<span style="color: #666666">=</span><span style="color: #BA2121">&#39;center&#39;</span>, ha<span style="color: #666666">=</span><span style="color: #BA2121">&#39;center&#39;</span>)
-
-    <span style="color: #408080; font-style: italic"># convert axis vaues to to string labels</span>
-    x<span style="color: #666666">=</span>[<span style="color: #008000">str</span>(i) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> x]
-    y<span style="color: #666666">=</span>[<span style="color: #008000">str</span>(i) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> y]
-
-
-    ax<span style="color: #666666">.</span>set_xticklabels([<span style="color: #BA2121">&#39;&#39;</span>]<span style="color: #666666">+</span>x)
-    ax<span style="color: #666666">.</span>set_yticklabels([<span style="color: #BA2121">&#39;&#39;</span>]<span style="color: #666666">+</span>y)
-
-    ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;$</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">mathrm{learning</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121"> rate}$&#39;</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&#39;$</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">mathrm{hidden</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121"> neurons}$&#39;</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    <span style="color: #008000; font-weight: bold">if</span> title <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
-        ax<span style="color: #666666">.</span>set_title(title)
-
-    plt<span style="color: #666666">.</span>tight_layout()
-
-    plt<span style="color: #666666">.</span>show()
-    
-plot_data(eta,n_neuron,Train_accuracy, <span style="color: #BA2121">&#39;training&#39;</span>)
-plot_data(eta,n_neuron,Test_accuracy, <span style="color: #BA2121">&#39;testing&#39;</span>)
-</pre>
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
+\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \boldsymbol{a}^1 \).
+</p>
 </div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
 </div>
 
 
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+</p>
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -489,7 +421,7 @@ plot_data(eta,n_neuron,Test_accuracy, <span style="color: #BA2121">&#39;testing&
   <li><a href="._week41-bs045.html">46</a></li>
   <li><a href="._week41-bs046.html">47</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs038.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs038.html b/doc/pub/week41/html/._week41-bs038.html
index 9d9f1e7b8..5c48f08da 100644
--- a/doc/pub/week41/html/._week41-bs038.html
+++ b/doc/pub/week41/html/._week41-bs038.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,29 +327,75 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0038"></a>
 <!-- !split -->
-<h2 id="fine-tuning-neural-network-hyperparameters" class="anchor">Fine-tuning neural network hyperparameters </h2>
+<h2 id="setting-up-the-back-propagation-algorithm" class="anchor">Setting up the Back propagation algorithm </h2>
 
-<p>The flexibility of neural networks is also one of their main
-drawbacks: there are many hyperparameters to tweak. Not only can you
-use any imaginable network topology (how neurons/nodes are interconnected),
-but even in a simple FFNN you can change the number of layers, the
-number of neurons per layer, the type of activation function to use in
-each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
-know what combination of hyperparameters is the best for your task?
+<p>The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
+\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \boldsymbol{a}^1 \).
+</p>
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+</div>
+
+
+<div class="panel panel-default">
+<div class="panel-body">
+<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
 </p>
 
-<ul>
-<li> You can use grid search with cross-validation to find the right hyperparameters.</li>
-</ul>
-<p>However,since there are many hyperparameters to tune, and since
-training a neural network on a large dataset takes a lot of time, you
-will only be able to explore a tiny part of the hyperparameter space.
-</p>
-
-<ul>
-<li> You can use randomized search.</li>
-<li> Or use tools like <a href="http://oscar.calldesk.ai/" target="_self">Oscar</a>, which implements more complex algorithms to help you find a good set of hyperparameters quickly.</li>  
-</ul>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -322,7 +421,7 @@ will only be able to explore a tiny part of the hyperparameter space.
   <li><a href="._week41-bs046.html">47</a></li>
   <li><a href="._week41-bs047.html">48</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs039.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs039.html b/doc/pub/week41/html/._week41-bs039.html
index 347297ca5..935b64e55 100644
--- a/doc/pub/week41/html/._week41-bs039.html
+++ b/doc/pub/week41/html/._week41-bs039.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,24 +326,43 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0039"></a>
-<!-- !split -->
-<h2 id="hidden-layers" class="anchor">Hidden layers </h2>
+<!-- !split  -->
+<h2 id="setting-up-a-multi-layer-perceptron-model-for-classification" class="anchor">Setting up a Multi-layer perceptron model for classification  </h2>
 
-<p>For many problems you can start with just one or two hidden layers and it will work just fine.
-For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
-few hundred neurons.
-You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
-neurons, in roughly the same amount of training time. 
+<p>We are now gong to develop an example based on the MNIST data
+base. This is a classification problem and we need to use our
+cross-entropy function we discussed in connection with logistic
+regression. The cross-entropy defines our cost function for the
+classificaton problems with neural networks.
 </p>
 
-<p>For more complex problems, you can gradually
-ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
-as large image classification or speech recognition, typically require networks with dozens of layers
-and they need a huge amount
-of training data. However, you will rarely have to train such networks from scratch: it is much more
-common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
+<p>In binary classification with two classes \( (0, 1) \) we define the
+logistic/sigmoid function as the probability that a particular input
+is in class \( 0 \) or \( 1 \).  This is possible because the logistic
+function takes any input from the real numbers and inputs a number
+between 0 and 1, and can therefore be interpreted as a probability. It
+also has other nice properties, such as a derivative that is simple to
+calculate.
 </p>
 
+<p>For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \)
+is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \)
+represents our activation values \( z \). We have
+</p>
+$$
+P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp{(- \boldsymbol{x}})} ,
+$$
+
+<p>and</p>
+$$
+P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) ,
+$$
+
+<p>where \( y \in \{0, 1\} \)  and \( \boldsymbol{\theta} \) represents the weights and biases
+of our network.
+</p>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -316,7 +388,7 @@ common to reuse parts of a pretrained state-of-the-art network that performs a s
   <li><a href="._week41-bs047.html">48</a></li>
   <li><a href="._week41-bs048.html">49</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs040.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs040.html b/doc/pub/week41/html/._week41-bs040.html
index 187bd5c8a..29fa78b6f 100644
--- a/doc/pub/week41/html/._week41-bs040.html
+++ b/doc/pub/week41/html/._week41-bs040.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,33 +326,58 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0040"></a>
-<!-- !split  -->
-<h2 id="which-activation-function-should-i-use" class="anchor">Which activation function should I use? </h2>
+<!-- !split -->
+<h2 id="defining-the-cost-function" class="anchor">Defining the cost function </h2>
 
-<p>The Back propagation algorithm we derived above works by going from
-the output layer to the input layer, propagating the error gradient on
-the way. Once the algorithm has computed the gradient of the cost
-function with regards to each parameter in the network, it uses these
-gradients to update each parameter with a Gradient Descent (GD) step.
+<p>Our cost function is given as (see the Logistic regression lectures)</p>
+$$
+\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n
+y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) .
+$$
+
+<p>This last equality means that we can interpret our <em>cost</em> function as a sum over the <em>loss</em> function
+for each point in the dataset \( \mathcal{L}_i(\boldsymbol{\theta}) \).  
+The negative sign is just so that we can think about our algorithm as minimizing a positive number, rather
+than maximizing a negative number.  
 </p>
 
-<p>Unfortunately for us, the gradients often get smaller and smaller as the
-algorithm progresses down to the first hidden layers. As a result, the
-GD update leaves the lower layer connection weights
-virtually unchanged, and training never converges to a good
-solution. This is known in the literature as 
-<b>the vanishing gradients problem</b>. 
+<p>In <em>multiclass</em> classification it is common to treat each integer label as a so called <em>one-hot</em> vector:  </p>
+
+<p>\( y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) , \) and</p>
+
+\( y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) , \)
+
+<p>i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset (numbers from \( 0 \) to \( 9 \))..  </p>
+
+<p>If \( \boldsymbol{x}_i \) is the \( i \)-th input (image), \( y_{ic} \) refers to the \( c \)-th component of the \( i \)-th
+output vector \( \boldsymbol{y}_i \).  
+The probability of \( \boldsymbol{x}_i \) being in class \( c \) will be given by the softmax function:  
 </p>
 
-<p>In other cases, the opposite can happen, namely the the gradients can grow bigger and
-bigger. The result is that many of the layers get large updates of the 
-weights the
-algorithm diverges. This is the <b>exploding gradients problem</b>, which is
-mostly encountered in recurrent neural networks. More generally, deep
-neural networks suffer from unstable gradients, different layers may
-learn at widely different speeds
+$$
+P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}}
+{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} ,
+$$
+
+<p>which reduces to the logistic function in the binary case.  
+The likelihood of this \( C \)-class classifier
+is now given as:  
 </p>
 
+$$
+P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} .
+$$
+
+<p>Again we take the negative log-likelihood to define our cost function:  </p>
+
+$$
+\mathcal{C}(\boldsymbol{\theta}) = - \log{P(\mathcal{D} \mid \boldsymbol{\theta})}.
+$$
+
+<p>See the logistic regression lectures for a full definition of the cost function.</p>
+
+<p>The back propagation equations need now only a small change, namely the definition of a new cost function. We are thus ready to use the same equations as before!</p>
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -325,7 +403,7 @@ learn at widely different speeds
   <li><a href="._week41-bs048.html">49</a></li>
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs041.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs041.html b/doc/pub/week41/html/._week41-bs041.html
index eb3668b18..4462c1a98 100644
--- a/doc/pub/week41/html/._week41-bs041.html
+++ b/doc/pub/week41/html/._week41-bs041.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,33 +326,51 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0041"></a>
-<!-- !split  -->
-<h2 id="is-the-logistic-activation-function-sigmoid-our-choice" class="anchor">Is the Logistic activation function (Sigmoid)  our choice? </h2>
+<!-- !split -->
+<h2 id="example-binary-classification-problem" class="anchor">Example: binary classification problem </h2>
 
-<p>Although this unfortunate behavior has been empirically observed for
-quite a while (it was one of the reasons why deep neural networks were
-mostly abandoned for a long time), it is only around 2010 that
-significant progress was made in understanding it.
-</p>
+<p>As an example of the above, relevant for project 2 as well, let us consider a binary class. As discussed in our logistic regression lectures, we defined a cost function in terms of the parameters \( \beta \) as</p>
+$$
+\mathcal{C}(\boldsymbol{\beta}) = - \sum_{i=1}^n \left(y_i\log{p(y_i \vert x_i,\boldsymbol{\beta})}+(1-y_i)\log{1-p(y_i \vert x_i,\boldsymbol{\beta})}\right),
+$$
 
-<p>A paper titled <a href="http://proceedings.mlr.press/v9/glorot10a.html" target="_self">Understanding the Difficulty of Training Deep
-Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio</a> found that
-the problems with the popular logistic
-sigmoid activation function and the weight initialization technique
-that was most popular at the time, namely random initialization using
-a normal distribution with a mean of 0 and a standard deviation of
-1. 
-</p>
+<p>where we had defined the logistic (sigmoid) function</p>
+$$
+p(y_i =1\vert x_i,\boldsymbol{\beta})=\frac{\exp{(\beta_0+\beta_1 x_i)}}{1+\exp{(\beta_0+\beta_1 x_i)}},
+$$
 
-<p>They showed that with this activation function and this
-initialization scheme, the variance of the outputs of each layer is
-much greater than the variance of its inputs. Going forward in the
-network, the variance keeps increasing after each layer until the
-activation function saturates at the top layers. This is actually made
-worse by the fact that the logistic function has a mean of 0.5, not 0
-(the hyperbolic tangent function has a mean of 0 and behaves slightly
-better than the logistic function in deep networks).
+<p>and</p>
+$$
+p(y_i =0\vert x_i,\boldsymbol{\beta})=1-p(y_i =1\vert x_i,\boldsymbol{\beta}).
+$$
+
+<p>The parameters \( \boldsymbol{\beta} \) were defined using a minimization method like gradient descent or Newton-Raphson's method. </p>
+
+<p>Now we replace \( x_i \) with the activation \( z_i^l \) for a given layer \( l \) and the outputs as \( y_i=a_i^l=f(z_i^l) \), with \( z_i^l \) now being a function of the weights \( w_{ij}^l \) and biases \( b_i^l \). 
+We have then
 </p>
+$$
+a_i^l = y_i = \frac{\exp{(z_i^l)}}{1+\exp{(z_i^l)}},
+$$
+
+<p>with </p>
+$$
+z_i^l = \sum_{j}w_{ij}^l a_j^{l-1}+b_i^l,
+$$
+
+<p>where the superscript \( l-1 \) indicates that these are the outputs from layer \( l-1 \).
+Our cost function at the final layer \( l=L \) is now
+</p>
+$$
+\mathcal{C}(\boldsymbol{W}) = - \sum_{i=1}^n \left(t_i\log{a_i^L}+(1-t_i)\log{(1-a_i^L)}\right),
+$$
+
+<p>where we have defined the targets \( t_i \). The derivatives of the cost function with respect to the output \( a_i^L \) are then easily calculated and we get</p>
+$$
+\frac{\partial \mathcal{C}(\boldsymbol{W})}{\partial a_i^L} = \frac{a_i^L-t_i}{a_i^L(1-a_i^L)}. 
+$$
+
+<p>In case we use another activation function than the logistic one, we need to evaluate other derivatives. </p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -326,7 +397,7 @@ better than the logistic function in deep networks).
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs042.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs042.html b/doc/pub/week41/html/._week41-bs042.html
index 60bec415a..8b6226082 100644
--- a/doc/pub/week41/html/._week41-bs042.html
+++ b/doc/pub/week41/html/._week41-bs042.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,38 +327,24 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0042"></a>
 <!-- !split -->
-<h2 id="the-derivative-of-the-logistic-funtion" class="anchor">The derivative of the Logistic funtion </h2>
+<h2 id="the-softmax-function" class="anchor">The Softmax function </h2>
+<p>In case we employ the more general case given by the Softmax equation, we need to evaluate the derivative of the activation function with respect to the activation \( z_i^l \), that is we need</p>
+$$
+\frac{\partial f(z_i^l)}{\partial w_{jk}^l} =
+\frac{\partial f(z_i^l)}{\partial z_j^l} \frac{\partial z_j^l}{\partial w_{jk}^l}= \frac{\partial f(z_i^l)}{\partial z_j^l}a_k^{l-1}.
+$$
 
-<p>Looking at the logistic activation function, when inputs become large
-(negative or positive), the function saturates at 0 or 1, with a
-derivative extremely close to 0. Thus when backpropagation kicks in,
-it has virtually no gradient to propagate back through the network,
-and what little gradient exists keeps getting diluted as
-backpropagation progresses down through the top layers, so there is
-really nothing left for the lower layers.
-</p>
+<p>For the Softmax function we have</p>
+$$
+f(z_i^l) = \frac{\exp{(z_i^l)}}{\sum_{m=1}^K\exp{(z_m^l)}}.
+$$
 
-<p>In their paper, Glorot and Bengio propose a way to significantly
-alleviate this problem. We need the signal to flow properly in both
-directions: in the forward direction when making predictions, and in
-the reverse direction when backpropagating gradients. We don&#8217;t want
-the signal to die out, nor do we want it to explode and saturate. For
-the signal to flow properly, the authors argue that we need the
-variance of the outputs of each layer to be equal to the variance of
-its inputs, and we also need the gradients to have equal variance
-before and after flowing through a layer in the reverse direction.
-</p>
+<p>Its derivative with respect to \( z_j^l \) gives </p>
+$$
+\frac{\partial f(z_i^l)}{\partial z_j^l}= f(z_i^l)\left(\delta_{ij}-f(z_j^l)\right), 
+$$
 
-<p>One of the insights in the 2010 paper by Glorot and Bengio was that
-the vanishing/exploding gradients problems were in part due to a poor
-choice of activation function. Until then most people had assumed that
-if Nature had chosen to use roughly sigmoid activation functions in
-biological neurons, they must be an excellent choice. But it turns out
-that other activation functions behave much better in deep neural
-networks, in particular the ReLU activation function, mostly because
-it does not saturate for positive values (and also because it is quite
-fast to compute).
-</p>
+<p>which in case of the simply binary model reduces to  having \( i=j \). </p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -331,6 +370,8 @@ fast to compute).
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs043.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs043.html b/doc/pub/week41/html/._week41-bs043.html
index b82ea26c7..a00096a31 100644
--- a/doc/pub/week41/html/._week41-bs043.html
+++ b/doc/pub/week41/html/._week41-bs043.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,32 +326,19 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0043"></a>
-<!-- !split -->
-<h2 id="the-relu-function-family" class="anchor">The RELU function family </h2>
-
-<p>The ReLU activation function suffers from a problem known as the dying
-ReLUs: during training, some neurons effectively die, meaning they
-stop outputting anything other than 0.
-</p>
-
-<p>In some cases, you may find that half of your network&#8217;s neurons are
-dead, especially if you used a large learning rate. During training,
-if a neuron&#8217;s weights get updated such that the weighted sum of the
-neuron&#8217;s inputs is negative, it will start outputting 0. When this
-happen, the neuron is unlikely to come back to life since the gradient
-of the ReLU function is 0 when its input is negative.
-</p>
-
-<p>To solve this problem, nowadays practitioners use a  variant of the ReLU
-function, such as the leaky ReLU discussed above or the so-called
-exponential linear unit (ELU) function
-</p>
-
-$$
-ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\  z & z \ge 0.\end{array}\right. 
-$$
+<!-- !split  -->
+<h2 id="developing-a-code-for-doing-neural-networks-with-back-propagation" class="anchor">Developing a code for doing neural networks with back propagation </h2>
 
+<p>One can identify a set of key steps when using neural networks to solve supervised learning problems:  </p>
 
+<ol>
+<li> Collect and pre-process data</li>  
+<li> Define model and architecture</li>  
+<li> Choose cost function and optimizer</li>  
+<li> Train the model</li>  
+<li> Evaluate model performance on test data</li>  
+<li> Adjust hyperparameters (if necessary, network architecture)</li>
+</ol>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -322,6 +362,9 @@ $$
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs044.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs044.html b/doc/pub/week41/html/._week41-bs044.html
index a38ddb8d0..27b5209d6 100644
--- a/doc/pub/week41/html/._week41-bs044.html
+++ b/doc/pub/week41/html/._week41-bs044.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,23 +327,117 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0044"></a>
 <!-- !split -->
-<h2 id="which-activation-function-should-we-use" class="anchor">Which activation function should we use? </h2>
+<h2 id="collect-and-pre-process-data" class="anchor">Collect and pre-process data </h2>
 
-<p>In general it seems that the ELU activation function is better than
-the leaky ReLU function (and its variants), which is better than
-ReLU. ReLU performs better than \( \tanh \) which in turn performs better
-than the logistic function. 
+<p>Here we will be using the MNIST dataset, which is readily available through the <b>scikit-learn</b>
+package. You may also find it for example <a href="http://yann.lecun.com/exdb/mnist/" target="_self">here</a>.  
+The <em>MNIST</em> (Modified National Institute of Standards and Technology) database is a large database
+of handwritten digits that is commonly used for training various image processing systems.  
+The MNIST dataset consists of 70 000 images of size \( 28\times 28 \) pixels, each labeled from 0 to 9.  
+The scikit-learn dataset we will use consists of a selection of 1797 images of size \( 8\times 8 \) collected and processed from this database.  
 </p>
 
-<p>If runtime
-performance is an issue, then you may opt for the  leaky ReLU function  over the 
-ELU function If you don&#8217;t
-want to tweak yet another hyperparameter, you may just use the default
-\( \alpha \) of \( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have
-spare time and computing power, you can use cross-validation or
-bootstrap to evaluate other activation functions.
+<p>To feed data into a feed-forward neural network we need to represent
+the inputs as a design/feature matrix \( X = (n_{inputs}, n_{features}) \).  Each
+row represents an <em>input</em>, in this case a handwritten digit, and
+each column represents a <em>feature</em>, in this case a pixel.  The
+correct answers, also known as <em>labels</em> or <em>targets</em> are
+represented as a 1D array of integers 
+\( Y = (n_{inputs}) = (5, 3, 1, 8,...) \).
 </p>
 
+<p>As an example, say we want to build a neural network using supervised learning to predict Body-Mass Index (BMI) from
+measurements of height (in m)  
+and weight (in kg). If we have measurements of 5 people the design/feature matrix could be for example:  
+</p>
+
+<p>$$ X = \begin{bmatrix}
+1.85 &amp; 81\\
+1.71 &amp; 65\\
+1.95 &amp; 103\\
+1.55 &amp; 42\\
+1.63 &amp; 56
+\end{bmatrix} ,$$  
+</p>
+
+<p>and the targets would be:  </p>
+
+<p>$$ Y = (23.7, 22.2, 27.1, 17.5, 21.1) $$  </p>
+
+<p>Since each input image is a 2D matrix, we need to flatten the image
+(i.e. "unravel" the 2D matrix into a 1D array) to turn the data into a
+design/feature matrix. This means we lose all spatial information in the
+image, such as locality and translational invariance. More complicated
+architectures such as Convolutional Neural Networks can take advantage
+of such information, and are most commonly applied when analyzing
+images.
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
+
+
+<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
+np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
+
+<span style="color: #408080; font-style: italic"># display images in notebook</span>
+<span style="color: #666666">%</span>matplotlib inline
+plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;figure.figsize&#39;</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
+
+
+<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
+digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
+
+<span style="color: #408080; font-style: italic"># define inputs and labels</span>
+inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
+labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
+
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
+
+
+<span style="color: #408080; font-style: italic"># flatten the image</span>
+<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
+n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
+inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;X = (n_inputs, n_features) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
+
+
+<span style="color: #408080; font-style: italic"># choose some random images to display</span>
+indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
+random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
+
+<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
+    plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
+    plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">&#39;off&#39;</span>)
+    plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>)
+    plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
+plt<span style="color: #666666">.</span>show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -313,6 +460,10 @@ bootstrap to evaluate other activation functions.
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs045.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs045.html b/doc/pub/week41/html/._week41-bs045.html
index c9d2bc7ec..c67a397a0 100644
--- a/doc/pub/week41/html/._week41-bs045.html
+++ b/doc/pub/week41/html/._week41-bs045.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,18 +327,69 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0045"></a>
 <!-- !split -->
-<h2 id="more-on-activation-functions-output-layers" class="anchor">More on activation functions, output layers </h2>
+<h2 id="train-and-test-datasets" class="anchor">Train and test datasets </h2>
 
-<p>In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).</p>
+<p>Performing analysis before partitioning the dataset is a major error, that can lead to incorrect conclusions.  </p>
 
-<p>It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.</p>
+<p>We will reserve \( 80 \% \) of our dataset for training and \( 20 \% \) for testing.  </p>
+
+<p>It is important that the train and test datasets are drawn randomly from our dataset, to ensure
+no bias in the sampling.  
+Say you are taking measurements of weather data to predict the weather in the coming 5 days.
+You don't want to train your model on measurements taken from the hours 00.00 to 12.00, and then test it on data
+collected from 12.00 to 24.00.
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
+
+<span style="color: #408080; font-style: italic"># one-liner from scikit-learn library</span>
+train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
+test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
+X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
+                                                    test_size<span style="color: #666666">=</span>test_size)
+
+<span style="color: #408080; font-style: italic"># equivalently in numpy</span>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train_test_split_numpy</span>(inputs, labels, train_size, test_size):
+    n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
+    inputs_shuffled <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>copy()
+    labels_shuffled <span style="color: #666666">=</span> labels<span style="color: #666666">.</span>copy()
+    
+    np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>shuffle(inputs_shuffled)
+    np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>shuffle(labels_shuffled)
+    
+    train_end <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n_inputs<span style="color: #666666">*</span>train_size)
+    X_train, X_test <span style="color: #666666">=</span> inputs_shuffled[:train_end], inputs_shuffled[train_end:]
+    Y_train, Y_test <span style="color: #666666">=</span> labels_shuffled[:train_end], labels_shuffled[train_end:]
+    
+    <span style="color: #008000; font-weight: bold">return</span> X_train, X_test, Y_train, Y_test
+
+<span style="color: #408080; font-style: italic">#X_train, X_test, Y_train, Y_test = train_test_split_numpy(inputs, labels, train_size, test_size)</span>
+
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Number of training images: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(<span style="color: #008000">len</span>(X_train)))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Number of test images: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(<span style="color: #008000">len</span>(X_test)))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<b>For the output layer:</b>
 
-<ul>
-<li> For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).</li>
-<li> For regression tasks, you can simply use no activation function at all.</li>
-</ul>
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -307,6 +411,11 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs046.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs046.html b/doc/pub/week41/html/._week41-bs046.html
index 8d5465484..62e7b6f63 100644
--- a/doc/pub/week41/html/._week41-bs046.html
+++ b/doc/pub/week41/html/._week41-bs046.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,22 +327,47 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0046"></a>
 <!-- !split -->
-<h2 id="batch-normalization" class="anchor">Batch Normalization </h2>
+<h2 id="define-model-and-architecture" class="anchor">Define model and architecture </h2>
 
-<p>Batch Normalization
-aims to address the vanishing/exploding gradients problems, and more generally the problem that the
-distribution of each layer&#8217;s inputs changes during training, as the parameters of the previous layers change.
+<p>Our simple feed-forward neural network will consist of an <em>input</em> layer, a single <em>hidden</em> layer and an <em>output</em> layer. The activation \( y \) of each neuron is a weighted sum of inputs, passed through an activation function. In case of the simple perceptron model we have </p>
+
+<p>$$ z = \sum_{i=1}^n w_i a_i ,$$</p>
+
+<p>$$ y = f(z) ,$$</p>
+
+<p>where \( f \) is the activation function, \( a_i \) represents input from neuron \( i \) in the preceding layer
+and \( w_i \) is the weight to input \( i \).  
+The activation of the neurons in the input layer is just the features (e.g. a pixel value).  
 </p>
 
-<p>The technique consists of adding an operation in the model just before the activation function of each
-layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
-parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
-learn the optimal scale and mean of the inputs for each layer.
-In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs&#8217; mean and
-standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
-mini-batch, from this the name batch normalization.
+<p>The simplest activation function for a neuron is the <em>Heaviside</em> function:</p>
+
+<p>$$ f(z) = 
+\begin{cases}
+1,  &  z > 0\\
+0,  & \text{otherwise}
+\end{cases}
+$$
 </p>
 
+<p>A feed-forward neural network with this activation is known as a <em>perceptron</em>.  
+For a binary classifier (i.e. two classes, 0 or 1, dog or not-dog) we can also use this in our output layer.  
+This activation can be generalized to \( k \) classes (using e.g. the <em>one-against-all</em> strategy), 
+and we call these architectures <em>multiclass perceptrons</em>.  
+</p>
+
+<p>However, it is now common to use the terms Single Layer Perceptron (SLP) (1 hidden layer) and  
+Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to feed-forward neural networks with any activation function.  
+</p>
+
+<p>Typical choices for activation functions include the sigmoid function, hyperbolic tangent, and Rectified Linear Unit (ReLU).  
+We will be using the sigmoid function \( \sigma(x) \):  
+</p>
+
+<p>$$ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,$$</p>
+
+<p>which is inspired by probability theory (see logistic regression) and was most commonly used until about 2011. See the discussion below concerning other activation functions.</p>
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -310,6 +388,12 @@ mini-batch, from this the name batch normalization.
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs047.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs047.html b/doc/pub/week41/html/._week41-bs047.html
index d831aac1c..b4be5339b 100644
--- a/doc/pub/week41/html/._week41-bs047.html
+++ b/doc/pub/week41/html/._week41-bs047.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,17 +326,45 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0047"></a>
-<!-- !split -->
-<h2 id="dropout" class="anchor">Dropout </h2>
+<!-- !split   -->
+<h2 id="layers" class="anchor">Layers </h2>
 
-<p>It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
-excluding the output neurons) has a probability \( p \) of being temporarily dropped out, meaning it will be
-entirely ignored during this training step, but it may be active during the next step.
+<ul>
+<li> Input</li> 
+</ul>
+<p>Since each input image has 8x8 = 64 pixels or features, we have an input layer of 64 neurons.  </p>
+
+<ul>
+<li> Hidden layer</li>
+</ul>
+<p>We will use 50 neurons in the hidden layer receiving input from the neurons in the input layer.  
+Since each neuron in the hidden layer is connected to the 64 inputs we have 64x50 = 3200 weights to the hidden layer.  
 </p>
 
-<p>The
-hyperparameter \( p \) is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
- It is viewed as one of the most popular regularization techniques.
+<ul>
+<li> Output</li>
+</ul>
+<p>If we were building a binary classifier, it would be sufficient with a single neuron in the output layer,
+which could output 0 or 1 according to the Heaviside function. This would be an example of a <em>hard</em> classifier, meaning it outputs the class of the input directly. However, if we are dealing with noisy data it is often beneficial to use a <em>soft</em> classifier, which outputs the probability of being in class 0 or 1.  
+</p>
+
+<p>For a soft binary classifier, we could use a single neuron and interpret the output as either being the probability of being in class 0 or the probability of being in class 1. Alternatively we could use 2 neurons, and interpret each neuron as the probability of being in each class.  </p>
+
+<p>Since we are doing multiclass classification, with 10 categories, it is natural to use 10 neurons in the output layer. We number the neurons \( j = 0,1,...,9 \). The activation of each output neuron \( j \) will be according to the <em>softmax</em> function:  </p>
+
+<p>$$ P(\text{class \( j \)} \mid \text{input \( \boldsymbol{a} \)}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}}
+{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,$$  
+</p>
+
+<p>i.e. each neuron \( j \) outputs the probability of being in class \( j \) given an input from the hidden layer \( \boldsymbol{a} \), with \( \boldsymbol{w}_j \) the weights of neuron \( j \) to the inputs.  
+The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.  
+The exponent is just the weighted sum of inputs as before:  
+</p>
+
+<p>$$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$  </p>
+
+<p>Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500
+weights to the output layer.
 </p>
 
 <p>
@@ -305,6 +386,13 @@ hyperparameter \( p \) is called the dropout rate, and it is typically set to 50
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="._week41-bs056.html">57</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs048.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs048.html b/doc/pub/week41/html/._week41-bs048.html
index 687842df7..7c2295ea6 100644
--- a/doc/pub/week41/html/._week41-bs048.html
+++ b/doc/pub/week41/html/._week41-bs048.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,20 +326,58 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0048"></a>
-<!-- !split -->
-<h2 id="gradient-clipping" class="anchor">Gradient Clipping </h2>
+<!-- !split   -->
+<h2 id="weights-and-biases" class="anchor">Weights and biases </h2>
 
-<p>A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
-backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
-networks).
+<p>Typically weights are initialized with small values distributed around zero, drawn from a uniform
+or normal distribution. Setting all weights to zero means all neurons give the same output, making the network useless.  
 </p>
 
-<p>This technique is called Gradient Clipping.</p>
-
-<p>In general however, Batch
-Normalization is preferred.
+<p>Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range
+of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):  
 </p>
 
+<p>$$ z_j = \sum_{i=1}^n w_ {ij} a_i + b_j.$$  </p>
+
+<p>The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.</p>
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># building our neural network</span>
+
+n_inputs, n_features <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape
+n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">50</span>
+n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
+
+<span style="color: #408080; font-style: italic"># we make the weights normally distributed using numpy.random.randn</span>
+
+<span style="color: #408080; font-style: italic"># weights and bias in the hidden layer</span>
+hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_features, n_hidden_neurons)
+hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+<span style="color: #408080; font-style: italic"># weights and bias in the output layer</span>
+output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_hidden_neurons, n_categories)
+output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -305,6 +396,14 @@ Normalization is preferred.
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="._week41-bs056.html">57</a></li>
+  <li><a href="._week41-bs057.html">58</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs049.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs049.html b/doc/pub/week41/html/._week41-bs049.html
index bda2e4540..b1e1825fb 100644
--- a/doc/pub/week41/html/._week41-bs049.html
+++ b/doc/pub/week41/html/._week41-bs049.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,9 +327,27 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0049"></a>
 <!-- !split -->
-<h2 id="a-very-nice-website-on-neural-networks" class="anchor">A very nice website on Neural Networks </h2>
+<h2 id="feed-forward-pass" class="anchor">Feed-forward pass </h2>
 
-<p>You may find this <a href="https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false" target="_self">website</a> very useful. </p>
+<p>Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories.  
+For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):  
+</p>
+
+<p>$$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$</p>
+
+<p>this is then passed through our activation function  </p>
+
+<p>$$ a_{j}^{l} = f(z_{j}^{l}) .$$  </p>
+
+<p>We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:  </p>
+
+<p>$$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$  </p>
+
+<p>Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:  </p>
+
+<p>$$ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
+{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$  
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -295,6 +366,15 @@ MathJax.Hub.Config({
   <li class="active"><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="._week41-bs056.html">57</a></li>
+  <li><a href="._week41-bs057.html">58</a></li>
+  <li><a href="._week41-bs058.html">59</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs050.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs050.html b/doc/pub/week41/html/._week41-bs050.html
index 2f88d95ab..60699d7e7 100644
--- a/doc/pub/week41/html/._week41-bs050.html
+++ b/doc/pub/week41/html/._week41-bs050.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -273,46 +326,96 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0050"></a>
-<!-- !split  -->
-<h2 id="a-top-down-perspective-on-neural-networks" class="anchor">A top-down perspective on Neural networks </h2>
+<!-- !split    -->
+<h2 id="matrix-multiplications" class="anchor">Matrix multiplications </h2>
 
-<p>The first thing we would like to do is divide the data into two or three
-parts. A training set, a validation or dev (development) set, and a
-test set. The test set is the data on which we want to make
-predictions. The dev set is a subset of the training data we use to
-check how well we are doing out-of-sample, after training the model on
-the training dataset. We use the validation error as a proxy for the
-test error in order to make tweaks to our model. It is crucial that we
-do not use any of the test data to train the algorithm. This is a
-cardinal sin in ML. Then:
+<p>Since our data has the dimensions \( X = (n_{inputs}, n_{features}) \) and our weights to the hidden
+layer have the dimensions  
+\( W_{hidden} = (n_{features}, n_{hidden}) \),
+we can easily feed the network all our training data in one go by taking the matrix product  
 </p>
 
-<ul>
-<li> Estimate optimal error rate</li>
-<li> Minimize underfitting (bias) on training data set.</li>
-<li> Make sure you are not overfitting.</li>
-</ul>
-<p>If the validation and test sets are drawn from the same distributions,
-then a good performance on the validation set should lead to similarly
-good performance on the test set. 
+<p>$$ X W^{h} = (n_{inputs}, n_{hidden}),$$ </p>
+
+<p>and obtain a matrix that holds the weighted sum of inputs to the hidden layer
+for each input image and each hidden neuron.    
+We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):  
 </p>
 
-<p>However, sometimes
-the training data and test data differ in subtle ways because, for
-example, they are collected using slightly different methods, or
-because it is cheaper to collect data in one way versus another. In
-this case, there can be a mismatch between the training and test
-data. This can lead to the neural network overfitting these small
-differences between the test and training sets, and a poor performance
-on the test set despite having a good performance on the validation
-set. To rectify this, Andrew Ng suggests making two validation or dev
-sets, one constructed from the training data and one constructed from
-the test data. The difference between the performance of the algorithm
-on these two validation sets quantifies the train-test mismatch. This
-can serve as another important diagnostic when using DNNs for
-supervised learning.
+<p>$$ \boldsymbol{z}^{l} = \boldsymbol{X} \boldsymbol{W}^{l} + \boldsymbol{b}^{l} ,$$</p>
+
+<p>meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image.  
+This is then passed through the activation:  
 </p>
 
+<p>$$ \boldsymbol{a}^{l} = f(\boldsymbol{z}^l) .$$  </p>
+
+<p>This is fed to the output layer:  </p>
+
+<p>$$ \boldsymbol{z}^{L} = \boldsymbol{a}^{L} \boldsymbol{W}^{L} + \boldsymbol{b}^{L} .$$</p>
+
+<p>Finally we receive our output values for each image and each category by passing it through the softmax function:  </p>
+
+<p>$$ output = softmax (\boldsymbol{z}^{L}) = (n_{inputs}, n_{categories}) .$$</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># setup the feed-forward pass, subscript h = hidden layer</span>
+
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
+    <span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
+
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(X):
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
+    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
+    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
+    a_h <span style="color: #666666">=</span> sigmoid(z_h)
+    
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
+    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
+    <span style="color: #408080; font-style: italic"># softmax output</span>
+    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
+    exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
+    probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
+    
+    <span style="color: #008000; font-weight: bold">return</span> probabilities
+
+probabilities <span style="color: #666666">=</span> feed_forward(X_train)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probabilities = (n_inputs, n_categories) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities<span style="color: #666666">.</span>shape))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probability that image 0 is in category 0,1,2,...,9 = </span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities[<span style="color: #666666">0</span>]))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;probabilities sum up to: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(probabilities[<span style="color: #666666">0</span>]<span style="color: #666666">.</span>sum()))
+<span style="color: #008000">print</span>()
+
+<span style="color: #408080; font-style: italic"># we obtain a prediction by taking the class with the highest likelihood</span>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(X):
+    probabilities <span style="color: #666666">=</span> feed_forward(X)
+    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
+
+predictions <span style="color: #666666">=</span> predict(X_train)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;predictions = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(predictions<span style="color: #666666">.</span>shape))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;prediction for image 0: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(predictions[<span style="color: #666666">0</span>]))
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;correct label for image 0: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(Y_train[<span style="color: #666666">0</span>]))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -329,6 +432,16 @@ supervised learning.
   <li><a href="._week41-bs049.html">50</a></li>
   <li class="active"><a href="._week41-bs050.html">51</a></li>
   <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="._week41-bs056.html">57</a></li>
+  <li><a href="._week41-bs057.html">58</a></li>
+  <li><a href="._week41-bs058.html">59</a></li>
+  <li><a href="._week41-bs059.html">60</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs051.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs051.html b/doc/pub/week41/html/._week41-bs051.html
index 67210f960..8dc2f7187 100644
--- a/doc/pub/week41/html/._week41-bs051.html
+++ b/doc/pub/week41/html/._week41-bs051.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -274,26 +327,32 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0051"></a>
 <!-- !split -->
-<h2 id="limitations-of-supervised-learning-with-deep-networks" class="anchor">Limitations of supervised learning with deep networks </h2>
+<h2 id="choose-cost-function-and-optimizer" class="anchor">Choose cost function and optimizer </h2>
 
-<p>Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
+<p>To measure how well our neural network is doing we need to introduce a cost function.  
+We will call the function that gives the error of a single sample output the <em>loss</em> function, and the function
+that gives the total error of our network across all samples the <em>cost</em> function.
+A typical choice for multiclass classification is the <em>cross-entropy</em> loss, also known as the negative log likelihood.  
 </p>
 
-<p>Here we list some of the important limitations of supervised neural network based models. </p>
+<p>In <em>multiclass</em> classification it is common to treat each integer label as a so called <em>one-hot</em> vector:  </p>
+
+<p>$$ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,$$  </p>
+
+<p>$$ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,$$  </p>
+
+<p>i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset.  </p>
+
+<p>Let \( y_{ic} \) denote the \( c \)-th component of the \( i \)-th one-hot vector.  
+We define the cost function \( \mathcal{C} \) as a sum over the cross-entropy loss for each point \( \boldsymbol{x}_i \) in the dataset.
+</p>
+
+<p>In the one-hot representation only one of the terms in the loss function is non-zero, namely the
+probability of the correct category \( c' \)  
+(i.e. the category \( c' \) such that \( y_{ic'} = 1 \)). This means that the cross entropy loss only punishes you for how wrong
+you got the correct label. The probability of category \( c \) is given by the softmax function. The vector \( \boldsymbol{\theta} \) represents the parameters of our network, i.e. all the weights and biases.  
+</p>
 
-<ul>
-<li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
-<li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
-<li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.&nbsp;some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
-<li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
-</ul>
-<p>Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -310,6 +369,18 @@ features).
   <li><a href="._week41-bs049.html">50</a></li>
   <li><a href="._week41-bs050.html">51</a></li>
   <li class="active"><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs052.html">53</a></li>
+  <li><a href="._week41-bs053.html">54</a></li>
+  <li><a href="._week41-bs054.html">55</a></li>
+  <li><a href="._week41-bs055.html">56</a></li>
+  <li><a href="._week41-bs056.html">57</a></li>
+  <li><a href="._week41-bs057.html">58</a></li>
+  <li><a href="._week41-bs058.html">59</a></li>
+  <li><a href="._week41-bs059.html">60</a></li>
+  <li><a href="._week41-bs060.html">61</a></li>
+  <li><a href="">...</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
+  <li><a href="._week41-bs052.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
 </div>  <!-- end container -->
diff --git a/doc/pub/week41/html/._week41-bs052.html b/doc/pub/week41/html/._week41-bs052.html
index d4282d204..f625d8990 100644
--- a/doc/pub/week41/html/._week41-bs052.html
+++ b/doc/pub/week41/html/._week41-bs052.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,26 +327,39 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0052"></a>
 <!-- !split -->
-<h2 id="limitations-of-supervised-learning-with-deep-networks" class="anchor">Limitations of supervised learning with deep networks </h2>
+<h2 id="optimizing-the-cost-function" class="anchor">Optimizing the cost function </h2>
 
-<p>Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
+<p>The network is trained by finding the weights and biases that minimize the cost function. One of the most widely used classes of methods is <em>gradient descent</em> and its generalizations. The idea behind gradient descent
+is simply to adjust the weights in the direction where the gradient of the cost function is large and negative. This ensures we flow toward a <em>local</em> minimum of the cost function.  
+Each parameter \( \theta \) is iteratively adjusted according to the rule  
 </p>
 
-<p>Here we list some of the important limitations of supervised neural network based models. </p>
+<p>$$ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,$$</p>
 
-<ul>
-<li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
-<li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
-<li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.&nbsp;some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
-<li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
-</ul>
-<p>Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.</p>
+<p>where \( \eta \) is known as the <em>learning rate</em>, which controls how big a step we take towards the minimum.  
+This update can be repeated for any number of iterations, or until we are satisfied with the result.  
+</p>
+
+<p>A simple and effective improvement is a variant called <em>Batch Gradient Descent</em>.  
+Instead of calculating the gradient on the whole dataset, we calculate an approximation of the gradient
+on a subset of the data called a <em>minibatch</em>.  
+If there are \( N \) data points and we have a minibatch size of \( M \), the total number of batches
+is \( N/M \).  
+We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient then becomes:  
+</p>
+
+<p>$$ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
+\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,$$
+</p>
+
+<p>i.e. instead of averaging the loss over the entire dataset, we average over a minibatch.  </p>
+
+<p>This has two important benefits:  </p>
+<ol>
+<li> Introducing stochasticity decreases the chance that the algorithm becomes stuck in a local minima.</li>  
+<li> It significantly speeds up the calculation, since we do not have to use the entire dataset to calculate the gradient.</li>  
+</ol>
+<p>The various optmization  methods, with codes and algorithms,  are discussed in our lectures on <a href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html" target="_self">Gradient descent approaches</a>.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -505,7 +386,7 @@ features).
   <li><a href="._week41-bs060.html">61</a></li>
   <li><a href="._week41-bs061.html">62</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs053.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs053.html b/doc/pub/week41/html/._week41-bs053.html
index d0f6abfea..1bcf724cb 100644
--- a/doc/pub/week41/html/._week41-bs053.html
+++ b/doc/pub/week41/html/._week41-bs053.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -458,30 +326,34 @@ MathJax.Hub.Config({
 <div class="container">
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0053"></a>
-<!-- !split -->
-<h2 id="overarching-views-a-personal-note" class="anchor">Overarching Views, a personal  note </h2>
+<!-- !split   -->
+<h2 id="regularization" class="anchor">Regularization </h2>
 
-<p>The author of these lecture notes has an overarching take on many of
-the machine learning algorithms we discuss here. 
+<p>It is common to add an extra term to the cost function, proportional
+to the size of the weights.  This is equivalent to constraining the
+size of the weights, so that they do not grow out of control.
+Constraining the size of the weights means that the weights cannot
+grow arbitrarily large to fit the training data, and in this way
+reduces <em>overfitting</em>.
 </p>
 
-<p>If we wish to understand complex systems, we need to find some
-effective degrees of freedom or features that we find essential,
-simply in order to reduce the complexity of the systems we are
-studying.  This leads, in one way or the other to dimensionality
-reductions. Most of the Machine Learning methods we encounter deal
-with this, whether we opt for a principal component analysis, or
-clustering, or convolutional neural networks, or Ridge or Lasso
-regression or random forest, yes, perhaps most machine learning
-methods at large.
+<p>We will measure the size of the weights using the so called <em>L2-norm</em>, meaning our cost function becomes:  </p>
+
+<p>$$  \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) \quad \rightarrow \quad
+\frac{1}{N} \sum_{i=1}^N  \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2 
+= \frac{1}{N} \sum_{i=1}^N  \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,$$  
 </p>
 
-<p>For neural networks and our previous discussion, we have seen that we
-in essence end up with matrix-matrix and matrix-vector
-multiplications. In all cases, our matrices are dense ones, and the
-more data we deal with the larger the dimensionalities of the matrices
-and vectors. How can we reduce such dimensionalities?  One possible
-answer is offered by <b>convolutional neural networks</b> (CNN), to be discussed next week.
+<p>i.e. we sum up all the weights squared. The factor \( \lambda \) is known as a regularization parameter.</p>
+
+<p>In order to train the model, we need to calculate the derivative of
+the cost function with respect to every bias and weight in the
+network.  In total our network has \( (64 + 1)\times 50=3250 \) weights in
+the hidden layer and \( (50 + 1)\times 10=510 \) weights to the output
+layer (\( +1 \) for the bias), and the gradient must be calculated for
+every parameter.  We use the <em>backpropagation</em> algorithm discussed
+above. This is a clever use of the chain rule that allows us to
+calculate the gradient efficently. 
 </p>
 
 <p>
@@ -509,7 +381,7 @@ answer is offered by <b>convolutional neural networks</b> (CNN), to be discussed
   <li><a href="._week41-bs061.html">62</a></li>
   <li><a href="._week41-bs062.html">63</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs054.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs054.html b/doc/pub/week41/html/._week41-bs054.html
index 5ef10b46f..2698295d4 100644
--- a/doc/pub/week41/html/._week41-bs054.html
+++ b/doc/pub/week41/html/._week41-bs054.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,12 +327,132 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0054"></a>
 <!-- !split -->
-<h2 id="using-automatic-differentiation" class="anchor">Using Automatic differentiation </h2>
+<h2 id="matrix-multiplication" class="anchor">Matrix  multiplication </h2>
 
-<p>In our discussions of ordinary differential equations 
-we will also study the usage of <a href="https://www.youtube.com/watch?v=fRf4l5qaX1M&ab_channel=AlexSmola" target="_self">Autograd</a> in computing gradients for deep learning. For the documentation of Autograd and examples see the lectures slides from <a href="https://compphysics.github.io/MachineLearning/doc/pub/week39/html/week39.html" target="_self">week 40</a> and the <a href="https://github.com/HIPS/autograd" target="_self">Autograd documentation</a>.
+<p>To more efficently train our network these equations are implemented using matrix operations.  
+The error in the output layer is calculated simply as, with \( \boldsymbol{t} \) being our targets,  
 </p>
 
+<p>$$ \delta_L = \boldsymbol{t} - \boldsymbol{y} = (n_{inputs}, n_{categories}) .$$  </p>
+
+<p>The gradient for the output weights is calculated as  </p>
+
+<p>$$ \nabla W_{L} = \boldsymbol{a}^T \delta_L   = (n_{hidden}, n_{categories}) ,$$</p>
+
+<p>where \( \boldsymbol{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input.  
+Since we are going backwards we have to transpose the activation matrix.  
+</p>
+
+<p>The gradient with respect to the output bias is then  </p>
+
+<p>$$ \nabla \boldsymbol{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$  </p>
+
+<p>The error in the hidden layer is  </p>
+
+<p>$$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$  </p>
+
+<p>where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean
+that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes
+the <em>Hadamard product</em>, meaning element-wise multiplication.  
+</p>
+
+<p>This again gives us the gradients in the hidden layer:  </p>
+
+<p>$$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$  </p>
+
+<p>$$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># to categorical turns our integer vector into a onehot representation</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
+
+<span style="color: #408080; font-style: italic"># one-hot in numpy</span>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">to_categorical_numpy</span>(integer_vector):
+    n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(integer_vector)
+    n_categories <span style="color: #666666">=</span> np<span style="color: #666666">.</span>max(integer_vector) <span style="color: #666666">+</span> <span style="color: #666666">1</span>
+    onehot_vector <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((n_inputs, n_categories))
+    onehot_vector[<span style="color: #008000">range</span>(n_inputs), integer_vector] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
+    
+    <span style="color: #008000; font-weight: bold">return</span> onehot_vector
+
+<span style="color: #408080; font-style: italic">#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)</span>
+Y_train_onehot, Y_test_onehot <span style="color: #666666">=</span> to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
+
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward_train</span>(X):
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
+    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
+    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
+    a_h <span style="color: #666666">=</span> sigmoid(z_h)
+    
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
+    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
+    <span style="color: #408080; font-style: italic"># softmax output</span>
+    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
+    exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
+    probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
+    
+    <span style="color: #408080; font-style: italic"># for backpropagation need activations in hidden and output layers</span>
+    <span style="color: #008000; font-weight: bold">return</span> a_h, probabilities
+
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backpropagation</span>(X, Y):
+    a_h, probabilities <span style="color: #666666">=</span> feed_forward_train(X)
+    
+    <span style="color: #408080; font-style: italic"># error in the output layer</span>
+    error_output <span style="color: #666666">=</span> probabilities <span style="color: #666666">-</span> Y
+    <span style="color: #408080; font-style: italic"># error in the hidden layer</span>
+    error_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(error_output, output_weights<span style="color: #666666">.</span>T) <span style="color: #666666">*</span> a_h <span style="color: #666666">*</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> a_h)
+    
+    <span style="color: #408080; font-style: italic"># gradients for the output layer</span>
+    output_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h<span style="color: #666666">.</span>T, error_output)
+    output_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_output, axis<span style="color: #666666">=0</span>)
+    
+    <span style="color: #408080; font-style: italic"># gradient for the hidden layer</span>
+    hidden_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X<span style="color: #666666">.</span>T, error_hidden)
+    hidden_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_hidden, axis<span style="color: #666666">=0</span>)
+
+    <span style="color: #008000; font-weight: bold">return</span> output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
+
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Old accuracy on training data: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(accuracy_score(predict(X_train), Y_train)))
+
+eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
+lmbd <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
+<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1000</span>):
+    <span style="color: #408080; font-style: italic"># calculate gradients</span>
+    dWo, dBo, dWh, dBh <span style="color: #666666">=</span> backpropagation(X_train, Y_train_onehot)
+    
+    <span style="color: #408080; font-style: italic"># regularization term gradients</span>
+    dWo <span style="color: #666666">+=</span> lmbd <span style="color: #666666">*</span> output_weights
+    dWh <span style="color: #666666">+=</span> lmbd <span style="color: #666666">*</span> hidden_weights
+    
+    <span style="color: #408080; font-style: italic"># update weights and biases</span>
+    output_weights <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dWo
+    output_bias <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dBo
+    hidden_weights <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dWh
+    hidden_bias <span style="color: #666666">-=</span> eta <span style="color: #666666">*</span> dBh
+
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;New accuracy on training data: &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(accuracy_score(predict(X_train), Y_train)))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -490,7 +478,7 @@ we will also study the usage of <a href="https://www.youtube.com/watch?v=fRf4l5q
   <li><a href="._week41-bs062.html">63</a></li>
   <li><a href="._week41-bs063.html">64</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs055.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs055.html b/doc/pub/week41/html/._week41-bs055.html
index 06aaff09c..9ae30b75c 100644
--- a/doc/pub/week41/html/._week41-bs055.html
+++ b/doc/pub/week41/html/._week41-bs055.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,36 +327,21 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0055"></a>
 <!-- !split -->
-<h2 id="solving-odes-with-deep-learning" class="anchor">Solving ODEs with Deep Learning </h2>
+<h2 id="improving-performance" class="anchor">Improving performance </h2>
 
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>The Universal Approximation Theorem states that a neural network can
-approximate any function at a single hidden layer along with one input
-and output layer to any given precision.  
+<p>As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image.  
+In order to obtain a network that does something useful, we will have to do a bit more work.  
 </p>
-</div>
-</div>
 
+<p>The choice of <em>hyperparameters</em> such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a <em>grid-search</em> is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates \( \eta = 10^{-6}, 10^{-5},...,10^{-1} \) with different regularization parameters \( \lambda = 10^{-6},...,10^{-0} \).  </p>
 
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p><a href="https://www.springer.com/gp/book/9789401798150" target="_self">An Introduction to Neural Network Methods for Differential Equations</a>, by Yadav and Kumar.</p>
-</div>
-</div>
-
-
-<div class="panel panel-default">
-<div class="panel-body">
-<!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
-<p>The lectures on differential equations were developed by Kristine Baluka Hein, now PhD student at IFI.
-A great thanks to Kristine.
+<p>Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an <em>iteration</em>, and a full training period
+going through the entire dataset (\( n/M \) batches) an <em>epoch</em>.
 </p>
-</div>
-</div>
 
+<p>If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers.  
+Andrew Ng goes through some of these considerations in this <a href="https://youtu.be/F1ka6a13S9I" target="_self">video</a>. You can find a summary of the video <a href="https://kevinzakka.github.io/2016/09/26/applying-deep-learning/" target="_self">here</a>.  
+</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -515,7 +368,7 @@ A great thanks to Kristine.
   <li><a href="._week41-bs063.html">64</a></li>
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs056.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs056.html b/doc/pub/week41/html/._week41-bs056.html
index 7e785f0bc..350285660 100644
--- a/doc/pub/week41/html/._week41-bs056.html
+++ b/doc/pub/week41/html/._week41-bs056.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,27 +327,134 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0056"></a>
 <!-- !split -->
-<h2 id="ordinary-differential-equations" class="anchor">Ordinary Differential Equations </h2>
+<h2 id="full-object-oriented-implementation" class="anchor">Full object-oriented implementation </h2>
 
-<p>An ordinary differential equation (ODE) is an equation involving functions having one variable.</p>
-
-<p>In general, an ordinary differential equation looks like</p>
-
-$$
-\begin{equation} \tag{1}
-f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right) = 0
-\end{equation}
-$$
-
-<p>where \( g(x) \) is the function to find, and \( g^{(n)}(x) \) is the \( n \)-th derivative of \( g(x) \).</p>
-
-<p>The \( f\left(x, g(x), g'(x), g''(x), \, \dots \, , g^{(n)}(x)\right) \) is just a way to write that there is an expression involving \( x \) and \( g(x), \ g'(x), \ g''(x), \, \dots \, , \text{ and } g^{(n)}(x) \) on the left side of the equality sign in <a href="#mjx-eqn-1">(1)</a>.
-The highest order of derivative, that is the value of \( n \), determines to the order of the equation.
-The equation is referred to as a \( n \)-th order ODE.
-Along with <a href="#mjx-eqn-1">(1)</a>, some additional conditions of the function \( g(x) \) are typically given
-for the solution to be unique.
+<p>It is very natural to think of the network as an object, with specific instances of the network
+being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below.
 </p>
 
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">NeuralNetwork</span>:
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
+            <span style="color: #008000">self</span>,
+            X_data,
+            Y_data,
+            n_hidden_neurons<span style="color: #666666">=50</span>,
+            n_categories<span style="color: #666666">=10</span>,
+            epochs<span style="color: #666666">=10</span>,
+            batch_size<span style="color: #666666">=100</span>,
+            eta<span style="color: #666666">=0.1</span>,
+            lmbd<span style="color: #666666">=0.0</span>):
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data_full <span style="color: #666666">=</span> X_data
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data_full <span style="color: #666666">=</span> Y_data
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">=</span> X_data<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_features <span style="color: #666666">=</span> X_data<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons <span style="color: #666666">=</span> n_hidden_neurons
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories <span style="color: #666666">=</span> n_categories
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>epochs <span style="color: #666666">=</span> epochs
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size <span style="color: #666666">=</span> batch_size
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>iterations <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">=</span> eta
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">=</span> lmbd
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>create_biases_and_weights()
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_biases_and_weights</span>(<span style="color: #008000">self</span>):
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_features, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons)
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_hidden_neurons, <span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories)
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(<span style="color: #008000">self</span>):
+        <span style="color: #408080; font-style: italic"># feed-forward for training</span>
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_data, <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h <span style="color: #666666">=</span> sigmoid(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_h)
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>a_h, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias
+
+        exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_o)
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward_out</span>(<span style="color: #008000">self</span>, X):
+        <span style="color: #408080; font-style: italic"># feed-forward for output</span>
+        z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias
+        a_h <span style="color: #666666">=</span> sigmoid(z_h)
+
+        z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights) <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias
+        
+        exp_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(z_o)
+        probabilities <span style="color: #666666">=</span> exp_term <span style="color: #666666">/</span> np<span style="color: #666666">.</span>sum(exp_term, axis<span style="color: #666666">=1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
+        <span style="color: #008000; font-weight: bold">return</span> probabilities
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">backpropagation</span>(<span style="color: #008000">self</span>):
+        error_output <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>probabilities <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data
+        error_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(error_output, <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights<span style="color: #666666">.</span>T) <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h <span style="color: #666666">*</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_h)
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>a_h<span style="color: #666666">.</span>T, error_output)
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_output, axis<span style="color: #666666">=0</span>)
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_data<span style="color: #666666">.</span>T, error_hidden)
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias_gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(error_hidden, axis<span style="color: #666666">=0</span>)
+
+        <span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">&gt;</span> <span style="color: #666666">0.0</span>:
+            <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights
+            <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>lmbd <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights
+
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_weights_gradient
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>output_bias_gradient
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_weights_gradient
+        <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias <span style="color: #666666">-=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>hidden_bias_gradient
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(<span style="color: #008000">self</span>, X):
+        probabilities <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward_out(X)
+        <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict_probabilities</span>(<span style="color: #008000">self</span>, X):
+        probabilities <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward_out(X)
+        <span style="color: #008000; font-weight: bold">return</span> probabilities
+
+    <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train</span>(<span style="color: #008000">self</span>):
+        data_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #008000">self</span><span style="color: #666666">.</span>n_inputs)
+
+        <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>epochs):
+            <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>iterations):
+                <span style="color: #408080; font-style: italic"># pick datapoints with replacement</span>
+                chosen_datapoints <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(
+                    data_indices, size<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>batch_size, replace<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>
+                )
+
+                <span style="color: #408080; font-style: italic"># minibatch training data</span>
+                <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_data_full[chosen_datapoints]
+                <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>Y_data_full[chosen_datapoints]
+
+                <span style="color: #008000">self</span><span style="color: #666666">.</span>feed_forward()
+                <span style="color: #008000">self</span><span style="color: #666666">.</span>backpropagation()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
 <p>
 <!-- navigation buttons at the bottom of the page -->
 <ul class="pagination">
@@ -505,7 +480,7 @@ for the solution to be unique.
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs057.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs057.html b/doc/pub/week41/html/._week41-bs057.html
index 820880c5b..14e4dcf36 100644
--- a/doc/pub/week41/html/._week41-bs057.html
+++ b/doc/pub/week41/html/._week41-bs057.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,30 +327,55 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0057"></a>
 <!-- !split -->
-<h2 id="the-trial-solution" class="anchor">The trial solution </h2>
+<h2 id="evaluate-model-performance-on-test-data" class="anchor">Evaluate model performance on test data </h2>
 
-<p>Let the trial solution \( g_t(x) \) be</p>
-
-$$
-\begin{equation}
-	g_t(x) = h_1(x) + h_2(x,N(x,P))
-\tag{2}
-\end{equation}
-$$
-
-<p>where \( h_1(x) \) is a function that makes \( g_t(x) \) satisfy a given set
-of conditions, \( N(x,P) \) a neural network with weights and biases
-described by \( P \) and \( h_2(x, N(x,P)) \) some expression involving the
-neural network.  The role of the function \( h_2(x, N(x,P)) \), is to
-ensure that the output from \( N(x,P) \) is zero when \( g_t(x) \) is
-evaluated at the values of \( x \) where the given conditions must be
-satisfied.  The function \( h_1(x) \) should alone make \( g_t(x) \) satisfy
-the conditions.
+<p>To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data.  
+We measure the performance of the network using the <em>accuracy</em> score.  
+The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \).  
 </p>
 
-<p>But what about the network \( N(x,P) \)?</p>
+<p>$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\tilde{y}_i = y_i)}{n} ,$$  </p>
+
+<p>where \( I \) is the indicator function, \( 1 \) if \( \tilde{y}_i = y_i \) and \( 0 \) otherwise.</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;">epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
+batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
+
+dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
+                    n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
+dnn<span style="color: #666666">.</span>train()
+test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
+
+<span style="color: #408080; font-style: italic"># accuracy score from scikit library</span>
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
+
+<span style="color: #408080; font-style: italic"># equivalent in numpy</span>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">accuracy_score_numpy</span>(Y_test, Y_pred):
+    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum(Y_test <span style="color: #666666">==</span> Y_pred) <span style="color: #666666">/</span> <span style="color: #008000">len</span>(Y_test)
+
+<span style="color: #408080; font-style: italic">#print(&quot;Accuracy score on test set: &quot;, accuracy_score_numpy(Y_test, test_predict))</span>
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>As described previously, an optimization method could be used to minimize the parameters of a neural network, that being its weights and biases, through backward propagation.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -508,8 +401,6 @@ the conditions.
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs058.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs058.html b/doc/pub/week41/html/._week41-bs058.html
index 6cce000b8..30785abe0 100644
--- a/doc/pub/week41/html/._week41-bs058.html
+++ b/doc/pub/week41/html/._week41-bs058.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,33 +327,54 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0058"></a>
 <!-- !split -->
-<h2 id="minimization-process" class="anchor">Minimization process </h2>
+<h2 id="adjust-hyperparameters" class="anchor">Adjust hyperparameters </h2>
 
-<p>For the minimization to be defined, we need to have a cost function at hand to minimize.</p>
-
-<p>It is given that \( f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right) \) should be equal to zero in <a href="._week41-bs056.html#mjx-eqn-1">(1)</a>.
-We can choose to consider the mean squared error as the cost function for an input \( x \).
-Since we are looking at one input, the cost function is just \( f \) squared.
-The cost function \( c\left(x, P \right) \) can therefore be expressed as
+<p>We now perform a grid search to find the optimal hyperparameters for the network.  
+Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate).
 </p>
 
-$$
-C\left(x, P\right) = \big(f\left(x, \, g(x), \, g'(x), \, g''(x), \, \dots \, , \, g^{(n)}(x)\right)\big)^2
-$$
 
-<p>If \( N \) inputs are given as a vector \( \boldsymbol{x} \) with elements \( x_i \) for \( i = 1,\dots,N \),
-the cost function becomes
-</p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;">eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
+lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
+<span style="color: #408080; font-style: italic"># store the models for later use</span>
+DNN_numpy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
 
-$$
-\begin{equation} \tag{3}
-	C\left(\boldsymbol{x}, P\right) = \frac{1}{N} \sum_{i=1}^N \big(f\left(x_i, \, g(x_i), \, g'(x_i), \, g''(x_i), \, \dots \, , \, g^{(n)}(x_i)\right)\big)^2
-\end{equation}
-$$
+<span style="color: #408080; font-style: italic"># grid search</span>
+<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
+    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
+        dnn <span style="color: #666666">=</span> NeuralNetwork(X_train, Y_train_onehot, eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size,
+                            n_hidden_neurons<span style="color: #666666">=</span>n_hidden_neurons, n_categories<span style="color: #666666">=</span>n_categories)
+        dnn<span style="color: #666666">.</span>train()
+        
+        DNN_numpy[i][j] <span style="color: #666666">=</span> dnn
+        
+        test_predict <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
+        
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, accuracy_score(Y_test, test_predict))
+        <span style="color: #008000">print</span>()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>The neural net should then find the parameters \( P \) that minimizes the cost function in
-<a href="#mjx-eqn-3">(3)</a> for a set of \( N \) training samples \( x_i \).
-</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -510,9 +399,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs059.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs059.html b/doc/pub/week41/html/._week41-bs059.html
index 239ca3d74..feca116a5 100644
--- a/doc/pub/week41/html/._week41-bs059.html
+++ b/doc/pub/week41/html/._week41-bs059.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,15 +327,63 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0059"></a>
 <!-- !split -->
-<h2 id="minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" class="anchor">Minimizing the cost function using gradient descent and automatic differentiation </h2>
+<h2 id="visualization" class="anchor">Visualization </h2>
 
-<p>To perform the minimization using gradient descent, the gradient of \( C\left(\boldsymbol{x}, P\right) \) is needed.
-It might happen so that finding an analytical expression of the gradient of \( C(\boldsymbol{x}, P) \) from <a href="._week41-bs058.html#mjx-eqn-3">(3)</a> gets too messy, depending on which cost function one desires to use.
-</p>
 
-<p>Luckily, there exists libraries that makes the job for us through automatic differentiation.
-Automatic differentiation is a method of finding the derivatives numerically with very high precision.
-</p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># visual representation of grid search</span>
+<span style="color: #408080; font-style: italic"># uses seaborn heatmap, you can also do this with matplotlib imshow</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
+
+sns<span style="color: #666666">.</span>set()
+
+train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+
+<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
+    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
+        dnn <span style="color: #666666">=</span> DNN_numpy[i][j]
+        
+        train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train) 
+        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
+
+        train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
+        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
+
+        
+fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
+sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
+ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+plt<span style="color: #666666">.</span>show()
+
+fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
+sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
+ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+plt<span style="color: #666666">.</span>show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -491,10 +407,6 @@ Automatic differentiation is a method of finding the derivatives numerically wit
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs060.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs060.html b/doc/pub/week41/html/._week41-bs060.html
index 6b2b0d139..c26a79264 100644
--- a/doc/pub/week41/html/._week41-bs060.html
+++ b/doc/pub/week41/html/._week41-bs060.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,28 +327,60 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0060"></a>
 <!-- !split -->
-<h2 id="example-exponential-decay" class="anchor">Example: Exponential decay </h2>
+<h2 id="scikit-learn-implementation" class="anchor">scikit-learn implementation </h2>
 
-<p>An exponential decay of a quantity \( g(x) \) is described by the equation</p>
+<p><b>scikit-learn</b> focuses more
+on traditional machine learning methods, such as regression,
+clustering, decision trees, etc. As such, it has only two types of
+neural networks: Multi Layer Perceptron outputting continuous values,
+<em>MPLRegressor</em>, and Multi Layer Perceptron outputting labels,
+<em>MLPClassifier</em>. We will see how simple it is to use these classes.
+</p>
 
-$$
-\begin{equation} \tag{4}
-  g'(x) = -\gamma g(x)
-\end{equation}
-$$
+<p><b>scikit-learn</b> implements a few improvements from our neural network,
+such as early stopping, a varying learning rate, different
+optimization methods, etc. We would therefore expect a better
+performance overall.
+</p>
 
-<p>with \( g(0) = g_0 \) for some chosen initial value \( g_0 \).</p>
 
-<p>The analytical solution of <a href="#mjx-eqn-4">(4)</a> is</p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
+<span style="color: #408080; font-style: italic"># store models for later use</span>
+DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
 
-$$
-\begin{equation}
-  g(x) = g_0 \exp\left(-\gamma x\right)
-\tag{5}
-\end{equation}
-$$
+<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
+    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
+        dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;logistic&#39;</span>,
+                            alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
+        dnn<span style="color: #666666">.</span>fit(X_train, Y_train)
+        
+        DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
+        
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on test set: &quot;</span>, dnn<span style="color: #666666">.</span>score(X_test, Y_test))
+        <span style="color: #008000">print</span>()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>Having an analytical solution at hand, it is possible to use it to compare how well a neural network finds a solution of <a href="#mjx-eqn-4">(4)</a>.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -503,11 +403,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs061.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs061.html b/doc/pub/week41/html/._week41-bs061.html
index 7787863e3..02e389166 100644
--- a/doc/pub/week41/html/._week41-bs061.html
+++ b/doc/pub/week41/html/._week41-bs061.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,19 +327,63 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0061"></a>
 <!-- !split -->
-<h2 id="the-function-to-solve-for" class="anchor">The function to solve for </h2>
+<h2 id="visualization" class="anchor">Visualization </h2>
 
-<p>The program will use a neural network to solve</p>
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># optional</span>
+<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
+<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
 
-$$
-\begin{equation} \tag{6}
-g'(x) = -\gamma g(x)
-\end{equation}
-$$
+sns<span style="color: #666666">.</span>set()
 
-<p>where \( g(0) = g_0 \) with \( \gamma \) and \( g_0 \) being some chosen values.</p>
+train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+
+<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
+    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
+        dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
+        
+        train_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_train) 
+        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X_test)
+
+        train_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_train, train_pred)
+        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(Y_test, test_pred)
+
+        
+fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
+sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
+ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+plt<span style="color: #666666">.</span>show()
+
+fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
+sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
+ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+plt<span style="color: #666666">.</span>show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>In this example, \( \gamma = 2 \) and \( g_0 = 10 \).</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -493,12 +405,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs062.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs062.html b/doc/pub/week41/html/._week41-bs062.html
index 2545a0aff..7e7403115 100644
--- a/doc/pub/week41/html/._week41-bs062.html
+++ b/doc/pub/week41/html/._week41-bs062.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,14 +327,29 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0062"></a>
 <!-- !split -->
-<h2 id="the-trial-solution" class="anchor">The trial solution </h2>
-<p>To begin with, a trial solution \( g_t(t) \) must be chosen. A general trial solution for ordinary differential equations could be</p>
+<h2 id="testing-our-code-for-the-xor-or-and-and-gates" class="anchor">Testing our code for the XOR, OR and AND gates </h2>
 
-$$
-g_t(x, P) = h_1(x) + h_2(x, N(x, P))
-$$
+<p>Last week we discussed three different types of gates, the so-called
+XOR, the OR and the AND gates.  Their inputs and outputs can be
+summarized using the following tables, first for the OR gate with
+inputs \( x_1 \) and \( x_2 \) and outputs \( y \):
+</p>
 
-<p>with \( h_1(x) \) ensuring that \( g_t(x) \) satisfies some conditions and \( h_2(x,N(x, P)) \) an expression involving \( x \) and the output from the neural network \( N(x,P) \) with \( P  \) being the collection of the weights and biases for each layer. For now, it is assumed that the network consists of one input layer, one hidden layer, and one output layer.</p>
+<div class="row">
+  <div class="col-xs-3">
+    <table class="dotable table-striped table-hover table-condensed">
+<thead>
+<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
+</thead>
+<tbody>
+<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
+<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   1          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
+</tbody>
+    </table>
+  </div> <!-- col-xs-3 -->
+</div> <!-- cell row -->
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -487,13 +370,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="._week41-bs071.html">72</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs063.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs063.html b/doc/pub/week41/html/._week41-bs063.html
index 7c56301ea..a86ab25a8 100644
--- a/doc/pub/week41/html/._week41-bs063.html
+++ b/doc/pub/week41/html/._week41-bs063.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,26 +327,43 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0063"></a>
 <!-- !split -->
-<h2 id="setup-of-network" class="anchor">Setup of Network </h2>
+<h2 id="the-and-and-xor-gates" class="anchor">The AND and XOR Gates </h2>
 
-<p>In this network, there are no weights and bias at the input layer, so \( P = \{ P_{\text{hidden}},  P_{\text{output}} \} \).
-If there are \( N_{\text{hidden} } \) neurons in the hidden layer, then \( P_{\text{hidden}} \) is a \( N_{\text{hidden} } \times (1 + N_{\text{input}}) \) matrix, given that there are \( N_{\text{input}} \) neurons in the input layer.
-</p>
+<p>The AND gate is defined as</p>
 
-<p>The first column in \( P_{\text{hidden} } \) represents the bias for each neuron in the hidden layer and the second column represents the weights for each neuron in the hidden layer from the input layer.
-If there are \( N_{\text{output} } \) neurons in the output layer, then \( P_{\text{output}}  \) is a \( N_{\text{output} } \times (1 + N_{\text{hidden} }) \) matrix.
-</p>
+<div class="row">
+  <div class="col-xs-3">
+    <table class="dotable table-striped table-hover table-condensed">
+<thead>
+<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
+</thead>
+<tbody>
+<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
+<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   0          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
+</tbody>
+    </table>
+  </div> <!-- col-xs-3 -->
+</div> <!-- cell row -->
 
-<p>Its first column represents the bias of each neuron and the remaining columns represents the weights to each neuron.</p>
-
-<p>It is given that \( g(0) = g_0 \). The trial solution must fulfill this condition to be a proper solution of <a href="._week41-bs061.html#mjx-eqn-6">(6)</a>. A possible way to ensure that \( g_t(0, P) = g_0 \), is to let \( F(N(x,P)) = x \cdot N(x,P) \) and \( A(x) = g_0 \). This gives the following trial solution:</p>
-
-$$
-\begin{equation} \tag{7}
-g_t(x, P) = g_0 + x \cdot N(x, P)
-\end{equation}
-$$
+<p>And finally we have the XOR gate</p>
 
+<div class="row">
+  <div class="col-xs-3">
+    <table class="dotable table-striped table-hover table-condensed">
+<thead>
+<tr><td align="center">\( x_1 \)</td> <td align="center">\( x_2 \)</td> <td align="center">\( y \)</td> </tr>
+</thead>
+<tbody>
+<tr><td align="center">   0            </td> <td align="center">   0            </td> <td align="center">   0          </td> </tr>
+<tr><td align="center">   0            </td> <td align="center">   1            </td> <td align="center">   1          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   0            </td> <td align="center">   1          </td> </tr>
+<tr><td align="center">   1            </td> <td align="center">   1            </td> <td align="center">   0          </td> </tr>
+</tbody>
+    </table>
+  </div> <!-- col-xs-3 -->
+</div> <!-- cell row -->
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -498,14 +383,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="._week41-bs071.html">72</a></li>
-  <li><a href="._week41-bs072.html">73</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs064.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs064.html b/doc/pub/week41/html/._week41-bs064.html
index 9f5db30b7..bd4b8de8d 100644
--- a/doc/pub/week41/html/._week41-bs064.html
+++ b/doc/pub/week41/html/._week41-bs064.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,31 +327,18 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0064"></a>
 <!-- !split -->
-<h2 id="reformulating-the-problem" class="anchor">Reformulating the problem </h2>
+<h2 id="representing-the-data-sets" class="anchor">Representing the Data Sets </h2>
 
-<p>We wish that our neural network manages to minimize a given cost function.</p>
-
-<p>A reformulation of out equation, <a href="._week41-bs061.html#mjx-eqn-6">(6)</a>, must therefore be done,
-such that it describes the problem a neural network can solve for.
-</p>
-
-<p>The neural network must find the set of weights and biases \( P \) such that the trial solution in <a href="._week41-bs063.html#mjx-eqn-7">(7)</a> satisfies <a href="._week41-bs061.html#mjx-eqn-6">(6)</a>.</p>
-
-<p>The trial solution</p>
+<p>Our design matrix is defined by the input values \( x_1 \) and \( x_2 \). Since we have four possible outputs, our design matrix reads</p>
 
 $$
-g_t(x, P) = g_0 + x \cdot N(x, P)
+\boldsymbol{X}=\begin{bmatrix} 0 & 0 \\
+                       0 & 1 \\
+		       1 & 0 \\
+		       1 & 1 \end{bmatrix},
 $$
 
-<p>has been chosen such that it already solves the condition \( g(0) = g_0 \). What remains, is to find \( P \) such that</p>
-
-$$
-\begin{equation} \tag{8}
-g_t'(x, P) = - \gamma g_t(x, P)
-\end{equation}
-$$
-
-<p>is fulfilled as <em>best as possible</em>.</p>
+<p>while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate, \( \boldsymbol{y}^T=[0,0,0,1] \) for the AND gate and \( \boldsymbol{y}^T=[0,1,1,1] \) for the OR gate.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -502,15 +357,6 @@ $$
   <li class="active"><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="._week41-bs071.html">72</a></li>
-  <li><a href="._week41-bs072.html">73</a></li>
-  <li><a href="._week41-bs073.html">74</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs065.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs065.html b/doc/pub/week41/html/._week41-bs065.html
index a9193715a..b6fc69a04 100644
--- a/doc/pub/week41/html/._week41-bs065.html
+++ b/doc/pub/week41/html/._week41-bs065.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,28 +327,98 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0065"></a>
 <!-- !split -->
-<h2 id="more-technicalities" class="anchor">More technicalities </h2>
+<h2 id="setting-up-the-neural-network" class="anchor">Setting up the Neural Network </h2>
 
-<p>The left hand side and right hand side of <a href="._week41-bs064.html#mjx-eqn-8">(8)</a> must be computed separately, and then the neural network must choose weights and biases, contained in \( P \), such that the sides are equal as best as possible.
-This means that the absolute or squared difference between the sides must be as close to zero, ideally equal to zero.
-In this case, the difference squared shows to be an appropriate measurement of how erroneous the trial solution is with respect to \( P \) of the neural network.
-</p>
+<p>We define first our design matrix and the various output vectors for the different gates.</p>
 
-<p>This gives the following cost function our neural network must solve for:</p>
 
-$$
-\min_{P}\Big\{ \big(g_t'(x, P) - ( -\gamma g_t(x, P) \big)^2 \Big\}
-$$
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<span style="color: #BA2121; font-style: italic">Simple code that tests XOR, OR and AND gates with linear regression</span>
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
 
-<p>(the notation \( \min_{P}\{ f(x, P) \} \) means that we desire to find \( P \) that yields the minimum of \( f(x, P) \))</p>
+<span style="color: #408080; font-style: italic"># import necessary packages</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
 
-<p>or, in terms of weights and biases for the hidden and output layer in our network:</p>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(x):
+    <span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x))
 
-$$
-\min_{P_{\text{hidden} }, \ P_{\text{output} }}\Big\{ \big(g_t'(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) - ( -\gamma g_t(x, \{ P_{\text{hidden} }, P_{\text{output} }\}) \big)^2 \Big\}
-$$
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">feed_forward</span>(X):
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the hidden layer</span>
+    z_h <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(X, hidden_weights) <span style="color: #666666">+</span> hidden_bias
+    <span style="color: #408080; font-style: italic"># activation in the hidden layer</span>
+    a_h <span style="color: #666666">=</span> sigmoid(z_h)
+    
+    <span style="color: #408080; font-style: italic"># weighted sum of inputs to the output layer</span>
+    z_o <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(a_h, output_weights) <span style="color: #666666">+</span> output_bias
+    <span style="color: #408080; font-style: italic"># softmax output</span>
+    <span style="color: #408080; font-style: italic"># axis 0 holds each input and axis 1 the probabilities of each category</span>
+    probabilities <span style="color: #666666">=</span> sigmoid(z_o)
+    <span style="color: #008000; font-weight: bold">return</span> probabilities
 
-<p>for an input value \( x \).</p>
+<span style="color: #408080; font-style: italic"># we obtain a prediction by taking the class with the highest likelihood</span>
+<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(X):
+    probabilities <span style="color: #666666">=</span> feed_forward(X)
+    <span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>argmax(probabilities, axis<span style="color: #666666">=1</span>)
+
+<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
+np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
+
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+
+<span style="color: #408080; font-style: italic"># The XOR gate</span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+<span style="color: #408080; font-style: italic"># The OR gate</span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+<span style="color: #408080; font-style: italic"># The AND gate</span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+
+<span style="color: #408080; font-style: italic"># Defining the neural network</span>
+n_inputs, n_features <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape
+n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+n_categories <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+n_features <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+
+<span style="color: #408080; font-style: italic"># we make the weights normally distributed using numpy.random.randn</span>
+
+<span style="color: #408080; font-style: italic"># weights and bias in the hidden layer</span>
+hidden_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_features, n_hidden_neurons)
+hidden_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_hidden_neurons) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+<span style="color: #408080; font-style: italic"># weights and bias in the output layer</span>
+output_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n_hidden_neurons, n_categories)
+output_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(n_categories) <span style="color: #666666">+</span> <span style="color: #666666">0.01</span>
+
+probabilities <span style="color: #666666">=</span> feed_forward(X)
+<span style="color: #008000">print</span>(probabilities)
+
+
+predictions <span style="color: #666666">=</span> predict(X)
+<span style="color: #008000">print</span>(predictions)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -498,16 +436,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li class="active"><a href="._week41-bs065.html">66</a></li>
   <li><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="._week41-bs071.html">72</a></li>
-  <li><a href="._week41-bs072.html">73</a></li>
-  <li><a href="._week41-bs073.html">74</a></li>
-  <li><a href="._week41-bs074.html">75</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
   <li><a href="._week41-bs066.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/._week41-bs066.html b/doc/pub/week41/html/._week41-bs066.html
index 85c2f017c..9d85181da 100644
--- a/doc/pub/week41/html/._week41-bs066.html
+++ b/doc/pub/week41/html/._week41-bs066.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,196 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('The Mathematics of Neural Networks',
-               2,
-               None,
-               'the-mathematics-of-neural-networks'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks'),
-              ('Overarching Views, a personal  note',
-               2,
-               None,
-               'overarching-views-a-personal-note'),
-              ('Using Automatic differentiation',
-               2,
-               None,
-               'using-automatic-differentiation'),
-              ('Solving ODEs with Deep Learning',
-               2,
-               None,
-               'solving-odes-with-deep-learning'),
-              ('Ordinary Differential Equations',
-               2,
-               None,
-               'ordinary-differential-equations'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Minimization process', 2, None, 'minimization-process'),
-              ('Minimizing the cost function using gradient descent and '
-               'automatic differentiation',
-               2,
-               None,
-               'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
-              ('Example: Exponential decay',
-               2,
-               None,
-               'example-exponential-decay'),
-              ('The function to solve for',
-               2,
-               None,
-               'the-function-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('Setup of Network', 2, None, 'setup-of-network'),
-              ('Reformulating the problem',
-               2,
-               None,
-               'reformulating-the-problem'),
-              ('More technicalities', 2, None, 'more-technicalities'),
-              ('More details', 2, None, 'more-details'),
-              ('A possible implementation of a neural network',
-               2,
-               None,
-               'a-possible-implementation-of-a-neural-network'),
-              ('Technicalities', 2, None, 'technicalities'),
-              ('Final technicalities I', 2, None, 'final-technicalities-i'),
-              ('Final technicalities II', 2, None, 'final-technicalities-ii'),
-              ('Final technicalities III', 2, None, 'final-technicalities-iii'),
-              ('Final technicalities IV', 2, None, 'final-technicalities-iv'),
-              ('Back propagation', 2, None, 'back-propagation'),
-              ('Gradient descent', 2, None, 'gradient-descent'),
-              ('The code for solving the ODE',
-               2,
-               None,
-               'the-code-for-solving-the-ode'),
-              ('The network with one input layer, specified number of hidden '
-               'layers, and one output layer',
-               2,
-               None,
-               'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
-              ('Example: Population growth',
-               2,
-               None,
-               'example-population-growth'),
-              ('Setting up the problem', 2, None, 'setting-up-the-problem'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The program using Autograd',
-               2,
-               None,
-               'the-program-using-autograd'),
-              ('Using forward Euler to solve the ODE',
-               2,
-               None,
-               'using-forward-euler-to-solve-the-ode'),
-              ('Example: Solving the one dimensional Poisson equation',
-               2,
-               None,
-               'example-solving-the-one-dimensional-poisson-equation'),
-              ('The specific equation to solve for',
-               2,
-               None,
-               'the-specific-equation-to-solve-for'),
-              ('Solving the equation using Autograd',
-               2,
-               None,
-               'solving-the-equation-using-autograd'),
-              ('Comparing with a numerical scheme',
-               2,
-               None,
-               'comparing-with-a-numerical-scheme'),
-              ('Setting up the code', 2, None, 'setting-up-the-code'),
-              ('Partial Differential Equations',
-               2,
-               None,
-               'partial-differential-equations'),
-              ('Type of problem', 2, None, 'type-of-problem'),
-              ('Network requirements', 2, None, 'network-requirements'),
-              ('More details', 2, None, 'more-details'),
-              ('Example: The diffusion equation',
-               2,
-               None,
-               'example-the-diffusion-equation'),
-              ('Defining the problem', 2, None, 'defining-the-problem'),
-              ('Setting up the network using Autograd',
-               2,
-               None,
-               'setting-up-the-network-using-autograd'),
-              ('Setting up the network using Autograd; The trial solution',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-trial-solution'),
-              ('Why the jacobian?', 2, None, 'why-the-jacobian'),
-              ('Setting up the network using Autograd; The full program',
-               2,
-               None,
-               'setting-up-the-network-using-autograd-the-full-program'),
-              ('Example: Solving the wave equation with Neural Networks',
-               2,
-               None,
-               'example-solving-the-wave-equation-with-neural-networks'),
-              ('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
-              ('The trial solution', 2, None, 'the-trial-solution'),
-              ('The analytical solution', 2, None, 'the-analytical-solution'),
-              ('Solving the wave equation - the full program using Autograd',
-               2,
-               None,
-               'solving-the-wave-equation-the-full-program-using-autograd'),
-              ('Resources on differential equations and deep learning',
-               2,
-               None,
-               'resources-on-differential-equations-and-deep-learning')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -339,115 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal  note</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs080.html#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -459,28 +327,87 @@ MathJax.Hub.Config({
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
 <a name="part0066"></a>
 <!-- !split -->
-<h2 id="more-details" class="anchor">More details </h2>
+<h2 id="the-code-using-scikit-learn" class="anchor">The Code using Scikit-Learn </h2>
 
-<p>If the neural network evaluates \( g_t(x, P) \) at more values for \( x \), say \( N \) values \( x_i \) for \( i = 1, \dots, N \), then the <em>total</em> error to minimize becomes</p>
 
-$$
-\begin{equation} \tag{9}
-\min_{P}\Big\{\frac{1}{N} \sum_{i=1}^N  \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2 \Big\}
-\end{equation}
-$$
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> accuracy_score
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
 
-<p>Letting \( \boldsymbol{x} \) be a vector with elements \( x_i \) and \( C(\boldsymbol{x}, P) = \frac{1}{N} \sum_i  \big(g_t'(x_i, P) - ( -\gamma g_t(x_i, P) \big)^2 \) denote the cost function, the minimization problem that our network must solve, becomes</p>
+<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
+np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
 
-$$
-\min_{P} C(\boldsymbol{x}, P)
-$$
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
 
-<p>In terms of \( P_{\text{hidden} } \) and \( P_{\text{output} } \), this could also be expressed as</p>
+<span style="color: #408080; font-style: italic"># The XOR gate</span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+<span style="color: #408080; font-style: italic"># The OR gate</span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+<span style="color: #408080; font-style: italic"># The AND gate</span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+
+<span style="color: #408080; font-style: italic"># Defining the neural network</span>
+n_inputs, n_features <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape
+n_hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+n_categories <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+n_features <span style="color: #666666">=</span> <span style="color: #666666">2</span>
+
+eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
+lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
+<span style="color: #408080; font-style: italic"># store models for later use</span>
+DNN_scikit <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
+epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
+
+<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
+    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
+        dnn <span style="color: #666666">=</span> MLPClassifier(hidden_layer_sizes<span style="color: #666666">=</span>(n_hidden_neurons), activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;logistic&#39;</span>,
+                            alpha<span style="color: #666666">=</span>lmbd, learning_rate_init<span style="color: #666666">=</span>eta, max_iter<span style="color: #666666">=</span>epochs)
+        dnn<span style="color: #666666">.</span>fit(X, yXOR)
+        DNN_scikit[i][j] <span style="color: #666666">=</span> dnn
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate  = &quot;</span>, eta)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
+        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Accuracy score on data set: &quot;</span>, dnn<span style="color: #666666">.</span>score(X, yXOR))
+        <span style="color: #008000">print</span>()
+
+sns<span style="color: #666666">.</span>set()
+test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
+<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
+    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
+        dnn <span style="color: #666666">=</span> DNN_scikit[i][j]
+        test_pred <span style="color: #666666">=</span> dnn<span style="color: #666666">.</span>predict(X)
+        test_accuracy[i][j] <span style="color: #666666">=</span> accuracy_score(yXOR, test_pred)
+
+fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
+sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
+ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
+plt<span style="color: #666666">.</span>show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 
-<p>$$
-\min_{P_{\text{hidden} }, \ P_{\text{output} }} C(\boldsymbol{x}, \{P_{\text{hidden} }, P_{\text{output} }\})
-$$
-</p>
 
 <p>
 <!-- navigation buttons at the bottom of the page -->
@@ -497,18 +424,6 @@ $$
   <li><a href="._week41-bs064.html">65</a></li>
   <li><a href="._week41-bs065.html">66</a></li>
   <li class="active"><a href="._week41-bs066.html">67</a></li>
-  <li><a href="._week41-bs067.html">68</a></li>
-  <li><a href="._week41-bs068.html">69</a></li>
-  <li><a href="._week41-bs069.html">70</a></li>
-  <li><a href="._week41-bs070.html">71</a></li>
-  <li><a href="._week41-bs071.html">72</a></li>
-  <li><a href="._week41-bs072.html">73</a></li>
-  <li><a href="._week41-bs073.html">74</a></li>
-  <li><a href="._week41-bs074.html">75</a></li>
-  <li><a href="._week41-bs075.html">76</a></li>
-  <li><a href="">...</a></li>
-  <li><a href="._week41-bs102.html">103</a></li>
-  <li><a href="._week41-bs067.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
 </div>  <!-- end container -->
diff --git a/doc/pub/week41/html/week41-bs.html b/doc/pub/week41/html/week41-bs.html
index 8b355d07a..9aad39fba 100644
--- a/doc/pub/week41/html/week41-bs.html
+++ b/doc/pub/week41/html/week41-bs.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <!-- Bootstrap style: bootstrap -->
 <!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
@@ -37,14 +37,107 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -122,62 +215,7 @@ doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=d
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -205,64 +243,79 @@ MathJax.Hub.Config({
       <span class="icon-bar"></span>
       <span class="icon-bar"></span>
     </button>
-    <a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
+    <a class="navbar-brand" href="week41-bs.html">Week 41 Neural networks and constructing a neural network code</a>
   </div>
   <div class="navbar-collapse collapse navbar-responsive-collapse">
     <ul class="nav navbar-nav navbar-right">
       <li class="dropdown">
         <a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
         <ul class="dropdown-menu">
-     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix  multiplication</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs038.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs039.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs040.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs041.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid)  our choice?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs043.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs044.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs045.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs046.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs047.html#dropout" style="font-size: 80%;">Dropout</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs048.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs049.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs050.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
-     <!-- navigation toc: --> <li><a href="._week41-bs051.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;"><b>Plan for week 41</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs002.html#lecture-thursday-october-12" style="font-size: 80%;"><b>Lecture Thursday October 12</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs003.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs004.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs005.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs006.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs007.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs008.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs009.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs010.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs011.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs012.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs013.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs014.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs015.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs020.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs021.html#matrix-vector-notation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs022.html#matrix-vector-notation-and-activation" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Matrix-vector notation  and activation</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs023.html#activation-functions" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs024.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Activation functions, Logistic and Hyperbolic ones</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs025.html#relevance" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Relevance</a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs026.html#the-multilayer-perceptron-mlp" style="font-size: 80%;"><b>The multilayer  perceptron (MLP)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs027.html#from-one-to-many-layers-the-universal-approximation-theorem" style="font-size: 80%;"><b>From one to many layers, the universal approximation theorem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs028.html#deriving-the-back-propagation-code-for-a-multilayer-perceptron-model" style="font-size: 80%;"><b>Deriving the back propagation code for a multilayer perceptron model</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs029.html#definitions" style="font-size: 80%;"><b>Definitions</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs030.html#derivatives-and-the-chain-rule" style="font-size: 80%;"><b>Derivatives and the chain rule</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs031.html#derivative-of-the-cost-function" style="font-size: 80%;"><b>Derivative of the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs032.html#bringing-it-together-first-back-propagation-equation" style="font-size: 80%;"><b>Bringing it together, first back propagation equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs033.html#derivatives-in-terms-of-z-j-l" style="font-size: 80%;"><b>Derivatives in terms of \( z_j^L \)</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs034.html#bringing-it-together" style="font-size: 80%;"><b>Bringing it together</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs035.html#final-back-propagating-equation" style="font-size: 80%;"><b>Final back propagating equation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs038.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;"><b>Setting up the Back propagation algorithm</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs039.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;"><b>Setting up a Multi-layer perceptron model for classification</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs040.html#defining-the-cost-function" style="font-size: 80%;"><b>Defining the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs041.html#example-binary-classification-problem" style="font-size: 80%;"><b>Example: binary classification problem</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs042.html#the-softmax-function" style="font-size: 80%;"><b>The Softmax function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs043.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;"><b>Developing a code for doing neural networks with back propagation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs044.html#collect-and-pre-process-data" style="font-size: 80%;"><b>Collect and pre-process data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs045.html#train-and-test-datasets" style="font-size: 80%;"><b>Train and test datasets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs046.html#define-model-and-architecture" style="font-size: 80%;"><b>Define model and architecture</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs047.html#layers" style="font-size: 80%;"><b>Layers</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs048.html#weights-and-biases" style="font-size: 80%;"><b>Weights and biases</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs049.html#feed-forward-pass" style="font-size: 80%;"><b>Feed-forward pass</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs050.html#matrix-multiplications" style="font-size: 80%;"><b>Matrix multiplications</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs051.html#choose-cost-function-and-optimizer" style="font-size: 80%;"><b>Choose cost function and optimizer</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs052.html#optimizing-the-cost-function" style="font-size: 80%;"><b>Optimizing the cost function</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs053.html#regularization" style="font-size: 80%;"><b>Regularization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs054.html#matrix-multiplication" style="font-size: 80%;"><b>Matrix  multiplication</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs055.html#improving-performance" style="font-size: 80%;"><b>Improving performance</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs056.html#full-object-oriented-implementation" style="font-size: 80%;"><b>Full object-oriented implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs057.html#evaluate-model-performance-on-test-data" style="font-size: 80%;"><b>Evaluate model performance on test data</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs058.html#adjust-hyperparameters" style="font-size: 80%;"><b>Adjust hyperparameters</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs060.html#scikit-learn-implementation" style="font-size: 80%;"><b>scikit-learn implementation</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs061.html#visualization" style="font-size: 80%;"><b>Visualization</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs062.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;"><b>Testing our code for the XOR, OR and AND gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs063.html#the-and-and-xor-gates" style="font-size: 80%;"><b>The AND and XOR Gates</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs064.html#representing-the-data-sets" style="font-size: 80%;"><b>Representing the Data Sets</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs065.html#setting-up-the-neural-network" style="font-size: 80%;"><b>Setting up the Neural Network</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week41-bs066.html#the-code-using-scikit-learn" style="font-size: 80%;"><b>The Code using Scikit-Learn</b></a></li>
 
         </ul>
       </li>
@@ -276,7 +329,7 @@ MathJax.Hub.Config({
 <!-- ------------------- main content ---------------------- -->
 <div class="jumbotron">
 <center>
-<h1>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1>
+<h1>Week 41 Neural networks and constructing a neural network code</h1>
 </center>  <!-- document title -->
 
 <!-- author(s): Morten Hjorth-Jensen -->
@@ -317,7 +370,7 @@ MathJax.Hub.Config({
   <li><a href="._week41-bs008.html">9</a></li>
   <li><a href="._week41-bs009.html">10</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week41-bs051.html">52</a></li>
+  <li><a href="._week41-bs066.html">67</a></li>
   <li><a href="._week41-bs001.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
diff --git a/doc/pub/week41/html/week41-reveal.html b/doc/pub/week41/html/week41-reveal.html
index 745b4fbee..f74e19303 100644
--- a/doc/pub/week41/html/week41-reveal.html
+++ b/doc/pub/week41/html/week41-reveal.html
@@ -9,8 +9,8 @@ doconce format html week41-reveal.html week41-reveal reveal --html_slide_theme=b
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 
 <!-- reveal.js: https://lab.hakim.se/reveal-js/ -->
 
@@ -168,7 +168,7 @@ MathJax.Hub.Config({
 <section>
 <!-- ------------------- main content ---------------------- -->
 <center>
-<h1 style="text-align: center;">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1>
+<h1 style="text-align: center;">Week 41 Neural networks and constructing a neural network code</h1>
 </center>  <!-- document title -->
 
 <!-- author(s): Morten Hjorth-Jensen -->
@@ -197,42 +197,1334 @@ MathJax.Hub.Config({
 <section>
 <h2 id="plan-for-week-41">Plan for week 41 </h2>
 
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the active learning sessions on Tuesday and Wednesday</b>
+<p>
 <ul>
-<p><li> Building our own Feed-forward Neural Network and discussion of project 2.</li>
-<p><li> Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks.</li> 
+
+<p><li> Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation</li>
+
+<p><li> One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code</li>
+
+<p><li> Discussion of project 2</li>
+
+<p><li> Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent</li>
+</ul>
+</div>
+  
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the lecture on Thursday October 12, 2023</b>
+<p>
+<ul>
+
+<p><li> Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.</li>
+
+<p><li> Building our own Feed-forward Neural Network</li>
+
+<p><li> Readings and Videos:</li>
+<ul>
+
+<p><li> These lecture notes</li>
+
+<p><li> For neural networks we recommend Goodfellow et al chapter 6.</li>
+
+<p><li> <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_blank">Neural Networks demystified</a></li>
+
+<p><li> <a href="https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex" target="_blank">Building Neural Networks from scratch</a></li>
+
+<p><li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
+
+<p><li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
 </ul>
 <p>
-<p>"
-Reading suggestions: These notes, <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 10-11</a>.
-For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read.
+</ul>
+<p>
+<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+</div>
+</section>
+
+<section>
+<h2 id="lecture-thursday-october-12">Lecture Thursday October 12 </h2>
+</section>
+
+<section>
+<h2 id="introduction-to-neural-networks">Introduction to Neural networks </h2>
+
+<p>Artificial neural networks are computational systems that can learn to
+perform tasks by considering examples, generally without being
+programmed with any task-specific rules. It is supposed to mimic a
+biological system, wherein neurons interact by sending signals in the
+form of mathematical functions between layers. All layers can contain
+an arbitrary number of neurons, and each connection is represented by
+a weight variable.
 </p>
 </section>
 
 <section>
-<h2 id="videos-on-neural-networks">Videos on Neural Networks </h2>
+<h2 id="artificial-neurons">Artificial neurons  </h2>
 
-<ul>
-<p><li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
-<p><li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
-</ul>
+<p>The field of artificial neural networks has a long history of
+development, and is closely connected with the advancement of computer
+science and computers in general. A model of artificial neurons was
+first developed by McCulloch and Pitts in 1943 to study signal
+processing in the brain and has later been refined by others. The
+general idea is to mimic neural networks in the human brain, which is
+composed of billions of neurons that communicate with each other by
+sending electrical signals.  Each neuron accumulates its incoming
+signals, which must exceed an activation threshold to yield an
+output. If the threshold is not overcome, the neuron remains inactive,
+i.e. has zero output.
+</p>
+
+<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
+\tag{1}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
+a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
+</p>
+
+<p>Conceptually, it is helpful to divide neural networks into four
+categories:
+</p>
+<ol>
+<p><li> general purpose neural networks for supervised learning,</li>
+<p><li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
+<p><li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
+<p><li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
+</ol>
 <p>
-<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+<p>In natural science, DNNs and CNNs have already found numerous
+applications. In statistical physics, they have been applied to detect
+phase transitions in 2D Ising and Potts models, lattice gauge
+theories, and different phases of polymers, or solving the
+Navier-Stokes equation in weather forecasting.  Deep learning has also
+found interesting applications in quantum physics. Various quantum
+phase transitions can be detected and studied using DNNs and CNNs,
+topological phases, and even non-equilibrium many-body
+localization. Representing quantum states as DNNs quantum state
+tomography are among some of the impressive achievements to reveal the
+potential of DNNs to facilitate the study of quantum systems.
+</p>
+
+<p>In quantum information theory, it has been shown that one can perform
+gate decompositions with the help of neural. 
+</p>
+
+<p>The applications are not limited to the natural sciences. There is a
+plethora of applications in essentially all disciplines, from the
+humanities to life science and medicine.
+</p>
 </section>
 
 <section>
-<h2 id="review-of-the-back-propagation-algorithm">Review of the back propagation algorithm </h2>
+<h2 id="neural-network-types">Neural network types </h2>
 
-<p>During the last lecture we discussed in detail the back propagation
-algorithm. This algorithm is based on a repeated application of the
-chain rule. Let us bring back the basic equation and at the same time
-link this with the basic mathematics of automatic differentiation.
+<p>An artificial neural network (ANN), is a computational model that
+consists of layers of connected neurons, or nodes or units.  We will
+refer to these interchangeably as units or nodes, and sometimes as
+neurons.
+</p>
+
+<p>It is supposed to mimic a biological nervous system by letting each
+neuron interact with other neurons by sending signals in the form of
+mathematical functions between layers.  A wide variety of different
+ANNs have been developed, but most of them consist of an input layer,
+an output layer and eventual layers in-between, called <em>hidden
+layers</em>. All layers can contain an arbitrary number of nodes, and each
+connection between two nodes is associated with a weight variable.
+</p>
+
+<p>Neural networks (also called neural nets) are neural-inspired
+nonlinear models for supervised learning.  As we will see, neural nets
+can be viewed as natural, more powerful extensions of supervised
+learning methods such as linear and logistic regression and soft-max
+methods we discussed earlier.
+</p>
+</section>
+
+<section>
+<h2 id="feed-forward-neural-networks">Feed-forward neural networks </h2>
+
+<p>The feed-forward neural network (FFNN) was the first and simplest type
+of ANNs that were devised. In this network, the information moves in
+only one direction: forward through the layers.
+</p>
+
+<p>Nodes are represented by circles, while the arrows display the
+connections between the nodes, including the direction of information
+flow. Additionally, each arrow corresponds to a weight variable
+(figure to come).  We observe that each node in a layer is connected
+to <em>all</em> nodes in the subsequent layer, making this a so-called
+<em>fully-connected</em> FFNN.
+</p>
+</section>
+
+<section>
+<h2 id="convolutional-neural-network">Convolutional Neural Network </h2>
+
+<p>A different variant of FFNNs are <em>convolutional neural networks</em>
+(CNNs), which have a connectivity pattern inspired by the animal
+visual cortex. Individual neurons in the visual cortex only respond to
+stimuli from small sub-regions of the visual field, called a receptive
+field. This makes the neurons well-suited to exploit the strong
+spatially local correlation present in natural images. The response of
+each neuron can be approximated mathematically as a convolution
+operation.  (figure to come)
+</p>
+
+<p>Convolutional neural networks emulate the behaviour of neurons in the
+visual cortex by enforcing a <em>local</em> connectivity pattern between
+nodes of adjacent layers: Each node in a convolutional layer is
+connected only to a subset of the nodes in the previous layer, in
+contrast to the fully-connected FFNN.  Often, CNNs consist of several
+convolutional layers that learn local features of the input, with a
+fully-connected layer at the end, which gathers all the local data and
+produces the outputs. They have wide applications in image and video
+recognition.
+</p>
+</section>
+
+<section>
+<h2 id="recurrent-neural-networks">Recurrent neural networks </h2>
+
+<p>So far we have only mentioned ANNs where information flows in one
+direction: forward. <em>Recurrent neural networks</em> on the other hand,
+have connections between nodes that form directed <em>cycles</em>. This
+creates a form of internal memory which are able to capture
+information on what has been calculated before; the output is
+dependent on the previous computations. Recurrent NNs make use of
+sequential information by performing the same task for every element
+in a sequence, where each element depends on previous elements. An
+example of such information is sentences, making recurrent NNs
+especially well-suited for handwriting and speech recognition.
+</p>
+</section>
+
+<section>
+<h2 id="other-types-of-networks">Other types of networks </h2>
+
+<p>There are many other kinds of ANNs that have been developed. One type
+that is specifically designed for interpolation in multidimensional
+space is the radial basis function (RBF) network. RBFs are typically
+made up of three layers: an input layer, a hidden layer with
+non-linear radial symmetric activation functions and a linear output
+layer (''linear'' here means that each node in the output layer has a
+linear activation function). The layers are normally fully-connected
+and there are no cycles, thus RBFs can be viewed as a type of
+fully-connected FFNN. They are however usually treated as a separate
+type of NN due the unusual activation functions.
+</p>
+</section>
+
+<section>
+<h2 id="multilayer-perceptrons">Multilayer perceptrons  </h2>
+
+<p>One uses often so-called fully-connected feed-forward neural networks
+with three or more layers (an input layer, one or more hidden layers
+and an output layer) consisting of neurons that have non-linear
+activation functions.
+</p>
+
+<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
+</section>
+
+<section>
+<h2 id="why-multilayer-perceptrons">Why multilayer perceptrons?  </h2>
+
+<p>According to the <em>Universal approximation theorem</em>, a feed-forward
+neural network with just a single hidden layer containing a finite
+number of neurons can approximate a continuous multidimensional
+function to arbitrary accuracy, assuming the activation function for
+the hidden layer is a <b>non-constant, bounded and
+monotonically-increasing continuous function</b>.
+</p>
+
+<p>Note that the requirements on the activation function only applies to
+the hidden layer, the output nodes are always assumed to be linear, so
+as to not restrict the range of output values.
+</p>
+</section>
+
+<section>
+<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model">Illustration of a single perceptron model and a multi-perceptron model </h2>
+
+<center>  <!-- FIGURE -->
+<hr class="figure">
+<center>
+<p class="caption">Figure 1:  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer. </p>
+</center>
+<p><img src="figures/nns.png" width="600" align="bottom"></p>
+</center>
+</section>
+
+<section>
+<h2 id="examples-of-xor-or-and-and-gates">Examples of XOR, OR and AND gates </h2>
+
+<p>Let us first try to fit various gates using standard linear
+regression. The gates we are thinking of are the classical XOR, OR and
+AND gates, well-known elements in computer science. The tables here
+show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
+specific target \( y_i \).
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
+<span style="color: #CD5555">Simple code that tests XOR, OR and AND gates with linear regression</span>
+<span style="color: #CD5555">&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
+<span style="color: #228B22"># Design matrix</span>
+X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX  matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
+Xinv = np.linalg.pinv(X.T @ X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX  matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># The XOR gate </span>
+yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
+ThetaXOR  = Xinv @ X.T @ yXOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
+ThetaOR  = Xinv @ X.T @ yOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
+ThetaAND  = Xinv @ X.T @ yAND
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>What is happening here?</p>
+</section>
+
+<section>
+<h2 id="does-logistic-regression-do-a-better-job">Does Logistic Regression do a better Job? </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
+<span style="color: #CD5555">Simple code that tests XOR and OR gates with linear regression</span>
+<span style="color: #CD5555">and logistic regression</span>
+<span style="color: #CD5555">&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
+
+<span style="color: #228B22"># Design matrix</span>
+X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX  matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
+Xinv = np.linalg.pinv(X.T @ X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX  matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># The XOR gate </span>
+yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
+ThetaXOR  = Xinv @ X.T @ yXOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
+ThetaOR  = Xinv @ X.T @ yOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
+ThetaAND  = Xinv @ X.T @ yAND
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># Now we change to logistic regression</span>
+
+
+<span style="color: #228B22"># Logistic Regression</span>
+logreg = LogisticRegression()
+logreg.fit(X, yOR)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for OR gate: {:.2f}&quot;</span>.format(logreg.score(X,yOR)))
+
+logreg.fit(X, yXOR)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for XOR gate: {:.2f}&quot;</span>.format(logreg.score(X,yXOR)))
+
+
+logreg.fit(X, yAND)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for AND gate: {:.2f}&quot;</span>.format(logreg.score(X,yAND)))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>Not exactly impressive, but somewhat better.</p>
+</section>
+
+<section>
+<h2 id="adding-neural-networks">Adding Neural Networks </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># and now neural networks with Scikit-Learn and the XOR</span>
+
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.neural_network</span> <span style="color: #8B008B; font-weight: bold">import</span> MLPClassifier
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_classification
+X, yXOR = make_classification(n_samples=<span style="color: #B452CD">100</span>, random_state=<span style="color: #B452CD">1</span>)
+FFNN = MLPClassifier(random_state=<span style="color: #B452CD">1</span>, max_iter=<span style="color: #B452CD">300</span>).fit(X, yXOR)
+FFNN.predict_proba(X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Test set accuracy with Feed Forward Neural Network  for XOR gate:{</span>FFNN.score(X, yXOR)<span style="color: #CD5555">}&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+</section>
+
+<section>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output \( y \) is produced via the activation function \( f \)</p>
+<p>&nbsp;<br>
+$$
+ y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
+$$
+<p>&nbsp;<br>
+
+<p>This function receives \( x_i \) as inputs.
+Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \). 
+In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
+the neurons in the preceding layer. Furthermore, an MLP is
+fully-connected, which means that each neuron receives a weighted sum
+of the outputs of <em>all</em> neurons in the previous layer.
+</p>
+</section>
+
+<section>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
+\tag{2}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>Here \( b_i \) is the so-called bias which is normally needed in
+case of zero activation weights or inputs. How to fix the biases and
+the weights will be discussed below.  The value of \( z_i^1 \) is the
+argument to the activation function \( f_i \) of each node \( i \), The
+variable \( M \) stands for all possible inputs to a given node \( i \) in the
+first layer.  We define  the output \( y_i^1 \) of all neurons in layer 1 as
+</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
+\tag{3}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>where we assume that all nodes in the same layer have identical
+activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
+In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
+</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
+\tag{4}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
+all the nodes in the first hidden layer are computed, the values of
+the subsequent layer can be calculated and so forth until the output
+is obtained.
+</p>
+</section>
+
+<section>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output of neuron \( i \) in layer 2 is thus,</p>
+
+<p>&nbsp;<br>
+$$
+\begin{align}
+ y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) 
+\tag{5}\\
+ &= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
+\tag{6}
+\end{align}
+$$
+<p>&nbsp;<br>
+
+<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
+
+<p>&nbsp;<br>
+$$
+\begin{align}
+ y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) 
+\tag{7}\\
+ &= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+  + b_1^3\right]
+\tag{8}
+\end{align}
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>We can generalize this expression to an MLP with \( l \) hidden
+layers. The complete functional form is,
+</p>
+
+<p>&nbsp;<br>
+$$
+\begin{align}
+&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
+\tag{9}
+\end{align}
+$$
+<p>&nbsp;<br>
+
+<p>which illustrates a basic property of MLPs: The only independent
+variables are the input values \( x_n \).
+</p>
+</section>
+
+<section>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>This confirms that an MLP, despite its quite convoluted mathematical
+form, is nothing more than an analytic function, specifically a
+mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
+\hat{y} \in \mathbb{R}^m \).
+</p>
+
+<p>Furthermore, the flexibility and universality of an MLP can be
+illustrated by realizing that the expression is essentially a nested
+sum of scaled activation functions of the form
+</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ f(x) = c_1 f(c_2 x + c_3) + c_4
+\tag{10}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>where the parameters \( c_i \) are weights and biases. By adjusting these
+parameters, the activation functions can be shifted up and down or
+left and right, change slope or be rescaled which is the key to the
+flexibility of a neural network.
+</p>
+</section>
+
+<section>
+<h3 id="matrix-vector-notation">Matrix-vector notation </h3>
+
+<p>We can introduce a more convenient notation for the activations in an A NN. </p>
+
+<p>Additionally, we can represent the biases and activations
+as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector 
+is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively. 
+</p>
+
+<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors. 
+With this notation, the sum becomes a matrix-vector multiplication, and we can write
+the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
+</p>
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
+ f_2\left(\left[\begin{array}{ccc}
+    w^2_{11} &w^2_{12} &w^2_{13} \\
+    w^2_{21} &w^2_{22} &w^2_{23} \\
+    w^2_{31} &w^2_{32} &w^2_{33} \\
+    \end{array} \right] \cdot
+    \left[\begin{array}{c}
+           y^1_1 \\
+           y^1_2 \\
+           y^1_3 \\
+          \end{array}\right] + 
+    \left[\begin{array}{c}
+           b^2_1 \\
+           b^2_2 \\
+           b^2_3 \\
+          \end{array}\right]\right).
+\tag{11}
+\end{equation}
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h3 id="matrix-vector-notation-and-activation">Matrix-vector notation  and activation </h3>
+
+<p>The activation of node \( i \) in layer 2 is</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+ y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
+ f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
+\tag{12}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>This is not just a convenient and compact notation, but also a useful
+and intuitive way to think about MLPs: The output is calculated by a
+series of matrix-vector multiplications and vector additions that are
+used as input to the activation functions. For each operation
+\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
+</p>
+</section>
+
+<section>
+<h3 id="activation-functions">Activation functions  </h3>
+
+<p>A property that characterizes a neural network, other than its
+connectivity, is the choice of activation function(s).  As described
+in, the following restrictions are imposed on an activation function
+for a FFNN to fulfill the universal approximation theorem
+</p>
+
+<ul>
+
+<p><li> Non-constant</li>
+
+<p><li> Bounded</li>
+
+<p><li> Monotonically-increasing</li>
+
+<p><li> Continuous</li>
+</ul>
+</section>
+
+<section>
+<h3 id="activation-functions-logistic-and-hyperbolic-ones">Activation functions, Logistic and Hyperbolic ones  </h3>
+
+<p>The second requirement excludes all linear functions. Furthermore, in
+a MLP with only linear activation functions, each layer simply
+performs a linear transformation of its inputs.
+</p>
+
+<p>Regardless of the number of layers, the output of the NN will be
+nothing but a linear function of the inputs. Thus we need to introduce
+some kind of non-linearity to the NN to be able to fit non-linear
+functions Typical examples are the logistic <em>Sigmoid</em>
+</p>
+
+<p>&nbsp;<br>
+$$
+ f(x) = \frac{1}{1 + e^{-x}},
+$$
+<p>&nbsp;<br>
+
+<p>and the <em>hyperbolic tangent</em> function</p>
+<p>&nbsp;<br>
+$$
+ f(x) = \tanh(x)
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h3 id="relevance">Relevance </h3>
+
+<p>The <em>sigmoid</em> function are more biologically plausible because the
+output of inactive neurons are zero. Such activation function are
+called <em>one-sided</em>. However, it has been shown that the hyperbolic
+tangent performs better than the sigmoid for training MLPs.  has
+become the most popular for <em>deep neural networks</em>
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;The sigmoid function (or the logistic curve) is a </span>
+<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
+<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
+<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm.&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
+
+z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.1</span>)
+sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
+sigma = sigma_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, sigma)
+ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
+ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;sigmoid function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Step Function&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.02</span>)
+step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z &gt;= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
+step = step_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, step)
+ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
+ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;step function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Sine Function&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
+t = numpy.sin(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, t)
+ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
+ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;sine function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Plots a graph of the squashing function used by a rectified linear</span>
+<span style="color: #CD5555">unit&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>, <span style="color: #B452CD">.1</span>)
+zero = numpy.zeros(<span style="color: #658b00">len</span>(z))
+y = numpy.max([zero, z], axis=<span style="color: #B452CD">0</span>)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, y)
+ax.set_ylim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
+ax.set_xlim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;Rectified linear unit&#39;</span>)
+
+plt.show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+</section>
+
+<section>
+<h2 id="the-multilayer-perceptron-mlp">The multilayer  perceptron (MLP) </h2>
+
+<p>The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of</p>
+<ol>
+<p><li> A neural network with one or more layers of nodes between the input and the output nodes.</li>
+<p><li> The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.</li>
+<p><li> The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.</li>
+</ol>
+<p>
+<p>As a convention it is normal to call  a  network with one layer of input units, one layer of hidden
+units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
+</p>
+
+<p>For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.
+Hereafter we will call the various entities of a layer for nodes.
+There are also no connections within a single layer.
+</p>
+
+<p>The number of input nodes does not need to equal the number of output
+nodes. This applies also to the hidden layers. Each layer may have its
+own number of nodes and activation functions.
+</p>
+
+<p>The hidden layers have their name from the fact that they are not
+linked to observables and as we will see below when we define the
+so-called activation \( \hat{z} \), we can think of this as a basis
+expansion of the original inputs \( \hat{x} \). The difference however
+between neural networks and say linear regression is that now these
+basis functions (which will correspond to the weights in the network)
+are learned from data. This results in  an important difference between
+neural networks and deep learning approaches on one side and methods
+like logistic regression or linear regression and their modifications on the other side.
+</p>
+</section>
+
+<section>
+<h2 id="from-one-to-many-layers-the-universal-approximation-theorem">From one to many layers, the universal approximation theorem </h2>
+
+<p>A neural network with only one layer, what we called the simple
+perceptron, is best suited if we have a standard binary model with
+clear (linear) boundaries between the outcomes. As such it could
+equally well be replaced by standard linear regression or logistic
+regression. Networks with one or more hidden layers approximate
+systems with more complex boundaries.
+</p>
+
+<p>As stated earlier, 
+an important theorem in studies of neural networks, restated without
+proof here, is the <a href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf" target="_blank">universal approximation
+theorem</a>.
+</p>
+
+<p>It states that a feed-forward network with a single hidden layer
+containing a finite number of neurons can approximate continuous
+functions on compact subsets of real functions. The theorem thus
+states that simple neural networks can represent a wide variety of
+interesting functions when given appropriate parameters. It is the
+multilayer feedforward architecture itself which gives neural networks
+the potential of being universal approximators.
+</p>
+</section>
+
+<section>
+<h2 id="deriving-the-back-propagation-code-for-a-multilayer-perceptron-model">Deriving the back propagation code for a multilayer perceptron model </h2>
+
+<p>As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.
+The unknowwn quantities are our weights \( w_{ij} \) and we need to find an algorithm for changing them so that our errors are as small as possible.
+This leads us to the famous <a href="https://www.nature.com/articles/323533a0" target="_blank">back propagation algorithm</a>.
+</p>
+
+<p>The questions we want to ask are how do changes in the biases and the
+weights in our network change the cost function and how can we use the
+final output to modify the weights?
+</p>
+
+<p>To derive these equations let us start with a plain regression problem
+and define our cost function as
+</p>
+
+<p>&nbsp;<br>
+$$
+{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2, 
+$$
+<p>&nbsp;<br>
+
+<p>where the $t_i$s are our \( n \) targets (the values we want to
+reproduce), while the outputs of the network after having propagated
+all inputs \( \hat{x} \) are given by \( y_i \).  Below we will demonstrate
+how the basic equations arising from the back propagation algorithm
+can be modified in order to study classification problems with \( K \)
+classes.
+</p>
+</section>
+
+<section>
+<h2 id="definitions">Definitions </h2>
+
+<p>With our definition of the targets \( \hat{t} \), the outputs of the
+network \( \hat{y} \) and the inputs \( \hat{x} \) we
+define now the activation \( z_j^l \) of node/neuron/unit \( j \) of the
+\( l \)-th layer as a function of the bias, the weights which add up from
+the previous layer \( l-1 \) and the forward passes/outputs
+\( \hat{a}^{l-1} \) from the previous layer as
+</p>
+
+<p>&nbsp;<br>
+$$
+z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,
+$$
+<p>&nbsp;<br>
+
+<p>where \( b_k^l \) are the biases from layer \( l \).  Here \( M_{l-1} \)
+represents the total number of nodes/neurons/units of layer \( l-1 \). The
+figure here illustrates this equation.  We can rewrite this in a more
+compact form as the matrix-vector products we discussed earlier,
+</p>
+
+<p>&nbsp;<br>
+$$
+\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
+$$
+<p>&nbsp;<br>
+
+<p>With the activation values \( \hat{z}^l \) we can in turn define the
+output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
+activation function. In the examples here we will use the sigmoid
+function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
+and their nodes.  It means we have
+</p>
+
+<p>&nbsp;<br>
+$$
+a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h2 id="derivatives-and-the-chain-rule">Derivatives and the chain rule </h2>
+
+<p>From the definition of the activation \( z_j^l \) we have</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
+$$
+<p>&nbsp;<br>
+
+<p>and</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. 
+$$
+<p>&nbsp;<br>
+
+<p>With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)). 
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h2 id="derivative-of-the-cost-function">Derivative of the cost function </h2>
+
+<p>With these definitions we can now compute the derivative of the cost function in terms of the weights.</p>
+
+<p>Let us specialize to the output layer \( l=L \). Our cost function is</p>
+<p>&nbsp;<br>
+$$
+{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2, 
+$$
+<p>&nbsp;<br>
+
+<p>The derivative of this function with respect to the weights is</p>
+
+<p>&nbsp;<br>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}}, 
+$$
+<p>&nbsp;<br>
+
+<p>The last partial derivative can easily be computed and reads (by applying the chain rule)</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},  
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h2 id="bringing-it-together-first-back-propagation-equation">Bringing it together, first back propagation equation </h2>
+
+<p>We have thus</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1}, 
+$$
+<p>&nbsp;<br>
+
+<p>Defining</p>
+<p>&nbsp;<br>
+$$
+\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+$$
+<p>&nbsp;<br>
+
+<p>and using the Hadamard product of two vectors we can write this as</p>
+<p>&nbsp;<br>
+$$
+\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
+$$
+<p>&nbsp;<br>
+
+<p>This is an important expression. The second term on the right handside
+measures how fast the cost function is changing as a function of the $j$th
+output activation.  If, for example, the cost function doesn't depend
+much on a particular output node \( j \), then \( \delta_j^L \) will be small,
+which is what we would expect. The first term on the right, measures
+how fast the activation function \( f \) is changing at a given activation
+value \( z_j^L \).
+</p>
+
+<p>Notice that everything in the above equations is easily computed.  In
+particular, we compute \( z_j^L \) while computing the behaviour of the
+network, and it is only a small additional overhead to compute
+\( f'(z^L_j) \).  The exact form of the derivative with respect to the
+output depends on the form of the cost function.
+However, provided the cost function is known there should be little
+trouble in calculating
+</p>
+
+<p>&nbsp;<br>
+$$
+\frac{\partial {\cal C}}{\partial (a_j^L)}
+$$
+<p>&nbsp;<br>
+
+<p>With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely</p>
+<p>&nbsp;<br>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
+$$
+<p>&nbsp;<br>
+</section>
+
+<section>
+<h2 id="derivatives-in-terms-of-z-j-l">Derivatives in terms of \( z_j^L \) </h2>
+
+<p>It is also easy to see that our previous equation can be written as</p>
+
+<p>&nbsp;<br>
+$$
+\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
+$$
+<p>&nbsp;<br>
+
+<p>which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely</p>
+<p>&nbsp;<br>
+$$
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
+$$
+<p>&nbsp;<br>
+
+<p>That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias. </p>
+</section>
+
+<section>
+<h2 id="bringing-it-together">Bringing it together </h2>
+
+<p>We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>The starting equations</b>
+<p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
+\tag{13}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>and</p>
+<p>&nbsp;<br>
+$$
+\begin{equation}
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\tag{14}
+\end{equation}
+$$
+<p>&nbsp;<br>
+
+<p>and</p>
+
+<p>&nbsp;<br>
+$$
+\begin{equation}
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
+\tag{15}
+\end{equation}
+$$
+<p>&nbsp;<br>
+</div>
+
+<p>An interesting consequence of the above equations is that when the
+activation \( a_k^{L-1} \) is small, the gradient term, that is the
+derivative of the cost function with respect to the weights, will also
+tend to be small. We say then that the weight learns slowly, meaning
+that it changes slowly when we minimize the weights via say gradient
+descent. In this case we say the system learns slowly.
+</p>
+
+<p>Another interesting feature is that is when the activation function,
+represented by the sigmoid function here, is rather flat when we move towards
+its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
+cases, the derivatives of the activation function will also be close
+to zero, meaning again that the gradients will be small and the
+network learns slowly again.
+</p>
+
+<p>We need a fourth equation and we are set. We are going to propagate
+backwards in order to the determine the weights and biases. In order
+to do so we need to represent the error in the layer before the final
+one \( L-1 \) in terms of the errors in the final output layer.
+</p>
+</section>
+
+<section>
+<h2 id="final-back-propagating-equation">Final back propagating equation </h2>
+
+<p>We have that (replacing \( L \) with a general layer \( l \))</p>
+<p>&nbsp;<br>
+$$
+\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
+$$
+<p>&nbsp;<br>
+
+<p>We want to express this in terms of the equations for layer \( l+1 \). Using the chain rule and summing over all \( k \) entries we have</p>
+
+<p>&nbsp;<br>
+$$
+\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
+$$
+<p>&nbsp;<br>
+
+<p>and recalling that</p>
+<p>&nbsp;<br>
+$$
+z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
+$$
+<p>&nbsp;<br>
+
+<p>with \( M_l \) being the number of nodes in layer \( l \), we obtain</p>
+<p>&nbsp;<br>
+$$
+\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
+$$
+<p>&nbsp;<br>
+
+<p>This is our final equation.</p>
+
+<p>We are now ready to set up the algorithm for back propagation and learning the weights and biases.</p>
+</section>
+
+<section>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \hat{x} \) and the activations
+\( \hat{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \hat{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \hat{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \hat{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \hat{\delta}^L \) by computing all</p>
+<p>&nbsp;<br>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+<p>&nbsp;<br>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+<p>&nbsp;<br>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+<p>&nbsp;<br>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+<p>&nbsp;<br>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+<p>&nbsp;<br>
+
+<p>&nbsp;<br>
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+<p>&nbsp;<br>
+</div>
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
 </p>
 </section>
 
 <section>
 <h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
 
-<p>The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+<p>The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
+\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \boldsymbol{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
+<p>&nbsp;<br>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+<p>&nbsp;<br>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+<p>&nbsp;<br>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+<p>&nbsp;<br>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+<p>&nbsp;<br>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+<p>&nbsp;<br>
+
+<p>&nbsp;<br>
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+<p>&nbsp;<br>
+</div>
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+</p>
+</section>
+
+<section>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
 
 <div class="alert alert-block alert-block alert-text-normal">
 <b></b>
@@ -1896,898 +3188,6 @@ plt.show()
 </div>
 </section>
 
-<section>
-<h2 id="building-neural-networks-in-tensorflow-and-keras">Building neural networks in Tensorflow and Keras </h2>
-
-<p>Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn
-and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
-and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  
-</p>
-
-<p>In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
-clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
-NumPy arrays.
-</p>
-</section>
-
-<section>
-<h2 id="tensorflow">Tensorflow </h2>
-
-<p>Tensorflow is an open source library machine learning library
-developed by the Google Brain team for internal use. It was released
-under the Apache 2.0 open source license in November 9, 2015.
-</p>
-
-<p>Tensorflow is a computational framework that allows you to construct
-machine learning models at different levels of abstraction, from
-high-level, object-oriented APIs like Keras, down to the C++ kernels
-that Tensorflow is built upon. The higher levels of abstraction are
-simpler to use, but less flexible, and our choice of implementation
-should reflect the problems we are trying to solve.
-</p>
-
-<p><a href="https://www.tensorflow.org/guide/graphs" target="_blank">Tensorflow uses</a> so-called graphs to represent your computation
-in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
-to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
-</p>
-
-<p>In this guide we will analyze the same data as we did in our NumPy and
-scikit-learn tutorial, gathered from the MNIST database of images. We
-will give an introduction to the lower level Python Application
-Program Interfaces (APIs), and see how we use them to build our graph.
-Then we will build (effectively) the same graph in Keras, to see just
-how simple solving a machine learning problem can be.
-</p>
-
-<p>To install tensorflow on Unix/Linux systems, use pip as</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">pip3 install tensorflow
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
-(current release of CPU-only TensorFlow)
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">conda create -n tf tensorflow
-conda activate tf
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>To install the current release of GPU TensorFlow</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">conda create -n tf-gpu tensorflow-gpu
-conda activate tf-gpu
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-</section>
-
-<section>
-<h2 id="using-keras">Using Keras </h2>
-
-<p>Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
-that supports Tensorflow, CTNK and Theano as backends.  
-If you have Anaconda installed you may run the following command
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">conda install keras
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>You can look up the <a href="https://keras.io/" target="_blank">instructions here</a> for more information.</p>
-
-<p>We will to a large extent use <b>keras</b> in this course. </p>
-</section>
-
-<section>
-<h2 id="collect-and-pre-process-data">Collect and pre-process data </h2>
-
-<p>Let us look again at the MINST data set.</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># import necessary packages</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
-
-
-<span style="color: #228B22"># ensure the same random numbers appear every time</span>
-np.random.seed(<span style="color: #B452CD">0</span>)
-
-<span style="color: #228B22"># display images in notebook</span>
-%matplotlib inline
-plt.rcParams[<span style="color: #CD5555">&#39;figure.figsize&#39;</span>] = (<span style="color: #B452CD">12</span>,<span style="color: #B452CD">12</span>)
-
-
-<span style="color: #228B22"># download MNIST dataset</span>
-digits = datasets.load_digits()
-
-<span style="color: #228B22"># define inputs and labels</span>
-inputs = digits.images
-labels = digits.target
-
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;labels = (n_inputs) = &quot;</span> + <span style="color: #658b00">str</span>(labels.shape))
-
-
-<span style="color: #228B22"># flatten the image</span>
-<span style="color: #228B22"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
-n_inputs = <span style="color: #658b00">len</span>(inputs)
-inputs = inputs.reshape(n_inputs, -<span style="color: #B452CD">1</span>)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;X = (n_inputs, n_features) = &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-
-
-<span style="color: #228B22"># choose some random images to display</span>
-indices = np.arange(n_inputs)
-random_indices = np.random.choice(indices, size=<span style="color: #B452CD">5</span>)
-
-<span style="color: #8B008B; font-weight: bold">for</span> i, image <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(digits.images[random_indices]):
-    plt.subplot(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">5</span>, i+<span style="color: #B452CD">1</span>)
-    plt.axis(<span style="color: #CD5555">&#39;off&#39;</span>)
-    plt.imshow(image, cmap=plt.cm.gray_r, interpolation=<span style="color: #CD5555">&#39;nearest&#39;</span>)
-    plt.title(<span style="color: #CD5555">&quot;Label: %d&quot;</span> % digits.target[random_indices[i]])
-plt.show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Sequential      <span style="color: #228B22">#This allows appending layers to existing models</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense           <span style="color: #228B22">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> optimizers             <span style="color: #228B22">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers           <span style="color: #228B22">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> to_categorical   <span style="color: #228B22">#This allows using categorical cross entropy as the cost function</span>
-
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
-
-<span style="color: #228B22"># one-hot representation of labels</span>
-labels = to_categorical(labels)
-
-<span style="color: #228B22"># split into train and test data</span>
-train_size = <span style="color: #B452CD">0.8</span>
-test_size = <span style="color: #B452CD">1</span> - train_size
-X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
-                                                    test_size=test_size)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">epochs = <span style="color: #B452CD">100</span>
-batch_size = <span style="color: #B452CD">100</span>
-n_neurons_layer1 = <span style="color: #B452CD">100</span>
-n_neurons_layer2 = <span style="color: #B452CD">50</span>
-n_categories = <span style="color: #B452CD">10</span>
-eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
-lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">create_neural_network_keras</span>(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
-    model = Sequential()
-    model.add(Dense(n_neurons_layer1, activation=<span style="color: #CD5555">&#39;sigmoid&#39;</span>, kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_neurons_layer2, activation=<span style="color: #CD5555">&#39;sigmoid&#39;</span>, kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_categories, activation=<span style="color: #CD5555">&#39;softmax&#39;</span>))
-    
-    sgd = optimizers.SGD(lr=eta)
-    model.compile(loss=<span style="color: #CD5555">&#39;categorical_crossentropy&#39;</span>, optimizer=sgd, metrics=[<span style="color: #CD5555">&#39;accuracy&#39;</span>])
-    
-    <span style="color: #8B008B; font-weight: bold">return</span> model
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;">DNN_keras = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
-        
-<span style="color: #8B008B; font-weight: bold">for</span> i, eta <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(eta_vals):
-    <span style="color: #8B008B; font-weight: bold">for</span> j, lmbd <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(lmbd_vals):
-        DNN = create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
-                                         eta=eta, lmbd=lmbd)
-        DNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=<span style="color: #B452CD">0</span>)
-        scores = DNN.evaluate(X_test, Y_test)
-        
-        DNN_keras[i][j] = DNN
-        
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Learning rate = &quot;</span>, eta)
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test accuracy: %.3f&quot;</span> % scores[<span style="color: #B452CD">1</span>])
-        <span style="color: #658b00">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># optional</span>
-<span style="color: #228B22"># visual representation of grid search</span>
-<span style="color: #228B22"># uses seaborn heatmap, could probably do this in matplotlib</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
-
-sns.set()
-
-train_accuracy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)))
-test_accuracy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)))
-
-<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(eta_vals)):
-    <span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(lmbd_vals)):
-        DNN = DNN_keras[i][j]
-
-        train_accuracy[i][j] = DNN.evaluate(X_train, Y_train)[<span style="color: #B452CD">1</span>]
-        test_accuracy[i][j] = DNN.evaluate(X_test, Y_test)[<span style="color: #B452CD">1</span>]
-
-        
-fig, ax = plt.subplots(figsize = (<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
-sns.heatmap(train_accuracy, annot=<span style="color: #8B008B; font-weight: bold">True</span>, ax=ax, cmap=<span style="color: #CD5555">&quot;viridis&quot;</span>)
-ax.set_title(<span style="color: #CD5555">&quot;Training Accuracy&quot;</span>)
-ax.set_ylabel(<span style="color: #CD5555">&quot;$\eta$&quot;</span>)
-ax.set_xlabel(<span style="color: #CD5555">&quot;$\lambda$&quot;</span>)
-plt.show()
-
-fig, ax = plt.subplots(figsize = (<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
-sns.heatmap(test_accuracy, annot=<span style="color: #8B008B; font-weight: bold">True</span>, ax=ax, cmap=<span style="color: #CD5555">&quot;viridis&quot;</span>)
-ax.set_title(<span style="color: #CD5555">&quot;Test Accuracy&quot;</span>)
-ax.set_ylabel(<span style="color: #CD5555">&quot;$\eta$&quot;</span>)
-ax.set_xlabel(<span style="color: #CD5555">&quot;$\lambda$&quot;</span>)
-plt.show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-</section>
-
-<section>
-<h2 id="the-breast-cancer-data-now-with-keras">The Breast Cancer Data, now with Keras </h2>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Sequential      <span style="color: #228B22">#This allows appending layers to existing models</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense           <span style="color: #228B22">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> optimizers             <span style="color: #228B22">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers           <span style="color: #228B22">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> to_categorical   <span style="color: #228B22">#This allows using categorical cross entropy as the cost function</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split <span style="color: #8B008B; font-weight: bold">as</span> splitter
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span> 
-
-
-<span style="color: #CD5555">&quot;&quot;&quot;Load breast cancer dataset&quot;&quot;&quot;</span>
-
-np.random.seed(<span style="color: #B452CD">0</span>)        <span style="color: #228B22">#create same seed for random number every time</span>
-
-cancer=load_breast_cancer()      <span style="color: #228B22">#Download breast cancer dataset</span>
-
-inputs=cancer.data                     <span style="color: #228B22">#Feature matrix of 569 rows (samples) and 30 columns (parameters)</span>
-outputs=cancer.target                  <span style="color: #228B22">#Label array of 569 rows (0 for benign and 1 for malignant)</span>
-labels=cancer.feature_names[<span style="color: #B452CD">0</span>:<span style="color: #B452CD">30</span>]
-
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;The content of the breast cancer dataset is:&#39;</span>)      <span style="color: #228B22">#Print information about the datasets</span>
-<span style="color: #658b00">print</span>(labels)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;-------------------------&#39;</span>)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;inputs =  &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;outputs =  &quot;</span> + <span style="color: #658b00">str</span>(outputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;labels =  &quot;</span>+ <span style="color: #658b00">str</span>(labels.shape))
-
-x=inputs      <span style="color: #228B22">#Reassign the Feature and Label matrices to other variables</span>
-y=outputs
-
-<span style="color: #228B22">#%% </span>
-
-<span style="color: #228B22"># Visualisation of dataset (for correlation analysis)</span>
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">0</span>],x[:,<span style="color: #B452CD">2</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean radius&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean perimeter&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">5</span>],x[:,<span style="color: #B452CD">6</span>],s=<span style="color: #B452CD">40</span>,c=y, cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean compactness&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean concavity&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">0</span>],x[:,<span style="color: #B452CD">1</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean radius&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean texture&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">2</span>],x[:,<span style="color: #B452CD">1</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean perimeter&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean compactness&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-
-<span style="color: #228B22"># Generate training and testing datasets</span>
-
-<span style="color: #228B22">#Select features relevant to classification (texture,perimeter,compactness and symmetery) </span>
-<span style="color: #228B22">#and add to input matrix</span>
-
-temp1=np.reshape(x[:,<span style="color: #B452CD">1</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">1</span>]),<span style="color: #B452CD">1</span>))
-temp2=np.reshape(x[:,<span style="color: #B452CD">2</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">2</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((temp1,temp2))      
-temp=np.reshape(x[:,<span style="color: #B452CD">5</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">5</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((X,temp))       
-temp=np.reshape(x[:,<span style="color: #B452CD">8</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">8</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((X,temp))       
-
-X_train,X_test,y_train,y_test=splitter(X,y,test_size=<span style="color: #B452CD">0.1</span>)   <span style="color: #228B22">#Split datasets into training and testing</span>
-
-y_train=to_categorical(y_train)     <span style="color: #228B22">#Convert labels to categorical when using categorical cross entropy</span>
-y_test=to_categorical(y_test)
-
-<span style="color: #8B008B; font-weight: bold">del</span> temp1,temp2,temp
-
-<span style="color: #228B22"># %%</span>
-
-<span style="color: #228B22"># Define tunable parameters&quot;</span>
-
-eta=np.logspace(-<span style="color: #B452CD">3</span>,-<span style="color: #B452CD">1</span>,<span style="color: #B452CD">3</span>)                    <span style="color: #228B22">#Define vector of learning rates (parameter to SGD optimiser)</span>
-lamda=<span style="color: #B452CD">0.01</span>                                  <span style="color: #228B22">#Define hyperparameter</span>
-n_layers=<span style="color: #B452CD">2</span>                                  <span style="color: #228B22">#Define number of hidden layers in the model</span>
-n_neuron=np.logspace(<span style="color: #B452CD">0</span>,<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>,dtype=<span style="color: #658b00">int</span>)       <span style="color: #228B22">#Define number of neurons per layer</span>
-epochs=<span style="color: #B452CD">100</span>                                   <span style="color: #228B22">#Number of reiterations over the input data</span>
-batch_size=<span style="color: #B452CD">100</span>                              <span style="color: #228B22">#Number of samples per gradient update</span>
-
-<span style="color: #228B22"># %%</span>
-
-<span style="color: #CD5555">&quot;&quot;&quot;Define function to return Deep Neural Network model&quot;&quot;&quot;</span>
-
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">NN_model</span>(inputsize,n_layers,n_neuron,eta,lamda):
-    model=Sequential()      
-    <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_layers):       <span style="color: #228B22">#Run loop to add hidden layers to the model</span>
-        <span style="color: #8B008B; font-weight: bold">if</span> (i==<span style="color: #B452CD">0</span>):                  <span style="color: #228B22">#First layer requires input dimensions</span>
-            model.add(Dense(n_neuron,activation=<span style="color: #CD5555">&#39;relu&#39;</span>,kernel_regularizer=regularizers.l2(lamda),input_dim=inputsize))
-        <span style="color: #8B008B; font-weight: bold">else</span>:                       <span style="color: #228B22">#Subsequent layers are capable of automatic shape inferencing</span>
-            model.add(Dense(n_neuron,activation=<span style="color: #CD5555">&#39;relu&#39;</span>,kernel_regularizer=regularizers.l2(lamda)))
-    model.add(Dense(<span style="color: #B452CD">2</span>,activation=<span style="color: #CD5555">&#39;softmax&#39;</span>))  <span style="color: #228B22">#2 outputs - ordered and disordered (softmax for prob)</span>
-    sgd=optimizers.SGD(lr=eta)
-    model.compile(loss=<span style="color: #CD5555">&#39;categorical_crossentropy&#39;</span>,optimizer=sgd,metrics=[<span style="color: #CD5555">&#39;accuracy&#39;</span>])
-    <span style="color: #8B008B; font-weight: bold">return</span> model
-
-    
-Train_accuracy=np.zeros((<span style="color: #658b00">len</span>(n_neuron),<span style="color: #658b00">len</span>(eta)))      <span style="color: #228B22">#Define matrices to store accuracy scores as a function</span>
-Test_accuracy=np.zeros((<span style="color: #658b00">len</span>(n_neuron),<span style="color: #658b00">len</span>(eta)))       <span style="color: #228B22">#of learning rate and number of hidden neurons for </span>
-
-<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(n_neuron)):     <span style="color: #228B22">#run loops over hidden neurons and learning rates to calculate </span>
-    <span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(eta)):      <span style="color: #228B22">#accuracy scores </span>
-        DNN_model=NN_model(X_train.shape[<span style="color: #B452CD">1</span>],n_layers,n_neuron[i],eta[j],lamda)
-        DNN_model.fit(X_train,y_train,epochs=epochs,batch_size=batch_size,verbose=<span style="color: #B452CD">1</span>)
-        Train_accuracy[i,j]=DNN_model.evaluate(X_train,y_train)[<span style="color: #B452CD">1</span>]
-        Test_accuracy[i,j]=DNN_model.evaluate(X_test,y_test)[<span style="color: #B452CD">1</span>]
-               
-
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_data</span>(x,y,data,title=<span style="color: #8B008B; font-weight: bold">None</span>):
-
-    <span style="color: #228B22"># plot results</span>
-    fontsize=<span style="color: #B452CD">16</span>
-
-
-    fig = plt.figure()
-    ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
-    cax = ax.matshow(data, interpolation=<span style="color: #CD5555">&#39;nearest&#39;</span>, vmin=<span style="color: #B452CD">0</span>, vmax=<span style="color: #B452CD">1</span>)
-    
-    cbar=fig.colorbar(cax)
-    cbar.ax.set_ylabel(<span style="color: #CD5555">&#39;accuracy (%)&#39;</span>,rotation=<span style="color: #B452CD">90</span>,fontsize=fontsize)
-    cbar.set_ticks([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">.2</span>,<span style="color: #B452CD">.4</span>,<span style="color: #B452CD">0.6</span>,<span style="color: #B452CD">0.8</span>,<span style="color: #B452CD">1.0</span>])
-    cbar.set_ticklabels([<span style="color: #CD5555">&#39;0%&#39;</span>,<span style="color: #CD5555">&#39;20%&#39;</span>,<span style="color: #CD5555">&#39;40%&#39;</span>,<span style="color: #CD5555">&#39;60%&#39;</span>,<span style="color: #CD5555">&#39;80%&#39;</span>,<span style="color: #CD5555">&#39;100%&#39;</span>])
-
-    <span style="color: #228B22"># put text on matrix elements</span>
-    <span style="color: #8B008B; font-weight: bold">for</span> i, x_val <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(np.arange(<span style="color: #658b00">len</span>(x))):
-        <span style="color: #8B008B; font-weight: bold">for</span> j, y_val <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(np.arange(<span style="color: #658b00">len</span>(y))):
-            c = <span style="color: #CD5555">&quot;${0:.1f}\\%$&quot;</span>.format( <span style="color: #B452CD">100</span>*data[j,i])  
-            ax.text(x_val, y_val, c, va=<span style="color: #CD5555">&#39;center&#39;</span>, ha=<span style="color: #CD5555">&#39;center&#39;</span>)
-
-    <span style="color: #228B22"># convert axis vaues to to string labels</span>
-    x=[<span style="color: #658b00">str</span>(i) <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> x]
-    y=[<span style="color: #658b00">str</span>(i) <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> y]
-
-
-    ax.set_xticklabels([<span style="color: #CD5555">&#39;&#39;</span>]+x)
-    ax.set_yticklabels([<span style="color: #CD5555">&#39;&#39;</span>]+y)
-
-    ax.set_xlabel(<span style="color: #CD5555">&#39;$\\mathrm{learning\\ rate}$&#39;</span>,fontsize=fontsize)
-    ax.set_ylabel(<span style="color: #CD5555">&#39;$\\mathrm{hidden\\ neurons}$&#39;</span>,fontsize=fontsize)
-    <span style="color: #8B008B; font-weight: bold">if</span> title <span style="color: #8B008B">is</span> <span style="color: #8B008B">not</span> <span style="color: #8B008B; font-weight: bold">None</span>:
-        ax.set_title(title)
-
-    plt.tight_layout()
-
-    plt.show()
-    
-plot_data(eta,n_neuron,Train_accuracy, <span style="color: #CD5555">&#39;training&#39;</span>)
-plot_data(eta,n_neuron,Test_accuracy, <span style="color: #CD5555">&#39;testing&#39;</span>)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-</section>
-
-<section>
-<h2 id="fine-tuning-neural-network-hyperparameters">Fine-tuning neural network hyperparameters </h2>
-
-<p>The flexibility of neural networks is also one of their main
-drawbacks: there are many hyperparameters to tweak. Not only can you
-use any imaginable network topology (how neurons/nodes are interconnected),
-but even in a simple FFNN you can change the number of layers, the
-number of neurons per layer, the type of activation function to use in
-each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
-know what combination of hyperparameters is the best for your task?
-</p>
-
-<ul>
-<p><li> You can use grid search with cross-validation to find the right hyperparameters.</li>
-</ul>
-<p>
-<p>However,since there are many hyperparameters to tune, and since
-training a neural network on a large dataset takes a lot of time, you
-will only be able to explore a tiny part of the hyperparameter space.
-</p>
-
-<ul>
-<p><li> You can use randomized search.</li>
-<p><li> Or use tools like <a href="http://oscar.calldesk.ai/" target="_blank">Oscar</a>, which implements more complex algorithms to help you find a good set of hyperparameters quickly.</li>  
-</ul>
-</section>
-
-<section>
-<h2 id="hidden-layers">Hidden layers </h2>
-
-<p>For many problems you can start with just one or two hidden layers and it will work just fine.
-For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
-few hundred neurons.
-You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
-neurons, in roughly the same amount of training time. 
-</p>
-
-<p>For more complex problems, you can gradually
-ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
-as large image classification or speech recognition, typically require networks with dozens of layers
-and they need a huge amount
-of training data. However, you will rarely have to train such networks from scratch: it is much more
-common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
-</p>
-</section>
-
-<section>
-<h2 id="which-activation-function-should-i-use">Which activation function should I use? </h2>
-
-<p>The Back propagation algorithm we derived above works by going from
-the output layer to the input layer, propagating the error gradient on
-the way. Once the algorithm has computed the gradient of the cost
-function with regards to each parameter in the network, it uses these
-gradients to update each parameter with a Gradient Descent (GD) step.
-</p>
-
-<p>Unfortunately for us, the gradients often get smaller and smaller as the
-algorithm progresses down to the first hidden layers. As a result, the
-GD update leaves the lower layer connection weights
-virtually unchanged, and training never converges to a good
-solution. This is known in the literature as 
-<b>the vanishing gradients problem</b>. 
-</p>
-
-<p>In other cases, the opposite can happen, namely the the gradients can grow bigger and
-bigger. The result is that many of the layers get large updates of the 
-weights the
-algorithm diverges. This is the <b>exploding gradients problem</b>, which is
-mostly encountered in recurrent neural networks. More generally, deep
-neural networks suffer from unstable gradients, different layers may
-learn at widely different speeds
-</p>
-</section>
-
-<section>
-<h2 id="is-the-logistic-activation-function-sigmoid-our-choice">Is the Logistic activation function (Sigmoid)  our choice? </h2>
-
-<p>Although this unfortunate behavior has been empirically observed for
-quite a while (it was one of the reasons why deep neural networks were
-mostly abandoned for a long time), it is only around 2010 that
-significant progress was made in understanding it.
-</p>
-
-<p>A paper titled <a href="http://proceedings.mlr.press/v9/glorot10a.html" target="_blank">Understanding the Difficulty of Training Deep
-Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio</a> found that
-the problems with the popular logistic
-sigmoid activation function and the weight initialization technique
-that was most popular at the time, namely random initialization using
-a normal distribution with a mean of 0 and a standard deviation of
-1. 
-</p>
-
-<p>They showed that with this activation function and this
-initialization scheme, the variance of the outputs of each layer is
-much greater than the variance of its inputs. Going forward in the
-network, the variance keeps increasing after each layer until the
-activation function saturates at the top layers. This is actually made
-worse by the fact that the logistic function has a mean of 0.5, not 0
-(the hyperbolic tangent function has a mean of 0 and behaves slightly
-better than the logistic function in deep networks).
-</p>
-</section>
-
-<section>
-<h2 id="the-derivative-of-the-logistic-funtion">The derivative of the Logistic funtion </h2>
-
-<p>Looking at the logistic activation function, when inputs become large
-(negative or positive), the function saturates at 0 or 1, with a
-derivative extremely close to 0. Thus when backpropagation kicks in,
-it has virtually no gradient to propagate back through the network,
-and what little gradient exists keeps getting diluted as
-backpropagation progresses down through the top layers, so there is
-really nothing left for the lower layers.
-</p>
-
-<p>In their paper, Glorot and Bengio propose a way to significantly
-alleviate this problem. We need the signal to flow properly in both
-directions: in the forward direction when making predictions, and in
-the reverse direction when backpropagating gradients. We don&#8217;t want
-the signal to die out, nor do we want it to explode and saturate. For
-the signal to flow properly, the authors argue that we need the
-variance of the outputs of each layer to be equal to the variance of
-its inputs, and we also need the gradients to have equal variance
-before and after flowing through a layer in the reverse direction.
-</p>
-
-<p>One of the insights in the 2010 paper by Glorot and Bengio was that
-the vanishing/exploding gradients problems were in part due to a poor
-choice of activation function. Until then most people had assumed that
-if Nature had chosen to use roughly sigmoid activation functions in
-biological neurons, they must be an excellent choice. But it turns out
-that other activation functions behave much better in deep neural
-networks, in particular the ReLU activation function, mostly because
-it does not saturate for positive values (and also because it is quite
-fast to compute).
-</p>
-</section>
-
-<section>
-<h2 id="the-relu-function-family">The RELU function family </h2>
-
-<p>The ReLU activation function suffers from a problem known as the dying
-ReLUs: during training, some neurons effectively die, meaning they
-stop outputting anything other than 0.
-</p>
-
-<p>In some cases, you may find that half of your network&#8217;s neurons are
-dead, especially if you used a large learning rate. During training,
-if a neuron&#8217;s weights get updated such that the weighted sum of the
-neuron&#8217;s inputs is negative, it will start outputting 0. When this
-happen, the neuron is unlikely to come back to life since the gradient
-of the ReLU function is 0 when its input is negative.
-</p>
-
-<p>To solve this problem, nowadays practitioners use a  variant of the ReLU
-function, such as the leaky ReLU discussed above or the so-called
-exponential linear unit (ELU) function
-</p>
-
-<p>&nbsp;<br>
-$$
-ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\  z & z \ge 0.\end{array}\right. 
-$$
-<p>&nbsp;<br>
-</section>
-
-<section>
-<h2 id="which-activation-function-should-we-use">Which activation function should we use? </h2>
-
-<p>In general it seems that the ELU activation function is better than
-the leaky ReLU function (and its variants), which is better than
-ReLU. ReLU performs better than \( \tanh \) which in turn performs better
-than the logistic function. 
-</p>
-
-<p>If runtime
-performance is an issue, then you may opt for the  leaky ReLU function  over the 
-ELU function If you don&#8217;t
-want to tweak yet another hyperparameter, you may just use the default
-\( \alpha \) of \( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have
-spare time and computing power, you can use cross-validation or
-bootstrap to evaluate other activation functions.
-</p>
-</section>
-
-<section>
-<h2 id="more-on-activation-functions-output-layers">More on activation functions, output layers </h2>
-
-<p>In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).</p>
-
-<p>It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.</p>
-
-<b>For the output layer:</b>
-
-<ul>
-<p><li> For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).</li>
-<p><li> For regression tasks, you can simply use no activation function at all.</li>
-</ul>
-</section>
-
-<section>
-<h2 id="batch-normalization">Batch Normalization </h2>
-
-<p>Batch Normalization
-aims to address the vanishing/exploding gradients problems, and more generally the problem that the
-distribution of each layer&#8217;s inputs changes during training, as the parameters of the previous layers change.
-</p>
-
-<p>The technique consists of adding an operation in the model just before the activation function of each
-layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
-parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
-learn the optimal scale and mean of the inputs for each layer.
-In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs&#8217; mean and
-standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
-mini-batch, from this the name batch normalization.
-</p>
-</section>
-
-<section>
-<h2 id="dropout">Dropout </h2>
-
-<p>It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
-excluding the output neurons) has a probability \( p \) of being temporarily dropped out, meaning it will be
-entirely ignored during this training step, but it may be active during the next step.
-</p>
-
-<p>The
-hyperparameter \( p \) is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
- It is viewed as one of the most popular regularization techniques.
-</p>
-</section>
-
-<section>
-<h2 id="gradient-clipping">Gradient Clipping </h2>
-
-<p>A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
-backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
-networks).
-</p>
-
-<p>This technique is called Gradient Clipping.</p>
-
-<p>In general however, Batch
-Normalization is preferred.
-</p>
-</section>
-
-<section>
-<h2 id="a-very-nice-website-on-neural-networks">A very nice website on Neural Networks </h2>
-
-<p>You may find this <a href="https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false" target="_blank">website</a> very useful. </p>
-</section>
-
-<section>
-<h2 id="a-top-down-perspective-on-neural-networks">A top-down perspective on Neural networks </h2>
-
-<p>The first thing we would like to do is divide the data into two or three
-parts. A training set, a validation or dev (development) set, and a
-test set. The test set is the data on which we want to make
-predictions. The dev set is a subset of the training data we use to
-check how well we are doing out-of-sample, after training the model on
-the training dataset. We use the validation error as a proxy for the
-test error in order to make tweaks to our model. It is crucial that we
-do not use any of the test data to train the algorithm. This is a
-cardinal sin in ML. Then:
-</p>
-
-<ul>
-<p><li> Estimate optimal error rate</li>
-<p><li> Minimize underfitting (bias) on training data set.</li>
-<p><li> Make sure you are not overfitting.</li>
-</ul>
-<p>
-<p>If the validation and test sets are drawn from the same distributions,
-then a good performance on the validation set should lead to similarly
-good performance on the test set. 
-</p>
-
-<p>However, sometimes
-the training data and test data differ in subtle ways because, for
-example, they are collected using slightly different methods, or
-because it is cheaper to collect data in one way versus another. In
-this case, there can be a mismatch between the training and test
-data. This can lead to the neural network overfitting these small
-differences between the test and training sets, and a poor performance
-on the test set despite having a good performance on the validation
-set. To rectify this, Andrew Ng suggests making two validation or dev
-sets, one constructed from the training data and one constructed from
-the test data. The difference between the performance of the algorithm
-on these two validation sets quantifies the train-test mismatch. This
-can serve as another important diagnostic when using DNNs for
-supervised learning.
-</p>
-</section>
-
-<section>
-<h2 id="limitations-of-supervised-learning-with-deep-networks">Limitations of supervised learning with deep networks </h2>
-
-<p>Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
-</p>
-
-<p>Here we list some of the important limitations of supervised neural network based models. </p>
-
-<ul>
-<p><li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
-<p><li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
-<p><li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.&nbsp;some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
-<p><li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
-</ul>
-<p>
-<p>Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.</p>
-</section>
-
 
 
 </div> <!-- class="slides" -->
diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html
index 9d79fd083..72813e333 100644
--- a/doc/pub/week41/html/week41-solarized.html
+++ b/doc/pub/week41/html/week41-solarized.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --pygments_html_style=perldoc --html_style=sol
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <link href="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
 <script src="https://cdn.rawgit.com/doconce/doconce/master/bundled/html_styles/style_solarized_box/js/highlight.pack.js"></script>
 <script>hljs.initHighlightingOnLoad();</script>
@@ -64,14 +64,107 @@ div.toc p,a {
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -149,62 +242,7 @@ div.toc p,a {
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -226,7 +264,7 @@ MathJax.Hub.Config({
 
 <!-- ------------------- main content ---------------------- -->
 <center>
-<h1>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1>
+<h1>Week 41 Neural networks and constructing a neural network code</h1>
 </center>  <!-- document title -->
 
 <!-- author(s): Morten Hjorth-Jensen -->
@@ -249,37 +287,1199 @@ MathJax.Hub.Config({
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
 <h2 id="plan-for-week-41">Plan for week 41 </h2>
 
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the active learning sessions on Tuesday and Wednesday</b>
+<p>
 <ul>
-<li> Building our own Feed-forward Neural Network and discussion of project 2.</li>
-<li> Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks.</li> 
+  <li> Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation</li>
+  <li> One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code</li>
+  <li> Discussion of project 2</li>
+  <li> Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent</li>
 </ul>
-<p>"
-Reading suggestions: These notes, <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 10-11</a>.
-For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read.
+</div>
+  
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the lecture on Thursday October 12, 2023</b>
+<p>
+<ul>
+  <li> Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.</li>
+  <li> Building our own Feed-forward Neural Network</li>
+  <li> Readings and Videos:</li>
+<ul>
+    <li> These lecture notes</li>
+    <li> For neural networks we recommend Goodfellow et al chapter 6.</li>
+    <li> <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_blank">Neural Networks demystified</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex" target="_blank">Building Neural Networks from scratch</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
+</ul>
+</ul>
+<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="lecture-thursday-october-12">Lecture Thursday October 12 </h2>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="introduction-to-neural-networks">Introduction to Neural networks </h2>
+
+<p>Artificial neural networks are computational systems that can learn to
+perform tasks by considering examples, generally without being
+programmed with any task-specific rules. It is supposed to mimic a
+biological system, wherein neurons interact by sending signals in the
+form of mathematical functions between layers. All layers can contain
+an arbitrary number of neurons, and each connection is represented by
+a weight variable.
 </p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="videos-on-neural-networks">Videos on Neural Networks </h2>
+<h2 id="artificial-neurons">Artificial neurons  </h2>
 
-<ul>
-<li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
-<li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
-</ul>
-<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+<p>The field of artificial neural networks has a long history of
+development, and is closely connected with the advancement of computer
+science and computers in general. A model of artificial neurons was
+first developed by McCulloch and Pitts in 1943 to study signal
+processing in the brain and has later been refined by others. The
+general idea is to mimic neural networks in the human brain, which is
+composed of billions of neurons that communicate with each other by
+sending electrical signals.  Each neuron accumulates its incoming
+signals, which must exceed an activation threshold to yield an
+output. If the threshold is not overcome, the neuron remains inactive,
+i.e. has zero output.
+</p>
+
+<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
+
+$$
+\begin{equation}
+ y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
+\label{artificialNeuron}
+\end{equation}
+$$
+
+<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
+a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
+</p>
+
+<p>Conceptually, it is helpful to divide neural networks into four
+categories:
+</p>
+<ol>
+<li> general purpose neural networks for supervised learning,</li>
+<li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
+<li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
+<li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
+</ol>
+<p>In natural science, DNNs and CNNs have already found numerous
+applications. In statistical physics, they have been applied to detect
+phase transitions in 2D Ising and Potts models, lattice gauge
+theories, and different phases of polymers, or solving the
+Navier-Stokes equation in weather forecasting.  Deep learning has also
+found interesting applications in quantum physics. Various quantum
+phase transitions can be detected and studied using DNNs and CNNs,
+topological phases, and even non-equilibrium many-body
+localization. Representing quantum states as DNNs quantum state
+tomography are among some of the impressive achievements to reveal the
+potential of DNNs to facilitate the study of quantum systems.
+</p>
+
+<p>In quantum information theory, it has been shown that one can perform
+gate decompositions with the help of neural. 
+</p>
+
+<p>The applications are not limited to the natural sciences. There is a
+plethora of applications in essentially all disciplines, from the
+humanities to life science and medicine.
+</p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="review-of-the-back-propagation-algorithm">Review of the back propagation algorithm </h2>
+<h2 id="neural-network-types">Neural network types </h2>
 
-<p>During the last lecture we discussed in detail the back propagation
-algorithm. This algorithm is based on a repeated application of the
-chain rule. Let us bring back the basic equation and at the same time
-link this with the basic mathematics of automatic differentiation.
+<p>An artificial neural network (ANN), is a computational model that
+consists of layers of connected neurons, or nodes or units.  We will
+refer to these interchangeably as units or nodes, and sometimes as
+neurons.
+</p>
+
+<p>It is supposed to mimic a biological nervous system by letting each
+neuron interact with other neurons by sending signals in the form of
+mathematical functions between layers.  A wide variety of different
+ANNs have been developed, but most of them consist of an input layer,
+an output layer and eventual layers in-between, called <em>hidden
+layers</em>. All layers can contain an arbitrary number of nodes, and each
+connection between two nodes is associated with a weight variable.
+</p>
+
+<p>Neural networks (also called neural nets) are neural-inspired
+nonlinear models for supervised learning.  As we will see, neural nets
+can be viewed as natural, more powerful extensions of supervised
+learning methods such as linear and logistic regression and soft-max
+methods we discussed earlier.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="feed-forward-neural-networks">Feed-forward neural networks </h2>
+
+<p>The feed-forward neural network (FFNN) was the first and simplest type
+of ANNs that were devised. In this network, the information moves in
+only one direction: forward through the layers.
+</p>
+
+<p>Nodes are represented by circles, while the arrows display the
+connections between the nodes, including the direction of information
+flow. Additionally, each arrow corresponds to a weight variable
+(figure to come).  We observe that each node in a layer is connected
+to <em>all</em> nodes in the subsequent layer, making this a so-called
+<em>fully-connected</em> FFNN.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="convolutional-neural-network">Convolutional Neural Network </h2>
+
+<p>A different variant of FFNNs are <em>convolutional neural networks</em>
+(CNNs), which have a connectivity pattern inspired by the animal
+visual cortex. Individual neurons in the visual cortex only respond to
+stimuli from small sub-regions of the visual field, called a receptive
+field. This makes the neurons well-suited to exploit the strong
+spatially local correlation present in natural images. The response of
+each neuron can be approximated mathematically as a convolution
+operation.  (figure to come)
+</p>
+
+<p>Convolutional neural networks emulate the behaviour of neurons in the
+visual cortex by enforcing a <em>local</em> connectivity pattern between
+nodes of adjacent layers: Each node in a convolutional layer is
+connected only to a subset of the nodes in the previous layer, in
+contrast to the fully-connected FFNN.  Often, CNNs consist of several
+convolutional layers that learn local features of the input, with a
+fully-connected layer at the end, which gathers all the local data and
+produces the outputs. They have wide applications in image and video
+recognition.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="recurrent-neural-networks">Recurrent neural networks </h2>
+
+<p>So far we have only mentioned ANNs where information flows in one
+direction: forward. <em>Recurrent neural networks</em> on the other hand,
+have connections between nodes that form directed <em>cycles</em>. This
+creates a form of internal memory which are able to capture
+information on what has been calculated before; the output is
+dependent on the previous computations. Recurrent NNs make use of
+sequential information by performing the same task for every element
+in a sequence, where each element depends on previous elements. An
+example of such information is sentences, making recurrent NNs
+especially well-suited for handwriting and speech recognition.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="other-types-of-networks">Other types of networks </h2>
+
+<p>There are many other kinds of ANNs that have been developed. One type
+that is specifically designed for interpolation in multidimensional
+space is the radial basis function (RBF) network. RBFs are typically
+made up of three layers: an input layer, a hidden layer with
+non-linear radial symmetric activation functions and a linear output
+layer (''linear'' here means that each node in the output layer has a
+linear activation function). The layers are normally fully-connected
+and there are no cycles, thus RBFs can be viewed as a type of
+fully-connected FFNN. They are however usually treated as a separate
+type of NN due the unusual activation functions.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="multilayer-perceptrons">Multilayer perceptrons  </h2>
+
+<p>One uses often so-called fully-connected feed-forward neural networks
+with three or more layers (an input layer, one or more hidden layers
+and an output layer) consisting of neurons that have non-linear
+activation functions.
+</p>
+
+<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="why-multilayer-perceptrons">Why multilayer perceptrons?  </h2>
+
+<p>According to the <em>Universal approximation theorem</em>, a feed-forward
+neural network with just a single hidden layer containing a finite
+number of neurons can approximate a continuous multidimensional
+function to arbitrary accuracy, assuming the activation function for
+the hidden layer is a <b>non-constant, bounded and
+monotonically-increasing continuous function</b>.
+</p>
+
+<p>Note that the requirements on the activation function only applies to
+the hidden layer, the output nodes are always assumed to be linear, so
+as to not restrict the range of output values.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model">Illustration of a single perceptron model and a multi-perceptron model </h2>
+
+<center>  <!-- FIGURE -->
+<hr class="figure">
+<center>
+<p class="caption">Figure 1:  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer. </p>
+</center>
+<p><img src="figures/nns.png" width="600" align="bottom"></p>
+</center>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="examples-of-xor-or-and-and-gates">Examples of XOR, OR and AND gates </h2>
+
+<p>Let us first try to fit various gates using standard linear
+regression. The gates we are thinking of are the classical XOR, OR and
+AND gates, well-known elements in computer science. The tables here
+show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
+specific target \( y_i \).
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
+<span style="color: #CD5555">Simple code that tests XOR, OR and AND gates with linear regression</span>
+<span style="color: #CD5555">&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
+<span style="color: #228B22"># Design matrix</span>
+X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX  matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
+Xinv = np.linalg.pinv(X.T @ X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX  matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># The XOR gate </span>
+yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
+ThetaXOR  = Xinv @ X.T @ yXOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
+ThetaOR  = Xinv @ X.T @ yOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
+ThetaAND  = Xinv @ X.T @ yAND
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>What is happening here?</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="does-logistic-regression-do-a-better-job">Does Logistic Regression do a better Job? </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
+<span style="color: #CD5555">Simple code that tests XOR and OR gates with linear regression</span>
+<span style="color: #CD5555">and logistic regression</span>
+<span style="color: #CD5555">&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
+
+<span style="color: #228B22"># Design matrix</span>
+X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX  matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
+Xinv = np.linalg.pinv(X.T @ X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX  matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># The XOR gate </span>
+yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
+ThetaXOR  = Xinv @ X.T @ yXOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
+ThetaOR  = Xinv @ X.T @ yOR
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
+
+
+<span style="color: #228B22"># The OR gate </span>
+yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
+ThetaAND  = Xinv @ X.T @ yAND
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction  for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
+
+<span style="color: #228B22"># Now we change to logistic regression</span>
+
+
+<span style="color: #228B22"># Logistic Regression</span>
+logreg = LogisticRegression()
+logreg.fit(X, yOR)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for OR gate: {:.2f}&quot;</span>.format(logreg.score(X,yOR)))
+
+logreg.fit(X, yXOR)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for XOR gate: {:.2f}&quot;</span>.format(logreg.score(X,yXOR)))
+
+
+logreg.fit(X, yAND)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for AND gate: {:.2f}&quot;</span>.format(logreg.score(X,yAND)))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>Not exactly impressive, but somewhat better.</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="adding-neural-networks">Adding Neural Networks </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="line-height: 125%;"><span style="color: #228B22"># and now neural networks with Scikit-Learn and the XOR</span>
+
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.neural_network</span> <span style="color: #8B008B; font-weight: bold">import</span> MLPClassifier
+<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_classification
+X, yXOR = make_classification(n_samples=<span style="color: #B452CD">100</span>, random_state=<span style="color: #B452CD">1</span>)
+FFNN = MLPClassifier(random_state=<span style="color: #B452CD">1</span>, max_iter=<span style="color: #B452CD">300</span>).fit(X, yXOR)
+FFNN.predict_proba(X)
+<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Test set accuracy with Feed Forward Neural Network  for XOR gate:{</span>FFNN.score(X, yXOR)<span style="color: #CD5555">}&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output \( y \) is produced via the activation function \( f \)</p>
+$$
+ y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
+$$
+
+<p>This function receives \( x_i \) as inputs.
+Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \). 
+In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
+the neurons in the preceding layer. Furthermore, an MLP is
+fully-connected, which means that each neuron receives a weighted sum
+of the outputs of <em>all</em> neurons in the previous layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
+
+$$
+\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
+\label{_auto1}
+\end{equation}
+$$
+
+<p>Here \( b_i \) is the so-called bias which is normally needed in
+case of zero activation weights or inputs. How to fix the biases and
+the weights will be discussed below.  The value of \( z_i^1 \) is the
+argument to the activation function \( f_i \) of each node \( i \), The
+variable \( M \) stands for all possible inputs to a given node \( i \) in the
+first layer.  We define  the output \( y_i^1 \) of all neurons in layer 1 as
+</p>
+
+$$
+\begin{equation}
+ y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
+\label{outputLayer1}
+\end{equation}
+$$
+
+<p>where we assume that all nodes in the same layer have identical
+activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
+In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
+</p>
+
+$$
+\begin{equation}
+ y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
+\label{generalLayer}
+\end{equation}
+$$
+
+<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
+all the nodes in the first hidden layer are computed, the values of
+the subsequent layer can be calculated and so forth until the output
+is obtained.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output of neuron \( i \) in layer 2 is thus,</p>
+
+$$
+\begin{align}
+ y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) 
+\label{_auto2}\\
+ &= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
+\label{outputLayer2}
+\end{align}
+$$
+
+<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
+
+$$
+\begin{align}
+ y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) 
+\label{_auto3}\\
+ &= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+  + b_1^3\right]
+\label{_auto4}
+\end{align}
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>We can generalize this expression to an MLP with \( l \) hidden
+layers. The complete functional form is,
+</p>
+
+$$
+\begin{align}
+&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
+\label{completeNN}
+\end{align}
+$$
+
+<p>which illustrates a basic property of MLPs: The only independent
+variables are the input values \( x_n \).
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>This confirms that an MLP, despite its quite convoluted mathematical
+form, is nothing more than an analytic function, specifically a
+mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
+\hat{y} \in \mathbb{R}^m \).
+</p>
+
+<p>Furthermore, the flexibility and universality of an MLP can be
+illustrated by realizing that the expression is essentially a nested
+sum of scaled activation functions of the form
+</p>
+
+$$
+\begin{equation}
+ f(x) = c_1 f(c_2 x + c_3) + c_4
+\label{_auto5}
+\end{equation}
+$$
+
+<p>where the parameters \( c_i \) are weights and biases. By adjusting these
+parameters, the activation functions can be shifted up and down or
+left and right, change slope or be rescaled which is the key to the
+flexibility of a neural network.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="matrix-vector-notation">Matrix-vector notation </h3>
+
+<p>We can introduce a more convenient notation for the activations in an A NN. </p>
+
+<p>Additionally, we can represent the biases and activations
+as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector 
+is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively. 
+</p>
+
+<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors. 
+With this notation, the sum becomes a matrix-vector multiplication, and we can write
+the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
+</p>
+$$
+\begin{equation}
+ \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
+ f_2\left(\left[\begin{array}{ccc}
+    w^2_{11} &w^2_{12} &w^2_{13} \\
+    w^2_{21} &w^2_{22} &w^2_{23} \\
+    w^2_{31} &w^2_{32} &w^2_{33} \\
+    \end{array} \right] \cdot
+    \left[\begin{array}{c}
+           y^1_1 \\
+           y^1_2 \\
+           y^1_3 \\
+          \end{array}\right] + 
+    \left[\begin{array}{c}
+           b^2_1 \\
+           b^2_2 \\
+           b^2_3 \\
+          \end{array}\right]\right).
+\label{_auto6}
+\end{equation}
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="matrix-vector-notation-and-activation">Matrix-vector notation  and activation </h3>
+
+<p>The activation of node \( i \) in layer 2 is</p>
+
+$$
+\begin{equation}
+ y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
+ f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
+\label{_auto7}
+\end{equation}
+$$
+
+<p>This is not just a convenient and compact notation, but also a useful
+and intuitive way to think about MLPs: The output is calculated by a
+series of matrix-vector multiplications and vector additions that are
+used as input to the activation functions. For each operation
+\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="activation-functions">Activation functions  </h3>
+
+<p>A property that characterizes a neural network, other than its
+connectivity, is the choice of activation function(s).  As described
+in, the following restrictions are imposed on an activation function
+for a FFNN to fulfill the universal approximation theorem
+</p>
+
+<ul>
+  <li> Non-constant</li>
+  <li> Bounded</li>
+  <li> Monotonically-increasing</li>
+  <li> Continuous</li>
+</ul>
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="activation-functions-logistic-and-hyperbolic-ones">Activation functions, Logistic and Hyperbolic ones  </h3>
+
+<p>The second requirement excludes all linear functions. Furthermore, in
+a MLP with only linear activation functions, each layer simply
+performs a linear transformation of its inputs.
+</p>
+
+<p>Regardless of the number of layers, the output of the NN will be
+nothing but a linear function of the inputs. Thus we need to introduce
+some kind of non-linearity to the NN to be able to fit non-linear
+functions Typical examples are the logistic <em>Sigmoid</em>
+</p>
+
+$$
+ f(x) = \frac{1}{1 + e^{-x}},
+$$
+
+<p>and the <em>hyperbolic tangent</em> function</p>
+$$
+ f(x) = \tanh(x)
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="relevance">Relevance </h3>
+
+<p>The <em>sigmoid</em> function are more biologically plausible because the
+output of inactive neurons are zero. Such activation function are
+called <em>one-sided</em>. However, it has been shown that the hyperbolic
+tangent performs better than the sigmoid for training MLPs.  has
+become the most popular for <em>deep neural networks</em>
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #eeeedd">
+  <pre style="line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;The sigmoid function (or the logistic curve) is a </span>
+<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
+<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
+<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm.&quot;&quot;&quot;</span>
+
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
+<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
+
+z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.1</span>)
+sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
+sigma = sigma_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, sigma)
+ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
+ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;sigmoid function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Step Function&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.02</span>)
+step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z &gt;= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
+step = step_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, step)
+ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
+ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;step function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Sine Function&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
+t = numpy.sin(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, t)
+ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
+ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;sine function&#39;</span>)
+
+plt.show()
+
+<span style="color: #CD5555">&quot;&quot;&quot;Plots a graph of the squashing function used by a rectified linear</span>
+<span style="color: #CD5555">unit&quot;&quot;&quot;</span>
+z = numpy.arange(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>, <span style="color: #B452CD">.1</span>)
+zero = numpy.zeros(<span style="color: #658b00">len</span>(z))
+y = numpy.max([zero, z], axis=<span style="color: #B452CD">0</span>)
+
+fig = plt.figure()
+ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
+ax.plot(z, y)
+ax.set_ylim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
+ax.set_xlim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
+ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
+ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
+ax.set_title(<span style="color: #CD5555">&#39;Rectified linear unit&#39;</span>)
+
+plt.show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="the-multilayer-perceptron-mlp">The multilayer  perceptron (MLP) </h2>
+
+<p>The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of</p>
+<ol>
+<li> A neural network with one or more layers of nodes between the input and the output nodes.</li>
+<li> The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.</li>
+<li> The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.</li>
+</ol>
+<p>As a convention it is normal to call  a  network with one layer of input units, one layer of hidden
+units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
+</p>
+
+<p>For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.
+Hereafter we will call the various entities of a layer for nodes.
+There are also no connections within a single layer.
+</p>
+
+<p>The number of input nodes does not need to equal the number of output
+nodes. This applies also to the hidden layers. Each layer may have its
+own number of nodes and activation functions.
+</p>
+
+<p>The hidden layers have their name from the fact that they are not
+linked to observables and as we will see below when we define the
+so-called activation \( \hat{z} \), we can think of this as a basis
+expansion of the original inputs \( \hat{x} \). The difference however
+between neural networks and say linear regression is that now these
+basis functions (which will correspond to the weights in the network)
+are learned from data. This results in  an important difference between
+neural networks and deep learning approaches on one side and methods
+like logistic regression or linear regression and their modifications on the other side.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="from-one-to-many-layers-the-universal-approximation-theorem">From one to many layers, the universal approximation theorem </h2>
+
+<p>A neural network with only one layer, what we called the simple
+perceptron, is best suited if we have a standard binary model with
+clear (linear) boundaries between the outcomes. As such it could
+equally well be replaced by standard linear regression or logistic
+regression. Networks with one or more hidden layers approximate
+systems with more complex boundaries.
+</p>
+
+<p>As stated earlier, 
+an important theorem in studies of neural networks, restated without
+proof here, is the <a href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf" target="_blank">universal approximation
+theorem</a>.
+</p>
+
+<p>It states that a feed-forward network with a single hidden layer
+containing a finite number of neurons can approximate continuous
+functions on compact subsets of real functions. The theorem thus
+states that simple neural networks can represent a wide variety of
+interesting functions when given appropriate parameters. It is the
+multilayer feedforward architecture itself which gives neural networks
+the potential of being universal approximators.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="deriving-the-back-propagation-code-for-a-multilayer-perceptron-model">Deriving the back propagation code for a multilayer perceptron model </h2>
+
+<p>As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.
+The unknowwn quantities are our weights \( w_{ij} \) and we need to find an algorithm for changing them so that our errors are as small as possible.
+This leads us to the famous <a href="https://www.nature.com/articles/323533a0" target="_blank">back propagation algorithm</a>.
+</p>
+
+<p>The questions we want to ask are how do changes in the biases and the
+weights in our network change the cost function and how can we use the
+final output to modify the weights?
+</p>
+
+<p>To derive these equations let us start with a plain regression problem
+and define our cost function as
+</p>
+
+$$
+{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2, 
+$$
+
+<p>where the $t_i$s are our \( n \) targets (the values we want to
+reproduce), while the outputs of the network after having propagated
+all inputs \( \hat{x} \) are given by \( y_i \).  Below we will demonstrate
+how the basic equations arising from the back propagation algorithm
+can be modified in order to study classification problems with \( K \)
+classes.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="definitions">Definitions </h2>
+
+<p>With our definition of the targets \( \hat{t} \), the outputs of the
+network \( \hat{y} \) and the inputs \( \hat{x} \) we
+define now the activation \( z_j^l \) of node/neuron/unit \( j \) of the
+\( l \)-th layer as a function of the bias, the weights which add up from
+the previous layer \( l-1 \) and the forward passes/outputs
+\( \hat{a}^{l-1} \) from the previous layer as
+</p>
+
+$$
+z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,
+$$
+
+<p>where \( b_k^l \) are the biases from layer \( l \).  Here \( M_{l-1} \)
+represents the total number of nodes/neurons/units of layer \( l-1 \). The
+figure here illustrates this equation.  We can rewrite this in a more
+compact form as the matrix-vector products we discussed earlier,
+</p>
+
+$$
+\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
+$$
+
+<p>With the activation values \( \hat{z}^l \) we can in turn define the
+output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
+activation function. In the examples here we will use the sigmoid
+function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
+and their nodes.  It means we have
+</p>
+
+$$
+a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivatives-and-the-chain-rule">Derivatives and the chain rule </h2>
+
+<p>From the definition of the activation \( z_j^l \) we have</p>
+$$
+\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
+$$
+
+<p>and</p>
+$$
+\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. 
+$$
+
+<p>With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))</p>
+$$
+\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)). 
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivative-of-the-cost-function">Derivative of the cost function </h2>
+
+<p>With these definitions we can now compute the derivative of the cost function in terms of the weights.</p>
+
+<p>Let us specialize to the output layer \( l=L \). Our cost function is</p>
+$$
+{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2, 
+$$
+
+<p>The derivative of this function with respect to the weights is</p>
+
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}}, 
+$$
+
+<p>The last partial derivative can easily be computed and reads (by applying the chain rule)</p>
+$$
+\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},  
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="bringing-it-together-first-back-propagation-equation">Bringing it together, first back propagation equation </h2>
+
+<p>We have thus</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1}, 
+$$
+
+<p>Defining</p>
+$$
+\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+$$
+
+<p>and using the Hadamard product of two vectors we can write this as</p>
+$$
+\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
+$$
+
+<p>This is an important expression. The second term on the right handside
+measures how fast the cost function is changing as a function of the $j$th
+output activation.  If, for example, the cost function doesn't depend
+much on a particular output node \( j \), then \( \delta_j^L \) will be small,
+which is what we would expect. The first term on the right, measures
+how fast the activation function \( f \) is changing at a given activation
+value \( z_j^L \).
+</p>
+
+<p>Notice that everything in the above equations is easily computed.  In
+particular, we compute \( z_j^L \) while computing the behaviour of the
+network, and it is only a small additional overhead to compute
+\( f'(z^L_j) \).  The exact form of the derivative with respect to the
+output depends on the form of the cost function.
+However, provided the cost function is known there should be little
+trouble in calculating
+</p>
+
+$$
+\frac{\partial {\cal C}}{\partial (a_j^L)}
+$$
+
+<p>With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivatives-in-terms-of-z-j-l">Derivatives in terms of \( z_j^L \) </h2>
+
+<p>It is also easy to see that our previous equation can be written as</p>
+
+$$
+\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
+$$
+
+<p>which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely</p>
+$$
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
+$$
+
+<p>That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias. </p>
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="bringing-it-together">Bringing it together </h2>
+
+<p>We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>The starting equations</b>
+<p>
+
+$$
+\begin{equation}
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
+\label{_auto8}
+\end{equation}
+$$
+
+<p>and</p>
+$$
+\begin{equation}
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\label{_auto9}
+\end{equation}
+$$
+
+<p>and</p>
+
+$$
+\begin{equation}
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
+\label{_auto10}
+\end{equation}
+$$
+</div>
+
+
+<p>An interesting consequence of the above equations is that when the
+activation \( a_k^{L-1} \) is small, the gradient term, that is the
+derivative of the cost function with respect to the weights, will also
+tend to be small. We say then that the weight learns slowly, meaning
+that it changes slowly when we minimize the weights via say gradient
+descent. In this case we say the system learns slowly.
+</p>
+
+<p>Another interesting feature is that is when the activation function,
+represented by the sigmoid function here, is rather flat when we move towards
+its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
+cases, the derivatives of the activation function will also be close
+to zero, meaning again that the gradients will be small and the
+network learns slowly again.
+</p>
+
+<p>We need a fourth equation and we are set. We are going to propagate
+backwards in order to the determine the weights and biases. In order
+to do so we need to represent the error in the layer before the final
+one \( L-1 \) in terms of the errors in the final output layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="final-back-propagating-equation">Final back propagating equation </h2>
+
+<p>We have that (replacing \( L \) with a general layer \( l \))</p>
+$$
+\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
+$$
+
+<p>We want to express this in terms of the equations for layer \( l+1 \). Using the chain rule and summing over all \( k \) entries we have</p>
+
+$$
+\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
+$$
+
+<p>and recalling that</p>
+$$
+z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
+$$
+
+<p>with \( M_l \) being the number of nodes in layer \( l \), we obtain</p>
+$$
+\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
+$$
+
+<p>This is our final equation.</p>
+
+<p>We are now ready to set up the algorithm for back propagation and learning the weights and biases.</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \hat{x} \) and the activations
+\( \hat{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \hat{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \hat{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \hat{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \hat{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
 </p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
 <h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
 
-<p>The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+<p>The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
+\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \boldsymbol{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
 
 <div class="alert alert-block alert-block alert-text-normal">
 <b></b>
@@ -1817,876 +3017,6 @@ plt.show()
 </div>
 
 
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="building-neural-networks-in-tensorflow-and-keras">Building neural networks in Tensorflow and Keras </h2>
-
-<p>Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn
-and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
-and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  
-</p>
-
-<p>In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
-clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
-NumPy arrays.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="tensorflow">Tensorflow </h2>
-
-<p>Tensorflow is an open source library machine learning library
-developed by the Google Brain team for internal use. It was released
-under the Apache 2.0 open source license in November 9, 2015.
-</p>
-
-<p>Tensorflow is a computational framework that allows you to construct
-machine learning models at different levels of abstraction, from
-high-level, object-oriented APIs like Keras, down to the C++ kernels
-that Tensorflow is built upon. The higher levels of abstraction are
-simpler to use, but less flexible, and our choice of implementation
-should reflect the problems we are trying to solve.
-</p>
-
-<p><a href="https://www.tensorflow.org/guide/graphs" target="_blank">Tensorflow uses</a> so-called graphs to represent your computation
-in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
-to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
-</p>
-
-<p>In this guide we will analyze the same data as we did in our NumPy and
-scikit-learn tutorial, gathered from the MNIST database of images. We
-will give an introduction to the lower level Python Application
-Program Interfaces (APIs), and see how we use them to build our graph.
-Then we will build (effectively) the same graph in Keras, to see just
-how simple solving a machine learning problem can be.
-</p>
-
-<p>To install tensorflow on Unix/Linux systems, use pip as</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">pip3 install tensorflow
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
-(current release of CPU-only TensorFlow)
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">conda create -n tf tensorflow
-conda activate tf
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>To install the current release of GPU TensorFlow</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">conda create -n tf-gpu tensorflow-gpu
-conda activate tf-gpu
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="using-keras">Using Keras </h2>
-
-<p>Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
-that supports Tensorflow, CTNK and Theano as backends.  
-If you have Anaconda installed you may run the following command
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">conda install keras
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>You can look up the <a href="https://keras.io/" target="_blank">instructions here</a> for more information.</p>
-
-<p>We will to a large extent use <b>keras</b> in this course. </p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="collect-and-pre-process-data">Collect and pre-process data </h2>
-
-<p>Let us look again at the MINST data set.</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;"><span style="color: #228B22"># import necessary packages</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets
-
-
-<span style="color: #228B22"># ensure the same random numbers appear every time</span>
-np.random.seed(<span style="color: #B452CD">0</span>)
-
-<span style="color: #228B22"># display images in notebook</span>
-%matplotlib inline
-plt.rcParams[<span style="color: #CD5555">&#39;figure.figsize&#39;</span>] = (<span style="color: #B452CD">12</span>,<span style="color: #B452CD">12</span>)
-
-
-<span style="color: #228B22"># download MNIST dataset</span>
-digits = datasets.load_digits()
-
-<span style="color: #228B22"># define inputs and labels</span>
-inputs = digits.images
-labels = digits.target
-
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;labels = (n_inputs) = &quot;</span> + <span style="color: #658b00">str</span>(labels.shape))
-
-
-<span style="color: #228B22"># flatten the image</span>
-<span style="color: #228B22"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
-n_inputs = <span style="color: #658b00">len</span>(inputs)
-inputs = inputs.reshape(n_inputs, -<span style="color: #B452CD">1</span>)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;X = (n_inputs, n_features) = &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-
-
-<span style="color: #228B22"># choose some random images to display</span>
-indices = np.arange(n_inputs)
-random_indices = np.random.choice(indices, size=<span style="color: #B452CD">5</span>)
-
-<span style="color: #8B008B; font-weight: bold">for</span> i, image <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(digits.images[random_indices]):
-    plt.subplot(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">5</span>, i+<span style="color: #B452CD">1</span>)
-    plt.axis(<span style="color: #CD5555">&#39;off&#39;</span>)
-    plt.imshow(image, cmap=plt.cm.gray_r, interpolation=<span style="color: #CD5555">&#39;nearest&#39;</span>)
-    plt.title(<span style="color: #CD5555">&quot;Label: %d&quot;</span> % digits.target[random_indices[i]])
-plt.show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Sequential      <span style="color: #228B22">#This allows appending layers to existing models</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense           <span style="color: #228B22">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> optimizers             <span style="color: #228B22">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers           <span style="color: #228B22">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> to_categorical   <span style="color: #228B22">#This allows using categorical cross entropy as the cost function</span>
-
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
-
-<span style="color: #228B22"># one-hot representation of labels</span>
-labels = to_categorical(labels)
-
-<span style="color: #228B22"># split into train and test data</span>
-train_size = <span style="color: #B452CD">0.8</span>
-test_size = <span style="color: #B452CD">1</span> - train_size
-X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
-                                                    test_size=test_size)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">epochs = <span style="color: #B452CD">100</span>
-batch_size = <span style="color: #B452CD">100</span>
-n_neurons_layer1 = <span style="color: #B452CD">100</span>
-n_neurons_layer2 = <span style="color: #B452CD">50</span>
-n_categories = <span style="color: #B452CD">10</span>
-eta_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
-lmbd_vals = np.logspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">7</span>)
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">create_neural_network_keras</span>(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
-    model = Sequential()
-    model.add(Dense(n_neurons_layer1, activation=<span style="color: #CD5555">&#39;sigmoid&#39;</span>, kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_neurons_layer2, activation=<span style="color: #CD5555">&#39;sigmoid&#39;</span>, kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_categories, activation=<span style="color: #CD5555">&#39;softmax&#39;</span>))
-    
-    sgd = optimizers.SGD(lr=eta)
-    model.compile(loss=<span style="color: #CD5555">&#39;categorical_crossentropy&#39;</span>, optimizer=sgd, metrics=[<span style="color: #CD5555">&#39;accuracy&#39;</span>])
-    
-    <span style="color: #8B008B; font-weight: bold">return</span> model
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;">DNN_keras = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)), dtype=<span style="color: #658b00">object</span>)
-        
-<span style="color: #8B008B; font-weight: bold">for</span> i, eta <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(eta_vals):
-    <span style="color: #8B008B; font-weight: bold">for</span> j, lmbd <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(lmbd_vals):
-        DNN = create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
-                                         eta=eta, lmbd=lmbd)
-        DNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=<span style="color: #B452CD">0</span>)
-        scores = DNN.evaluate(X_test, Y_test)
-        
-        DNN_keras[i][j] = DNN
-        
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Learning rate = &quot;</span>, eta)
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test accuracy: %.3f&quot;</span> % scores[<span style="color: #B452CD">1</span>])
-        <span style="color: #658b00">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;"><span style="color: #228B22"># optional</span>
-<span style="color: #228B22"># visual representation of grid search</span>
-<span style="color: #228B22"># uses seaborn heatmap, could probably do this in matplotlib</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
-
-sns.set()
-
-train_accuracy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)))
-test_accuracy = np.zeros((<span style="color: #658b00">len</span>(eta_vals), <span style="color: #658b00">len</span>(lmbd_vals)))
-
-<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(eta_vals)):
-    <span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(lmbd_vals)):
-        DNN = DNN_keras[i][j]
-
-        train_accuracy[i][j] = DNN.evaluate(X_train, Y_train)[<span style="color: #B452CD">1</span>]
-        test_accuracy[i][j] = DNN.evaluate(X_test, Y_test)[<span style="color: #B452CD">1</span>]
-
-        
-fig, ax = plt.subplots(figsize = (<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
-sns.heatmap(train_accuracy, annot=<span style="color: #8B008B; font-weight: bold">True</span>, ax=ax, cmap=<span style="color: #CD5555">&quot;viridis&quot;</span>)
-ax.set_title(<span style="color: #CD5555">&quot;Training Accuracy&quot;</span>)
-ax.set_ylabel(<span style="color: #CD5555">&quot;$\eta$&quot;</span>)
-ax.set_xlabel(<span style="color: #CD5555">&quot;$\lambda$&quot;</span>)
-plt.show()
-
-fig, ax = plt.subplots(figsize = (<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
-sns.heatmap(test_accuracy, annot=<span style="color: #8B008B; font-weight: bold">True</span>, ax=ax, cmap=<span style="color: #CD5555">&quot;viridis&quot;</span>)
-ax.set_title(<span style="color: #CD5555">&quot;Test Accuracy&quot;</span>)
-ax.set_ylabel(<span style="color: #CD5555">&quot;$\eta$&quot;</span>)
-ax.set_xlabel(<span style="color: #CD5555">&quot;$\lambda$&quot;</span>)
-plt.show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-breast-cancer-data-now-with-keras">The Breast Cancer Data, now with Keras </h2>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #eeeedd">
-  <pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Sequential      <span style="color: #228B22">#This allows appending layers to existing models</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense           <span style="color: #228B22">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> optimizers             <span style="color: #228B22">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers           <span style="color: #228B22">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> to_categorical   <span style="color: #228B22">#This allows using categorical cross entropy as the cost function</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">seaborn</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">sns</span>
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split <span style="color: #8B008B; font-weight: bold">as</span> splitter
-<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> load_breast_cancer
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">pickle</span>
-<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span> 
-
-
-<span style="color: #CD5555">&quot;&quot;&quot;Load breast cancer dataset&quot;&quot;&quot;</span>
-
-np.random.seed(<span style="color: #B452CD">0</span>)        <span style="color: #228B22">#create same seed for random number every time</span>
-
-cancer=load_breast_cancer()      <span style="color: #228B22">#Download breast cancer dataset</span>
-
-inputs=cancer.data                     <span style="color: #228B22">#Feature matrix of 569 rows (samples) and 30 columns (parameters)</span>
-outputs=cancer.target                  <span style="color: #228B22">#Label array of 569 rows (0 for benign and 1 for malignant)</span>
-labels=cancer.feature_names[<span style="color: #B452CD">0</span>:<span style="color: #B452CD">30</span>]
-
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;The content of the breast cancer dataset is:&#39;</span>)      <span style="color: #228B22">#Print information about the datasets</span>
-<span style="color: #658b00">print</span>(labels)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;-------------------------&#39;</span>)
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;inputs =  &quot;</span> + <span style="color: #658b00">str</span>(inputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;outputs =  &quot;</span> + <span style="color: #658b00">str</span>(outputs.shape))
-<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;labels =  &quot;</span>+ <span style="color: #658b00">str</span>(labels.shape))
-
-x=inputs      <span style="color: #228B22">#Reassign the Feature and Label matrices to other variables</span>
-y=outputs
-
-<span style="color: #228B22">#%% </span>
-
-<span style="color: #228B22"># Visualisation of dataset (for correlation analysis)</span>
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">0</span>],x[:,<span style="color: #B452CD">2</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean radius&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean perimeter&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">5</span>],x[:,<span style="color: #B452CD">6</span>],s=<span style="color: #B452CD">40</span>,c=y, cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean compactness&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean concavity&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">0</span>],x[:,<span style="color: #B452CD">1</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean radius&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean texture&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,<span style="color: #B452CD">2</span>],x[:,<span style="color: #B452CD">1</span>],s=<span style="color: #B452CD">40</span>,c=y,cmap=plt.cm.Spectral)
-plt.xlabel(<span style="color: #CD5555">&#39;Mean perimeter&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.ylabel(<span style="color: #CD5555">&#39;Mean compactness&#39;</span>,fontweight=<span style="color: #CD5555">&#39;bold&#39;</span>)
-plt.show()
-
-
-<span style="color: #228B22"># Generate training and testing datasets</span>
-
-<span style="color: #228B22">#Select features relevant to classification (texture,perimeter,compactness and symmetery) </span>
-<span style="color: #228B22">#and add to input matrix</span>
-
-temp1=np.reshape(x[:,<span style="color: #B452CD">1</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">1</span>]),<span style="color: #B452CD">1</span>))
-temp2=np.reshape(x[:,<span style="color: #B452CD">2</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">2</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((temp1,temp2))      
-temp=np.reshape(x[:,<span style="color: #B452CD">5</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">5</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((X,temp))       
-temp=np.reshape(x[:,<span style="color: #B452CD">8</span>],(<span style="color: #658b00">len</span>(x[:,<span style="color: #B452CD">8</span>]),<span style="color: #B452CD">1</span>))
-X=np.hstack((X,temp))       
-
-X_train,X_test,y_train,y_test=splitter(X,y,test_size=<span style="color: #B452CD">0.1</span>)   <span style="color: #228B22">#Split datasets into training and testing</span>
-
-y_train=to_categorical(y_train)     <span style="color: #228B22">#Convert labels to categorical when using categorical cross entropy</span>
-y_test=to_categorical(y_test)
-
-<span style="color: #8B008B; font-weight: bold">del</span> temp1,temp2,temp
-
-<span style="color: #228B22"># %%</span>
-
-<span style="color: #228B22"># Define tunable parameters&quot;</span>
-
-eta=np.logspace(-<span style="color: #B452CD">3</span>,-<span style="color: #B452CD">1</span>,<span style="color: #B452CD">3</span>)                    <span style="color: #228B22">#Define vector of learning rates (parameter to SGD optimiser)</span>
-lamda=<span style="color: #B452CD">0.01</span>                                  <span style="color: #228B22">#Define hyperparameter</span>
-n_layers=<span style="color: #B452CD">2</span>                                  <span style="color: #228B22">#Define number of hidden layers in the model</span>
-n_neuron=np.logspace(<span style="color: #B452CD">0</span>,<span style="color: #B452CD">3</span>,<span style="color: #B452CD">4</span>,dtype=<span style="color: #658b00">int</span>)       <span style="color: #228B22">#Define number of neurons per layer</span>
-epochs=<span style="color: #B452CD">100</span>                                   <span style="color: #228B22">#Number of reiterations over the input data</span>
-batch_size=<span style="color: #B452CD">100</span>                              <span style="color: #228B22">#Number of samples per gradient update</span>
-
-<span style="color: #228B22"># %%</span>
-
-<span style="color: #CD5555">&quot;&quot;&quot;Define function to return Deep Neural Network model&quot;&quot;&quot;</span>
-
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">NN_model</span>(inputsize,n_layers,n_neuron,eta,lamda):
-    model=Sequential()      
-    <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_layers):       <span style="color: #228B22">#Run loop to add hidden layers to the model</span>
-        <span style="color: #8B008B; font-weight: bold">if</span> (i==<span style="color: #B452CD">0</span>):                  <span style="color: #228B22">#First layer requires input dimensions</span>
-            model.add(Dense(n_neuron,activation=<span style="color: #CD5555">&#39;relu&#39;</span>,kernel_regularizer=regularizers.l2(lamda),input_dim=inputsize))
-        <span style="color: #8B008B; font-weight: bold">else</span>:                       <span style="color: #228B22">#Subsequent layers are capable of automatic shape inferencing</span>
-            model.add(Dense(n_neuron,activation=<span style="color: #CD5555">&#39;relu&#39;</span>,kernel_regularizer=regularizers.l2(lamda)))
-    model.add(Dense(<span style="color: #B452CD">2</span>,activation=<span style="color: #CD5555">&#39;softmax&#39;</span>))  <span style="color: #228B22">#2 outputs - ordered and disordered (softmax for prob)</span>
-    sgd=optimizers.SGD(lr=eta)
-    model.compile(loss=<span style="color: #CD5555">&#39;categorical_crossentropy&#39;</span>,optimizer=sgd,metrics=[<span style="color: #CD5555">&#39;accuracy&#39;</span>])
-    <span style="color: #8B008B; font-weight: bold">return</span> model
-
-    
-Train_accuracy=np.zeros((<span style="color: #658b00">len</span>(n_neuron),<span style="color: #658b00">len</span>(eta)))      <span style="color: #228B22">#Define matrices to store accuracy scores as a function</span>
-Test_accuracy=np.zeros((<span style="color: #658b00">len</span>(n_neuron),<span style="color: #658b00">len</span>(eta)))       <span style="color: #228B22">#of learning rate and number of hidden neurons for </span>
-
-<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(n_neuron)):     <span style="color: #228B22">#run loops over hidden neurons and learning rates to calculate </span>
-    <span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(eta)):      <span style="color: #228B22">#accuracy scores </span>
-        DNN_model=NN_model(X_train.shape[<span style="color: #B452CD">1</span>],n_layers,n_neuron[i],eta[j],lamda)
-        DNN_model.fit(X_train,y_train,epochs=epochs,batch_size=batch_size,verbose=<span style="color: #B452CD">1</span>)
-        Train_accuracy[i,j]=DNN_model.evaluate(X_train,y_train)[<span style="color: #B452CD">1</span>]
-        Test_accuracy[i,j]=DNN_model.evaluate(X_test,y_test)[<span style="color: #B452CD">1</span>]
-               
-
-<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_data</span>(x,y,data,title=<span style="color: #8B008B; font-weight: bold">None</span>):
-
-    <span style="color: #228B22"># plot results</span>
-    fontsize=<span style="color: #B452CD">16</span>
-
-
-    fig = plt.figure()
-    ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
-    cax = ax.matshow(data, interpolation=<span style="color: #CD5555">&#39;nearest&#39;</span>, vmin=<span style="color: #B452CD">0</span>, vmax=<span style="color: #B452CD">1</span>)
-    
-    cbar=fig.colorbar(cax)
-    cbar.ax.set_ylabel(<span style="color: #CD5555">&#39;accuracy (%)&#39;</span>,rotation=<span style="color: #B452CD">90</span>,fontsize=fontsize)
-    cbar.set_ticks([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">.2</span>,<span style="color: #B452CD">.4</span>,<span style="color: #B452CD">0.6</span>,<span style="color: #B452CD">0.8</span>,<span style="color: #B452CD">1.0</span>])
-    cbar.set_ticklabels([<span style="color: #CD5555">&#39;0%&#39;</span>,<span style="color: #CD5555">&#39;20%&#39;</span>,<span style="color: #CD5555">&#39;40%&#39;</span>,<span style="color: #CD5555">&#39;60%&#39;</span>,<span style="color: #CD5555">&#39;80%&#39;</span>,<span style="color: #CD5555">&#39;100%&#39;</span>])
-
-    <span style="color: #228B22"># put text on matrix elements</span>
-    <span style="color: #8B008B; font-weight: bold">for</span> i, x_val <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(np.arange(<span style="color: #658b00">len</span>(x))):
-        <span style="color: #8B008B; font-weight: bold">for</span> j, y_val <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(np.arange(<span style="color: #658b00">len</span>(y))):
-            c = <span style="color: #CD5555">&quot;${0:.1f}\\%$&quot;</span>.format( <span style="color: #B452CD">100</span>*data[j,i])  
-            ax.text(x_val, y_val, c, va=<span style="color: #CD5555">&#39;center&#39;</span>, ha=<span style="color: #CD5555">&#39;center&#39;</span>)
-
-    <span style="color: #228B22"># convert axis vaues to to string labels</span>
-    x=[<span style="color: #658b00">str</span>(i) <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> x]
-    y=[<span style="color: #658b00">str</span>(i) <span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> y]
-
-
-    ax.set_xticklabels([<span style="color: #CD5555">&#39;&#39;</span>]+x)
-    ax.set_yticklabels([<span style="color: #CD5555">&#39;&#39;</span>]+y)
-
-    ax.set_xlabel(<span style="color: #CD5555">&#39;$\\mathrm{learning\\ rate}$&#39;</span>,fontsize=fontsize)
-    ax.set_ylabel(<span style="color: #CD5555">&#39;$\\mathrm{hidden\\ neurons}$&#39;</span>,fontsize=fontsize)
-    <span style="color: #8B008B; font-weight: bold">if</span> title <span style="color: #8B008B">is</span> <span style="color: #8B008B">not</span> <span style="color: #8B008B; font-weight: bold">None</span>:
-        ax.set_title(title)
-
-    plt.tight_layout()
-
-    plt.show()
-    
-plot_data(eta,n_neuron,Train_accuracy, <span style="color: #CD5555">&#39;training&#39;</span>)
-plot_data(eta,n_neuron,Test_accuracy, <span style="color: #CD5555">&#39;testing&#39;</span>)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="fine-tuning-neural-network-hyperparameters">Fine-tuning neural network hyperparameters </h2>
-
-<p>The flexibility of neural networks is also one of their main
-drawbacks: there are many hyperparameters to tweak. Not only can you
-use any imaginable network topology (how neurons/nodes are interconnected),
-but even in a simple FFNN you can change the number of layers, the
-number of neurons per layer, the type of activation function to use in
-each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
-know what combination of hyperparameters is the best for your task?
-</p>
-
-<ul>
-<li> You can use grid search with cross-validation to find the right hyperparameters.</li>
-</ul>
-<p>However,since there are many hyperparameters to tune, and since
-training a neural network on a large dataset takes a lot of time, you
-will only be able to explore a tiny part of the hyperparameter space.
-</p>
-
-<ul>
-<li> You can use randomized search.</li>
-<li> Or use tools like <a href="http://oscar.calldesk.ai/" target="_blank">Oscar</a>, which implements more complex algorithms to help you find a good set of hyperparameters quickly.</li>  
-</ul>
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="hidden-layers">Hidden layers </h2>
-
-<p>For many problems you can start with just one or two hidden layers and it will work just fine.
-For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
-few hundred neurons.
-You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
-neurons, in roughly the same amount of training time. 
-</p>
-
-<p>For more complex problems, you can gradually
-ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
-as large image classification or speech recognition, typically require networks with dozens of layers
-and they need a huge amount
-of training data. However, you will rarely have to train such networks from scratch: it is much more
-common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
-</p>
-
-<!-- !split  -->
-<h2 id="which-activation-function-should-i-use">Which activation function should I use? </h2>
-
-<p>The Back propagation algorithm we derived above works by going from
-the output layer to the input layer, propagating the error gradient on
-the way. Once the algorithm has computed the gradient of the cost
-function with regards to each parameter in the network, it uses these
-gradients to update each parameter with a Gradient Descent (GD) step.
-</p>
-
-<p>Unfortunately for us, the gradients often get smaller and smaller as the
-algorithm progresses down to the first hidden layers. As a result, the
-GD update leaves the lower layer connection weights
-virtually unchanged, and training never converges to a good
-solution. This is known in the literature as 
-<b>the vanishing gradients problem</b>. 
-</p>
-
-<p>In other cases, the opposite can happen, namely the the gradients can grow bigger and
-bigger. The result is that many of the layers get large updates of the 
-weights the
-algorithm diverges. This is the <b>exploding gradients problem</b>, which is
-mostly encountered in recurrent neural networks. More generally, deep
-neural networks suffer from unstable gradients, different layers may
-learn at widely different speeds
-</p>
-
-<!-- !split  -->
-<h2 id="is-the-logistic-activation-function-sigmoid-our-choice">Is the Logistic activation function (Sigmoid)  our choice? </h2>
-
-<p>Although this unfortunate behavior has been empirically observed for
-quite a while (it was one of the reasons why deep neural networks were
-mostly abandoned for a long time), it is only around 2010 that
-significant progress was made in understanding it.
-</p>
-
-<p>A paper titled <a href="http://proceedings.mlr.press/v9/glorot10a.html" target="_blank">Understanding the Difficulty of Training Deep
-Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio</a> found that
-the problems with the popular logistic
-sigmoid activation function and the weight initialization technique
-that was most popular at the time, namely random initialization using
-a normal distribution with a mean of 0 and a standard deviation of
-1. 
-</p>
-
-<p>They showed that with this activation function and this
-initialization scheme, the variance of the outputs of each layer is
-much greater than the variance of its inputs. Going forward in the
-network, the variance keeps increasing after each layer until the
-activation function saturates at the top layers. This is actually made
-worse by the fact that the logistic function has a mean of 0.5, not 0
-(the hyperbolic tangent function has a mean of 0 and behaves slightly
-better than the logistic function in deep networks).
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-derivative-of-the-logistic-funtion">The derivative of the Logistic funtion </h2>
-
-<p>Looking at the logistic activation function, when inputs become large
-(negative or positive), the function saturates at 0 or 1, with a
-derivative extremely close to 0. Thus when backpropagation kicks in,
-it has virtually no gradient to propagate back through the network,
-and what little gradient exists keeps getting diluted as
-backpropagation progresses down through the top layers, so there is
-really nothing left for the lower layers.
-</p>
-
-<p>In their paper, Glorot and Bengio propose a way to significantly
-alleviate this problem. We need the signal to flow properly in both
-directions: in the forward direction when making predictions, and in
-the reverse direction when backpropagating gradients. We don&#8217;t want
-the signal to die out, nor do we want it to explode and saturate. For
-the signal to flow properly, the authors argue that we need the
-variance of the outputs of each layer to be equal to the variance of
-its inputs, and we also need the gradients to have equal variance
-before and after flowing through a layer in the reverse direction.
-</p>
-
-<p>One of the insights in the 2010 paper by Glorot and Bengio was that
-the vanishing/exploding gradients problems were in part due to a poor
-choice of activation function. Until then most people had assumed that
-if Nature had chosen to use roughly sigmoid activation functions in
-biological neurons, they must be an excellent choice. But it turns out
-that other activation functions behave much better in deep neural
-networks, in particular the ReLU activation function, mostly because
-it does not saturate for positive values (and also because it is quite
-fast to compute).
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-relu-function-family">The RELU function family </h2>
-
-<p>The ReLU activation function suffers from a problem known as the dying
-ReLUs: during training, some neurons effectively die, meaning they
-stop outputting anything other than 0.
-</p>
-
-<p>In some cases, you may find that half of your network&#8217;s neurons are
-dead, especially if you used a large learning rate. During training,
-if a neuron&#8217;s weights get updated such that the weighted sum of the
-neuron&#8217;s inputs is negative, it will start outputting 0. When this
-happen, the neuron is unlikely to come back to life since the gradient
-of the ReLU function is 0 when its input is negative.
-</p>
-
-<p>To solve this problem, nowadays practitioners use a  variant of the ReLU
-function, such as the leaky ReLU discussed above or the so-called
-exponential linear unit (ELU) function
-</p>
-
-$$
-ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\  z & z \ge 0.\end{array}\right. 
-$$
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="which-activation-function-should-we-use">Which activation function should we use? </h2>
-
-<p>In general it seems that the ELU activation function is better than
-the leaky ReLU function (and its variants), which is better than
-ReLU. ReLU performs better than \( \tanh \) which in turn performs better
-than the logistic function. 
-</p>
-
-<p>If runtime
-performance is an issue, then you may opt for the  leaky ReLU function  over the 
-ELU function If you don&#8217;t
-want to tweak yet another hyperparameter, you may just use the default
-\( \alpha \) of \( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have
-spare time and computing power, you can use cross-validation or
-bootstrap to evaluate other activation functions.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="more-on-activation-functions-output-layers">More on activation functions, output layers </h2>
-
-<p>In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).</p>
-
-<p>It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.</p>
-
-<b>For the output layer:</b>
-
-<ul>
-<li> For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).</li>
-<li> For regression tasks, you can simply use no activation function at all.</li>
-</ul>
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="batch-normalization">Batch Normalization </h2>
-
-<p>Batch Normalization
-aims to address the vanishing/exploding gradients problems, and more generally the problem that the
-distribution of each layer&#8217;s inputs changes during training, as the parameters of the previous layers change.
-</p>
-
-<p>The technique consists of adding an operation in the model just before the activation function of each
-layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
-parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
-learn the optimal scale and mean of the inputs for each layer.
-In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs&#8217; mean and
-standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
-mini-batch, from this the name batch normalization.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="dropout">Dropout </h2>
-
-<p>It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
-excluding the output neurons) has a probability \( p \) of being temporarily dropped out, meaning it will be
-entirely ignored during this training step, but it may be active during the next step.
-</p>
-
-<p>The
-hyperparameter \( p \) is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
- It is viewed as one of the most popular regularization techniques.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="gradient-clipping">Gradient Clipping </h2>
-
-<p>A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
-backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
-networks).
-</p>
-
-<p>This technique is called Gradient Clipping.</p>
-
-<p>In general however, Batch
-Normalization is preferred.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="a-very-nice-website-on-neural-networks">A very nice website on Neural Networks </h2>
-
-<p>You may find this <a href="https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false" target="_blank">website</a> very useful. </p>
-
-<!-- !split  -->
-<h2 id="a-top-down-perspective-on-neural-networks">A top-down perspective on Neural networks </h2>
-
-<p>The first thing we would like to do is divide the data into two or three
-parts. A training set, a validation or dev (development) set, and a
-test set. The test set is the data on which we want to make
-predictions. The dev set is a subset of the training data we use to
-check how well we are doing out-of-sample, after training the model on
-the training dataset. We use the validation error as a proxy for the
-test error in order to make tweaks to our model. It is crucial that we
-do not use any of the test data to train the algorithm. This is a
-cardinal sin in ML. Then:
-</p>
-
-<ul>
-<li> Estimate optimal error rate</li>
-<li> Minimize underfitting (bias) on training data set.</li>
-<li> Make sure you are not overfitting.</li>
-</ul>
-<p>If the validation and test sets are drawn from the same distributions,
-then a good performance on the validation set should lead to similarly
-good performance on the test set. 
-</p>
-
-<p>However, sometimes
-the training data and test data differ in subtle ways because, for
-example, they are collected using slightly different methods, or
-because it is cheaper to collect data in one way versus another. In
-this case, there can be a mismatch between the training and test
-data. This can lead to the neural network overfitting these small
-differences between the test and training sets, and a poor performance
-on the test set despite having a good performance on the validation
-set. To rectify this, Andrew Ng suggests making two validation or dev
-sets, one constructed from the training data and one constructed from
-the test data. The difference between the performance of the algorithm
-on these two validation sets quantifies the train-test mismatch. This
-can serve as another important diagnostic when using DNNs for
-supervised learning.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="limitations-of-supervised-learning-with-deep-networks">Limitations of supervised learning with deep networks </h2>
-
-<p>Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
-</p>
-
-<p>Here we list some of the important limitations of supervised neural network based models. </p>
-
-<ul>
-<li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
-<li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
-<li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.&nbsp;some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
-<li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
-</ul>
-<p>Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.</p>
-
 <!-- ------------------- end of main content --------------- -->
 <center style="font-size:80%">
 <!-- copyright --> &copy; 1999-2023, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html
index f6f34ab4a..3a8aa790c 100644
--- a/doc/pub/week41/html/week41.html
+++ b/doc/pub/week41/html/week41.html
@@ -8,8 +8,8 @@ doconce format html week41.do.txt --pygments_html_style=default --html_style=blo
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
-<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
+<meta name="description" content="Week 41 Neural networks and constructing a neural network code">
+<title>Week 41 Neural networks and constructing a neural network code</title>
 <style type="text/css">
 /* bloodish style */
 body {
@@ -141,14 +141,107 @@ div.toc p,a {
 <!-- tocinfo
 {'highest level': 2,
  'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
-              ('Videos on Neural Networks',
+              ('Lecture Thursday October 12',
                2,
                None,
-               'videos-on-neural-networks'),
-              ('Review of the back propagation algorithm',
+               'lecture-thursday-october-12'),
+              ('Introduction to Neural networks',
                2,
                None,
-               'review-of-the-back-propagation-algorithm'),
+               'introduction-to-neural-networks'),
+              ('Artificial neurons', 2, None, 'artificial-neurons'),
+              ('Neural network types', 2, None, 'neural-network-types'),
+              ('Feed-forward neural networks',
+               2,
+               None,
+               'feed-forward-neural-networks'),
+              ('Convolutional Neural Network',
+               2,
+               None,
+               'convolutional-neural-network'),
+              ('Recurrent neural networks',
+               2,
+               None,
+               'recurrent-neural-networks'),
+              ('Other types of networks', 2, None, 'other-types-of-networks'),
+              ('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
+              ('Why multilayer perceptrons?',
+               2,
+               None,
+               'why-multilayer-perceptrons'),
+              ('Illustration of a single perceptron model and a '
+               'multi-perceptron model',
+               2,
+               None,
+               'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
+              ('Examples of XOR, OR and AND gates',
+               2,
+               None,
+               'examples-of-xor-or-and-and-gates'),
+              ('Does Logistic Regression do a better Job?',
+               2,
+               None,
+               'does-logistic-regression-do-a-better-job'),
+              ('Adding Neural Networks', 2, None, 'adding-neural-networks'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Mathematical model', 2, None, 'mathematical-model'),
+              ('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
+              ('Matrix-vector notation  and activation',
+               3,
+               None,
+               'matrix-vector-notation-and-activation'),
+              ('Activation functions', 3, None, 'activation-functions'),
+              ('Activation functions, Logistic and Hyperbolic ones',
+               3,
+               None,
+               'activation-functions-logistic-and-hyperbolic-ones'),
+              ('Relevance', 3, None, 'relevance'),
+              ('The multilayer  perceptron (MLP)',
+               2,
+               None,
+               'the-multilayer-perceptron-mlp'),
+              ('From one to many layers, the universal approximation theorem',
+               2,
+               None,
+               'from-one-to-many-layers-the-universal-approximation-theorem'),
+              ('Deriving the back propagation code for a multilayer perceptron '
+               'model',
+               2,
+               None,
+               'deriving-the-back-propagation-code-for-a-multilayer-perceptron-model'),
+              ('Definitions', 2, None, 'definitions'),
+              ('Derivatives and the chain rule',
+               2,
+               None,
+               'derivatives-and-the-chain-rule'),
+              ('Derivative of the cost function',
+               2,
+               None,
+               'derivative-of-the-cost-function'),
+              ('Bringing it together, first back propagation equation',
+               2,
+               None,
+               'bringing-it-together-first-back-propagation-equation'),
+              ('Derivatives in terms of $z_j^L$',
+               2,
+               None,
+               'derivatives-in-terms-of-z-j-l'),
+              ('Bringing it together', 2, None, 'bringing-it-together'),
+              ('Final back propagating equation',
+               2,
+               None,
+               'final-back-propagating-equation'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
+              ('Setting up the Back propagation algorithm',
+               2,
+               None,
+               'setting-up-the-back-propagation-algorithm'),
               ('Setting up the Back propagation algorithm',
                2,
                None,
@@ -226,62 +319,7 @@ div.toc p,a {
               ('The Code using Scikit-Learn',
                2,
                None,
-               'the-code-using-scikit-learn'),
-              ('Building neural networks in Tensorflow and Keras',
-               2,
-               None,
-               'building-neural-networks-in-tensorflow-and-keras'),
-              ('Tensorflow', 2, None, 'tensorflow'),
-              ('Using Keras', 2, None, 'using-keras'),
-              ('Collect and pre-process data',
-               2,
-               None,
-               'collect-and-pre-process-data'),
-              ('The Breast Cancer Data, now with Keras',
-               2,
-               None,
-               'the-breast-cancer-data-now-with-keras'),
-              ('Fine-tuning neural network hyperparameters',
-               2,
-               None,
-               'fine-tuning-neural-network-hyperparameters'),
-              ('Hidden layers', 2, None, 'hidden-layers'),
-              ('Which activation function should I use?',
-               2,
-               None,
-               'which-activation-function-should-i-use'),
-              ('Is the Logistic activation function (Sigmoid)  our choice?',
-               2,
-               None,
-               'is-the-logistic-activation-function-sigmoid-our-choice'),
-              ('The derivative of the Logistic funtion',
-               2,
-               None,
-               'the-derivative-of-the-logistic-funtion'),
-              ('The RELU function family', 2, None, 'the-relu-function-family'),
-              ('Which activation function should we use?',
-               2,
-               None,
-               'which-activation-function-should-we-use'),
-              ('More on activation functions, output layers',
-               2,
-               None,
-               'more-on-activation-functions-output-layers'),
-              ('Batch Normalization', 2, None, 'batch-normalization'),
-              ('Dropout', 2, None, 'dropout'),
-              ('Gradient Clipping', 2, None, 'gradient-clipping'),
-              ('A very nice website on Neural Networks',
-               2,
-               None,
-               'a-very-nice-website-on-neural-networks'),
-              ('A top-down perspective on Neural networks',
-               2,
-               None,
-               'a-top-down-perspective-on-neural-networks'),
-              ('Limitations of supervised learning with deep networks',
-               2,
-               None,
-               'limitations-of-supervised-learning-with-deep-networks')]}
+               'the-code-using-scikit-learn')]}
 end of tocinfo -->
 
 <body>
@@ -303,7 +341,7 @@ MathJax.Hub.Config({
 
 <!-- ------------------- main content ---------------------- -->
 <center>
-<h1>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</h1>
+<h1>Week 41 Neural networks and constructing a neural network code</h1>
 </center>  <!-- document title -->
 
 <!-- author(s): Morten Hjorth-Jensen -->
@@ -326,37 +364,1199 @@ MathJax.Hub.Config({
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
 <h2 id="plan-for-week-41">Plan for week 41 </h2>
 
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the active learning sessions on Tuesday and Wednesday</b>
+<p>
 <ul>
-<li> Building our own Feed-forward Neural Network and discussion of project 2.</li>
-<li> Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks.</li> 
+  <li> Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation</li>
+  <li> One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code</li>
+  <li> Discussion of project 2</li>
+  <li> Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent</li>
 </ul>
-<p>"
-Reading suggestions: These notes, <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 10-11</a>.
-For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read.
+</div>
+  
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>Material for the lecture on Thursday October 12, 2023</b>
+<p>
+<ul>
+  <li> Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.</li>
+  <li> Building our own Feed-forward Neural Network</li>
+  <li> Readings and Videos:</li>
+<ul>
+    <li> These lecture notes</li>
+    <li> For neural networks we recommend Goodfellow et al chapter 6.</li>
+    <li> <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_blank">Neural Networks demystified</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex" target="_blank">Building Neural Networks from scratch</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
+    <li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
+</ul>
+</ul>
+<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="lecture-thursday-october-12">Lecture Thursday October 12 </h2>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="introduction-to-neural-networks">Introduction to Neural networks </h2>
+
+<p>Artificial neural networks are computational systems that can learn to
+perform tasks by considering examples, generally without being
+programmed with any task-specific rules. It is supposed to mimic a
+biological system, wherein neurons interact by sending signals in the
+form of mathematical functions between layers. All layers can contain
+an arbitrary number of neurons, and each connection is represented by
+a weight variable.
 </p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="videos-on-neural-networks">Videos on Neural Networks </h2>
+<h2 id="artificial-neurons">Artificial neurons  </h2>
 
-<ul>
-<li> <a href="https://www.youtube.com/watch?v=CqOfi41LfDw" target="_blank">Video on Neural Networks</a></li>
-<li> <a href="https://www.youtube.com/watch?v=Ilg3gGewQ5U" target="_blank">Video on the back propagation algorithm</a></li>
-</ul>
-<p>I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <a href="http://neuralnetworksanddeeplearning.com/chap4.html" target="_blank"><tt>http://neuralnetworksanddeeplearning.com/chap4.html</tt></a>.</p>
+<p>The field of artificial neural networks has a long history of
+development, and is closely connected with the advancement of computer
+science and computers in general. A model of artificial neurons was
+first developed by McCulloch and Pitts in 1943 to study signal
+processing in the brain and has later been refined by others. The
+general idea is to mimic neural networks in the human brain, which is
+composed of billions of neurons that communicate with each other by
+sending electrical signals.  Each neuron accumulates its incoming
+signals, which must exceed an activation threshold to yield an
+output. If the threshold is not overcome, the neuron remains inactive,
+i.e. has zero output.
+</p>
+
+<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
+
+$$
+\begin{equation}
+ y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
+\label{artificialNeuron}
+\end{equation}
+$$
+
+<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
+a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
+</p>
+
+<p>Conceptually, it is helpful to divide neural networks into four
+categories:
+</p>
+<ol>
+<li> general purpose neural networks for supervised learning,</li>
+<li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
+<li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
+<li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
+</ol>
+<p>In natural science, DNNs and CNNs have already found numerous
+applications. In statistical physics, they have been applied to detect
+phase transitions in 2D Ising and Potts models, lattice gauge
+theories, and different phases of polymers, or solving the
+Navier-Stokes equation in weather forecasting.  Deep learning has also
+found interesting applications in quantum physics. Various quantum
+phase transitions can be detected and studied using DNNs and CNNs,
+topological phases, and even non-equilibrium many-body
+localization. Representing quantum states as DNNs quantum state
+tomography are among some of the impressive achievements to reveal the
+potential of DNNs to facilitate the study of quantum systems.
+</p>
+
+<p>In quantum information theory, it has been shown that one can perform
+gate decompositions with the help of neural. 
+</p>
+
+<p>The applications are not limited to the natural sciences. There is a
+plethora of applications in essentially all disciplines, from the
+humanities to life science and medicine.
+</p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="review-of-the-back-propagation-algorithm">Review of the back propagation algorithm </h2>
+<h2 id="neural-network-types">Neural network types </h2>
 
-<p>During the last lecture we discussed in detail the back propagation
-algorithm. This algorithm is based on a repeated application of the
-chain rule. Let us bring back the basic equation and at the same time
-link this with the basic mathematics of automatic differentiation.
+<p>An artificial neural network (ANN), is a computational model that
+consists of layers of connected neurons, or nodes or units.  We will
+refer to these interchangeably as units or nodes, and sometimes as
+neurons.
+</p>
+
+<p>It is supposed to mimic a biological nervous system by letting each
+neuron interact with other neurons by sending signals in the form of
+mathematical functions between layers.  A wide variety of different
+ANNs have been developed, but most of them consist of an input layer,
+an output layer and eventual layers in-between, called <em>hidden
+layers</em>. All layers can contain an arbitrary number of nodes, and each
+connection between two nodes is associated with a weight variable.
+</p>
+
+<p>Neural networks (also called neural nets) are neural-inspired
+nonlinear models for supervised learning.  As we will see, neural nets
+can be viewed as natural, more powerful extensions of supervised
+learning methods such as linear and logistic regression and soft-max
+methods we discussed earlier.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="feed-forward-neural-networks">Feed-forward neural networks </h2>
+
+<p>The feed-forward neural network (FFNN) was the first and simplest type
+of ANNs that were devised. In this network, the information moves in
+only one direction: forward through the layers.
+</p>
+
+<p>Nodes are represented by circles, while the arrows display the
+connections between the nodes, including the direction of information
+flow. Additionally, each arrow corresponds to a weight variable
+(figure to come).  We observe that each node in a layer is connected
+to <em>all</em> nodes in the subsequent layer, making this a so-called
+<em>fully-connected</em> FFNN.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="convolutional-neural-network">Convolutional Neural Network </h2>
+
+<p>A different variant of FFNNs are <em>convolutional neural networks</em>
+(CNNs), which have a connectivity pattern inspired by the animal
+visual cortex. Individual neurons in the visual cortex only respond to
+stimuli from small sub-regions of the visual field, called a receptive
+field. This makes the neurons well-suited to exploit the strong
+spatially local correlation present in natural images. The response of
+each neuron can be approximated mathematically as a convolution
+operation.  (figure to come)
+</p>
+
+<p>Convolutional neural networks emulate the behaviour of neurons in the
+visual cortex by enforcing a <em>local</em> connectivity pattern between
+nodes of adjacent layers: Each node in a convolutional layer is
+connected only to a subset of the nodes in the previous layer, in
+contrast to the fully-connected FFNN.  Often, CNNs consist of several
+convolutional layers that learn local features of the input, with a
+fully-connected layer at the end, which gathers all the local data and
+produces the outputs. They have wide applications in image and video
+recognition.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="recurrent-neural-networks">Recurrent neural networks </h2>
+
+<p>So far we have only mentioned ANNs where information flows in one
+direction: forward. <em>Recurrent neural networks</em> on the other hand,
+have connections between nodes that form directed <em>cycles</em>. This
+creates a form of internal memory which are able to capture
+information on what has been calculated before; the output is
+dependent on the previous computations. Recurrent NNs make use of
+sequential information by performing the same task for every element
+in a sequence, where each element depends on previous elements. An
+example of such information is sentences, making recurrent NNs
+especially well-suited for handwriting and speech recognition.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="other-types-of-networks">Other types of networks </h2>
+
+<p>There are many other kinds of ANNs that have been developed. One type
+that is specifically designed for interpolation in multidimensional
+space is the radial basis function (RBF) network. RBFs are typically
+made up of three layers: an input layer, a hidden layer with
+non-linear radial symmetric activation functions and a linear output
+layer (''linear'' here means that each node in the output layer has a
+linear activation function). The layers are normally fully-connected
+and there are no cycles, thus RBFs can be viewed as a type of
+fully-connected FFNN. They are however usually treated as a separate
+type of NN due the unusual activation functions.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="multilayer-perceptrons">Multilayer perceptrons  </h2>
+
+<p>One uses often so-called fully-connected feed-forward neural networks
+with three or more layers (an input layer, one or more hidden layers
+and an output layer) consisting of neurons that have non-linear
+activation functions.
+</p>
+
+<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="why-multilayer-perceptrons">Why multilayer perceptrons?  </h2>
+
+<p>According to the <em>Universal approximation theorem</em>, a feed-forward
+neural network with just a single hidden layer containing a finite
+number of neurons can approximate a continuous multidimensional
+function to arbitrary accuracy, assuming the activation function for
+the hidden layer is a <b>non-constant, bounded and
+monotonically-increasing continuous function</b>.
+</p>
+
+<p>Note that the requirements on the activation function only applies to
+the hidden layer, the output nodes are always assumed to be linear, so
+as to not restrict the range of output values.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model">Illustration of a single perceptron model and a multi-perceptron model </h2>
+
+<center>  <!-- FIGURE -->
+<hr class="figure">
+<center>
+<p class="caption">Figure 1:  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer. </p>
+</center>
+<p><img src="figures/nns.png" width="600" align="bottom"></p>
+</center>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="examples-of-xor-or-and-and-gates">Examples of XOR, OR and AND gates </h2>
+
+<p>Let us first try to fit various gates using standard linear
+regression. The gates we are thinking of are the classical XOR, OR and
+AND gates, well-known elements in computer science. The tables here
+show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
+specific target \( y_i \).
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<span style="color: #BA2121; font-style: italic">Simple code that tests XOR, OR and AND gates with linear regression</span>
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The invers of X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+<span style="color: #408080; font-style: italic"># The XOR gate </span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+ThetaXOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+ThetaOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+ThetaAND  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>What is happening here?</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="does-logistic-regression-do-a-better-job">Does Logistic Regression do a better Job? </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+<span style="color: #BA2121; font-style: italic">Simple code that tests XOR and OR gates with linear regression</span>
+<span style="color: #BA2121; font-style: italic">and logistic regression</span>
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
+
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
+
+<span style="color: #408080; font-style: italic"># Design matrix</span>
+X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([ [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">1</span>], [<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>],[<span style="color: #666666">1</span>, <span style="color: #666666">1</span>, <span style="color: #666666">1</span>]],dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+Xinv <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The invers of X.TX  matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>Xinv<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+<span style="color: #408080; font-style: italic"># The XOR gate </span>
+yXOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">0</span>])
+ThetaXOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yXOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaXOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yOR <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">1</span> ,<span style="color: #666666">1</span>, <span style="color: #666666">1</span>])
+ThetaOR  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yOR
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the OR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaOR<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+
+<span style="color: #408080; font-style: italic"># The OR gate </span>
+yAND <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array( [ <span style="color: #666666">0</span>, <span style="color: #666666">0</span> ,<span style="color: #666666">0</span>, <span style="color: #666666">1</span>])
+ThetaAND  <span style="color: #666666">=</span> Xinv <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> yAND
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The values of theta for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;The linear regression prediction  for the AND gate:</span><span style="color: #BB6688; font-weight: bold">{</span>X <span style="color: #666666">@</span> ThetaAND<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+
+<span style="color: #408080; font-style: italic"># Now we change to logistic regression</span>
+
+
+<span style="color: #408080; font-style: italic"># Logistic Regression</span>
+logreg <span style="color: #666666">=</span> LogisticRegression()
+logreg<span style="color: #666666">.</span>fit(X, yOR)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for OR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yOR)))
+
+logreg<span style="color: #666666">.</span>fit(X, yXOR)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for XOR gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yXOR)))
+
+
+logreg<span style="color: #666666">.</span>fit(X, yAND)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Logistic Regression for AND gate: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X,yAND)))
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<p>Not exactly impressive, but somewhat better.</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="adding-neural-networks">Adding Neural Networks </h2>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># and now neural networks with Scikit-Learn and the XOR</span>
+
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.neural_network</span> <span style="color: #008000; font-weight: bold">import</span> MLPClassifier
+<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> make_classification
+X, yXOR <span style="color: #666666">=</span> make_classification(n_samples<span style="color: #666666">=100</span>, random_state<span style="color: #666666">=1</span>)
+FFNN <span style="color: #666666">=</span> MLPClassifier(random_state<span style="color: #666666">=1</span>, max_iter<span style="color: #666666">=300</span>)<span style="color: #666666">.</span>fit(X, yXOR)
+FFNN<span style="color: #666666">.</span>predict_proba(X)
+<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;Test set accuracy with Feed Forward Neural Network  for XOR gate:</span><span style="color: #BB6688; font-weight: bold">{</span>FFNN<span style="color: #666666">.</span>score(X, yXOR)<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output \( y \) is produced via the activation function \( f \)</p>
+$$
+ y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
+$$
+
+<p>This function receives \( x_i \) as inputs.
+Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \). 
+In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
+the neurons in the preceding layer. Furthermore, an MLP is
+fully-connected, which means that each neuron receives a weighted sum
+of the outputs of <em>all</em> neurons in the previous layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
+
+$$
+\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
+\label{_auto1}
+\end{equation}
+$$
+
+<p>Here \( b_i \) is the so-called bias which is normally needed in
+case of zero activation weights or inputs. How to fix the biases and
+the weights will be discussed below.  The value of \( z_i^1 \) is the
+argument to the activation function \( f_i \) of each node \( i \), The
+variable \( M \) stands for all possible inputs to a given node \( i \) in the
+first layer.  We define  the output \( y_i^1 \) of all neurons in layer 1 as
+</p>
+
+$$
+\begin{equation}
+ y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
+\label{outputLayer1}
+\end{equation}
+$$
+
+<p>where we assume that all nodes in the same layer have identical
+activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
+In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
+</p>
+
+$$
+\begin{equation}
+ y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
+\label{generalLayer}
+\end{equation}
+$$
+
+<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
+all the nodes in the first hidden layer are computed, the values of
+the subsequent layer can be calculated and so forth until the output
+is obtained.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>The output of neuron \( i \) in layer 2 is thus,</p>
+
+$$
+\begin{align}
+ y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) 
+\label{_auto2}\\
+ &= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
+\label{outputLayer2}
+\end{align}
+$$
+
+<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
+
+$$
+\begin{align}
+ y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) 
+\label{_auto3}\\
+ &= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+  + b_1^3\right]
+\label{_auto4}
+\end{align}
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>We can generalize this expression to an MLP with \( l \) hidden
+layers. The complete functional form is,
+</p>
+
+$$
+\begin{align}
+&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
+\label{completeNN}
+\end{align}
+$$
+
+<p>which illustrates a basic property of MLPs: The only independent
+variables are the input values \( x_n \).
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="mathematical-model">Mathematical model  </h2>
+
+<p>This confirms that an MLP, despite its quite convoluted mathematical
+form, is nothing more than an analytic function, specifically a
+mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
+\hat{y} \in \mathbb{R}^m \).
+</p>
+
+<p>Furthermore, the flexibility and universality of an MLP can be
+illustrated by realizing that the expression is essentially a nested
+sum of scaled activation functions of the form
+</p>
+
+$$
+\begin{equation}
+ f(x) = c_1 f(c_2 x + c_3) + c_4
+\label{_auto5}
+\end{equation}
+$$
+
+<p>where the parameters \( c_i \) are weights and biases. By adjusting these
+parameters, the activation functions can be shifted up and down or
+left and right, change slope or be rescaled which is the key to the
+flexibility of a neural network.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="matrix-vector-notation">Matrix-vector notation </h3>
+
+<p>We can introduce a more convenient notation for the activations in an A NN. </p>
+
+<p>Additionally, we can represent the biases and activations
+as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector 
+is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively. 
+</p>
+
+<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors. 
+With this notation, the sum becomes a matrix-vector multiplication, and we can write
+the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
+</p>
+$$
+\begin{equation}
+ \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
+ f_2\left(\left[\begin{array}{ccc}
+    w^2_{11} &w^2_{12} &w^2_{13} \\
+    w^2_{21} &w^2_{22} &w^2_{23} \\
+    w^2_{31} &w^2_{32} &w^2_{33} \\
+    \end{array} \right] \cdot
+    \left[\begin{array}{c}
+           y^1_1 \\
+           y^1_2 \\
+           y^1_3 \\
+          \end{array}\right] + 
+    \left[\begin{array}{c}
+           b^2_1 \\
+           b^2_2 \\
+           b^2_3 \\
+          \end{array}\right]\right).
+\label{_auto6}
+\end{equation}
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="matrix-vector-notation-and-activation">Matrix-vector notation  and activation </h3>
+
+<p>The activation of node \( i \) in layer 2 is</p>
+
+$$
+\begin{equation}
+ y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
+ f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
+\label{_auto7}
+\end{equation}
+$$
+
+<p>This is not just a convenient and compact notation, but also a useful
+and intuitive way to think about MLPs: The output is calculated by a
+series of matrix-vector multiplications and vector additions that are
+used as input to the activation functions. For each operation
+\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="activation-functions">Activation functions  </h3>
+
+<p>A property that characterizes a neural network, other than its
+connectivity, is the choice of activation function(s).  As described
+in, the following restrictions are imposed on an activation function
+for a FFNN to fulfill the universal approximation theorem
+</p>
+
+<ul>
+  <li> Non-constant</li>
+  <li> Bounded</li>
+  <li> Monotonically-increasing</li>
+  <li> Continuous</li>
+</ul>
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="activation-functions-logistic-and-hyperbolic-ones">Activation functions, Logistic and Hyperbolic ones  </h3>
+
+<p>The second requirement excludes all linear functions. Furthermore, in
+a MLP with only linear activation functions, each layer simply
+performs a linear transformation of its inputs.
+</p>
+
+<p>Regardless of the number of layers, the output of the NN will be
+nothing but a linear function of the inputs. Thus we need to introduce
+some kind of non-linearity to the NN to be able to fit non-linear
+functions Typical examples are the logistic <em>Sigmoid</em>
+</p>
+
+$$
+ f(x) = \frac{1}{1 + e^{-x}},
+$$
+
+<p>and the <em>hyperbolic tangent</em> function</p>
+$$
+ f(x) = \tanh(x)
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h3 id="relevance">Relevance </h3>
+
+<p>The <em>sigmoid</em> function are more biologically plausible because the
+output of inactive neurons are zero. Such activation function are
+called <em>one-sided</em>. However, it has been shown that the hyperbolic
+tangent performs better than the sigmoid for training MLPs.  has
+become the most popular for <em>deep neural networks</em>
+</p>
+
+
+<!-- code=python (!bc pycod) typeset with pygments style "default" -->
+<div class="cell border-box-sizing code_cell rendered">
+  <div class="input">
+    <div class="inner_cell">
+      <div class="input_area">
+        <div class="highlight" style="background: #f8f8f8">
+  <pre style="line-height: 125%;"><span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;The sigmoid function (or the logistic curve) is a </span>
+<span style="color: #BA2121; font-style: italic">function that takes any real number, z, and outputs a number (0,1).</span>
+<span style="color: #BA2121; font-style: italic">It is useful in neural networks for assigning weights on a relative scale.</span>
+<span style="color: #BA2121; font-style: italic">The value z is the weighted sum of parameters involved in the learning algorithm.&quot;&quot;&quot;</span>
+
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
+<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">mt</span>
+
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.1</span>)
+sigma_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1/</span>(<span style="color: #666666">1+</span>numpy<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z)))
+sigma <span style="color: #666666">=</span> sigma_fn(z)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, sigma)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.1</span>, <span style="color: #666666">1.1</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;sigmoid function&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Step Function&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-5</span>, <span style="color: #666666">5</span>, <span style="color: #666666">.02</span>)
+step_fn <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>vectorize(<span style="color: #008000; font-weight: bold">lambda</span> z: <span style="color: #666666">1.0</span> <span style="color: #008000; font-weight: bold">if</span> z <span style="color: #666666">&gt;=</span> <span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">else</span> <span style="color: #666666">0.0</span>)
+step <span style="color: #666666">=</span> step_fn(z)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, step)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-0.5</span>, <span style="color: #666666">1.5</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-5</span>,<span style="color: #666666">5</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;step function&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Sine Function&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi, <span style="color: #666666">0.1</span>)
+t <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>sin(z)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, t)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2*</span>mt<span style="color: #666666">.</span>pi,<span style="color: #666666">2*</span>mt<span style="color: #666666">.</span>pi])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;sine function&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+
+<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Plots a graph of the squashing function used by a rectified linear</span>
+<span style="color: #BA2121; font-style: italic">unit&quot;&quot;&quot;</span>
+z <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>arange(<span style="color: #666666">-2</span>, <span style="color: #666666">2</span>, <span style="color: #666666">.1</span>)
+zero <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>zeros(<span style="color: #008000">len</span>(z))
+y <span style="color: #666666">=</span> numpy<span style="color: #666666">.</span>max([zero, z], axis<span style="color: #666666">=0</span>)
+
+fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
+ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
+ax<span style="color: #666666">.</span>plot(z, y)
+ax<span style="color: #666666">.</span>set_ylim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
+ax<span style="color: #666666">.</span>set_xlim([<span style="color: #666666">-2.0</span>, <span style="color: #666666">2.0</span>])
+ax<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">True</span>)
+ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;z&#39;</span>)
+ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&#39;Rectified linear unit&#39;</span>)
+
+plt<span style="color: #666666">.</span>show()
+</pre>
+</div>
+      </div>
+    </div>
+  </div>
+  <div class="output_wrapper">
+    <div class="output">
+      <div class="output_area">
+        <div class="output_subarea output_stream output_stdout output_text">          
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="the-multilayer-perceptron-mlp">The multilayer  perceptron (MLP) </h2>
+
+<p>The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of</p>
+<ol>
+<li> A neural network with one or more layers of nodes between the input and the output nodes.</li>
+<li> The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.</li>
+<li> The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.</li>
+</ol>
+<p>As a convention it is normal to call  a  network with one layer of input units, one layer of hidden
+units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
+</p>
+
+<p>For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.
+Hereafter we will call the various entities of a layer for nodes.
+There are also no connections within a single layer.
+</p>
+
+<p>The number of input nodes does not need to equal the number of output
+nodes. This applies also to the hidden layers. Each layer may have its
+own number of nodes and activation functions.
+</p>
+
+<p>The hidden layers have their name from the fact that they are not
+linked to observables and as we will see below when we define the
+so-called activation \( \hat{z} \), we can think of this as a basis
+expansion of the original inputs \( \hat{x} \). The difference however
+between neural networks and say linear regression is that now these
+basis functions (which will correspond to the weights in the network)
+are learned from data. This results in  an important difference between
+neural networks and deep learning approaches on one side and methods
+like logistic regression or linear regression and their modifications on the other side.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="from-one-to-many-layers-the-universal-approximation-theorem">From one to many layers, the universal approximation theorem </h2>
+
+<p>A neural network with only one layer, what we called the simple
+perceptron, is best suited if we have a standard binary model with
+clear (linear) boundaries between the outcomes. As such it could
+equally well be replaced by standard linear regression or logistic
+regression. Networks with one or more hidden layers approximate
+systems with more complex boundaries.
+</p>
+
+<p>As stated earlier, 
+an important theorem in studies of neural networks, restated without
+proof here, is the <a href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf" target="_blank">universal approximation
+theorem</a>.
+</p>
+
+<p>It states that a feed-forward network with a single hidden layer
+containing a finite number of neurons can approximate continuous
+functions on compact subsets of real functions. The theorem thus
+states that simple neural networks can represent a wide variety of
+interesting functions when given appropriate parameters. It is the
+multilayer feedforward architecture itself which gives neural networks
+the potential of being universal approximators.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="deriving-the-back-propagation-code-for-a-multilayer-perceptron-model">Deriving the back propagation code for a multilayer perceptron model </h2>
+
+<p>As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.
+The unknowwn quantities are our weights \( w_{ij} \) and we need to find an algorithm for changing them so that our errors are as small as possible.
+This leads us to the famous <a href="https://www.nature.com/articles/323533a0" target="_blank">back propagation algorithm</a>.
+</p>
+
+<p>The questions we want to ask are how do changes in the biases and the
+weights in our network change the cost function and how can we use the
+final output to modify the weights?
+</p>
+
+<p>To derive these equations let us start with a plain regression problem
+and define our cost function as
+</p>
+
+$$
+{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2, 
+$$
+
+<p>where the $t_i$s are our \( n \) targets (the values we want to
+reproduce), while the outputs of the network after having propagated
+all inputs \( \hat{x} \) are given by \( y_i \).  Below we will demonstrate
+how the basic equations arising from the back propagation algorithm
+can be modified in order to study classification problems with \( K \)
+classes.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="definitions">Definitions </h2>
+
+<p>With our definition of the targets \( \hat{t} \), the outputs of the
+network \( \hat{y} \) and the inputs \( \hat{x} \) we
+define now the activation \( z_j^l \) of node/neuron/unit \( j \) of the
+\( l \)-th layer as a function of the bias, the weights which add up from
+the previous layer \( l-1 \) and the forward passes/outputs
+\( \hat{a}^{l-1} \) from the previous layer as
+</p>
+
+$$
+z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,
+$$
+
+<p>where \( b_k^l \) are the biases from layer \( l \).  Here \( M_{l-1} \)
+represents the total number of nodes/neurons/units of layer \( l-1 \). The
+figure here illustrates this equation.  We can rewrite this in a more
+compact form as the matrix-vector products we discussed earlier,
+</p>
+
+$$
+\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
+$$
+
+<p>With the activation values \( \hat{z}^l \) we can in turn define the
+output of layer \( l \) as \( \hat{a}^l = f(\hat{z}^l) \) where \( f \) is our
+activation function. In the examples here we will use the sigmoid
+function discussed in our logistic regression lectures. We will also use the same activation function \( f \) for all layers
+and their nodes.  It means we have
+</p>
+
+$$
+a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivatives-and-the-chain-rule">Derivatives and the chain rule </h2>
+
+<p>From the definition of the activation \( z_j^l \) we have</p>
+$$
+\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
+$$
+
+<p>and</p>
+$$
+\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. 
+$$
+
+<p>With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))</p>
+$$
+\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)). 
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivative-of-the-cost-function">Derivative of the cost function </h2>
+
+<p>With these definitions we can now compute the derivative of the cost function in terms of the weights.</p>
+
+<p>Let us specialize to the output layer \( l=L \). Our cost function is</p>
+$$
+{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2, 
+$$
+
+<p>The derivative of this function with respect to the weights is</p>
+
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}}, 
+$$
+
+<p>The last partial derivative can easily be computed and reads (by applying the chain rule)</p>
+$$
+\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},  
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="bringing-it-together-first-back-propagation-equation">Bringing it together, first back propagation equation </h2>
+
+<p>We have thus</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1}, 
+$$
+
+<p>Defining</p>
+$$
+\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+$$
+
+<p>and using the Hadamard product of two vectors we can write this as</p>
+$$
+\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
+$$
+
+<p>This is an important expression. The second term on the right handside
+measures how fast the cost function is changing as a function of the $j$th
+output activation.  If, for example, the cost function doesn't depend
+much on a particular output node \( j \), then \( \delta_j^L \) will be small,
+which is what we would expect. The first term on the right, measures
+how fast the activation function \( f \) is changing at a given activation
+value \( z_j^L \).
+</p>
+
+<p>Notice that everything in the above equations is easily computed.  In
+particular, we compute \( z_j^L \) while computing the behaviour of the
+network, and it is only a small additional overhead to compute
+\( f'(z^L_j) \).  The exact form of the derivative with respect to the
+output depends on the form of the cost function.
+However, provided the cost function is known there should be little
+trouble in calculating
+</p>
+
+$$
+\frac{\partial {\cal C}}{\partial (a_j^L)}
+$$
+
+<p>With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely</p>
+$$
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
+$$
+
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="derivatives-in-terms-of-z-j-l">Derivatives in terms of \( z_j^L \) </h2>
+
+<p>It is also easy to see that our previous equation can be written as</p>
+
+$$
+\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
+$$
+
+<p>which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely</p>
+$$
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
+$$
+
+<p>That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias. </p>
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="bringing-it-together">Bringing it together </h2>
+
+<p>We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b>The starting equations</b>
+<p>
+
+$$
+\begin{equation}
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
+\label{_auto8}
+\end{equation}
+$$
+
+<p>and</p>
+$$
+\begin{equation}
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\label{_auto9}
+\end{equation}
+$$
+
+<p>and</p>
+
+$$
+\begin{equation}
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
+\label{_auto10}
+\end{equation}
+$$
+</div>
+
+
+<p>An interesting consequence of the above equations is that when the
+activation \( a_k^{L-1} \) is small, the gradient term, that is the
+derivative of the cost function with respect to the weights, will also
+tend to be small. We say then that the weight learns slowly, meaning
+that it changes slowly when we minimize the weights via say gradient
+descent. In this case we say the system learns slowly.
+</p>
+
+<p>Another interesting feature is that is when the activation function,
+represented by the sigmoid function here, is rather flat when we move towards
+its end values \( 0 \) and \( 1 \) (see the above Python codes). In these
+cases, the derivatives of the activation function will also be close
+to zero, meaning again that the gradients will be small and the
+network learns slowly again.
+</p>
+
+<p>We need a fourth equation and we are set. We are going to propagate
+backwards in order to the determine the weights and biases. In order
+to do so we need to represent the error in the layer before the final
+one \( L-1 \) in terms of the errors in the final output layer.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="final-back-propagating-equation">Final back propagating equation </h2>
+
+<p>We have that (replacing \( L \) with a general layer \( l \))</p>
+$$
+\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
+$$
+
+<p>We want to express this in terms of the equations for layer \( l+1 \). Using the chain rule and summing over all \( k \) entries we have</p>
+
+$$
+\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
+$$
+
+<p>and recalling that</p>
+$$
+z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
+$$
+
+<p>with \( M_l \) being the number of nodes in layer \( l \), we obtain</p>
+$$
+\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
+$$
+
+<p>This is our final equation.</p>
+
+<p>We are now ready to set up the algorithm for back propagation and learning the weights and biases.</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \hat{x} \) and the activations
+\( \hat{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \hat{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \hat{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \hat{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \hat{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
 </p>
 
 <!-- !split --><br><br><br><br><br><br><br><br><br><br>
 <h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
 
-<p>The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+<p>The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>First, we set up the input data \( \boldsymbol{x} \) and the activations
+\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and
+the pertinent outputs \( \boldsymbol{a}^1 \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Secondly, we perform then the feed forward till we reach the output
+layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the
+activation function and the pertinent outputs \( \boldsymbol{a}^l \) for
+\( l=2,3,\dots,L \).
+</p>
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all</p>
+$$
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as</p>
+$$
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+$$
+</div>
+
+
+<div class="alert alert-block alert-block alert-text-normal">
+<b></b>
+<p>
+<p>Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules</p>
+$$
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+$$
+
+
+$$
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+$$
+</div>
+
+
+<p>The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+</p>
+
+<!-- !split --><br><br><br><br><br><br><br><br><br><br>
+<h2 id="setting-up-the-back-propagation-algorithm">Setting up the Back propagation algorithm </h2>
+
+<p>The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.</p>
 
 <div class="alert alert-block alert-block alert-text-normal">
 <b></b>
@@ -1894,876 +3094,6 @@ plt<span style="color: #666666">.</span>show()
 </div>
 
 
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="building-neural-networks-in-tensorflow-and-keras">Building neural networks in Tensorflow and Keras </h2>
-
-<p>Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn
-and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
-and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  
-</p>
-
-<p>In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
-clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
-NumPy arrays.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="tensorflow">Tensorflow </h2>
-
-<p>Tensorflow is an open source library machine learning library
-developed by the Google Brain team for internal use. It was released
-under the Apache 2.0 open source license in November 9, 2015.
-</p>
-
-<p>Tensorflow is a computational framework that allows you to construct
-machine learning models at different levels of abstraction, from
-high-level, object-oriented APIs like Keras, down to the C++ kernels
-that Tensorflow is built upon. The higher levels of abstraction are
-simpler to use, but less flexible, and our choice of implementation
-should reflect the problems we are trying to solve.
-</p>
-
-<p><a href="https://www.tensorflow.org/guide/graphs" target="_blank">Tensorflow uses</a> so-called graphs to represent your computation
-in terms of the dependencies between individual operations, such that you first build a Tensorflow <em>graph</em>
-to represent your model, and then create a Tensorflow <em>session</em> to run the graph.
-</p>
-
-<p>In this guide we will analyze the same data as we did in our NumPy and
-scikit-learn tutorial, gathered from the MNIST database of images. We
-will give an introduction to the lower level Python Application
-Program Interfaces (APIs), and see how we use them to build our graph.
-Then we will build (effectively) the same graph in Keras, to see just
-how simple solving a machine learning problem can be.
-</p>
-
-<p>To install tensorflow on Unix/Linux systems, use pip as</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">pip3 install tensorflow
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>and/or if you use <b>anaconda</b>, just write (or install from the graphical user interface)
-(current release of CPU-only TensorFlow)
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda create <span style="color: #666666">-</span>n tf tensorflow
-conda activate tf
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>To install the current release of GPU TensorFlow</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda create <span style="color: #666666">-</span>n tf<span style="color: #666666">-</span>gpu tensorflow<span style="color: #666666">-</span>gpu
-conda activate tf<span style="color: #666666">-</span>gpu
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="using-keras">Using Keras </h2>
-
-<p>Keras is a high level <a href="https://en.wikipedia.org/wiki/Application_programming_interface" target="_blank">neural network</a>
-that supports Tensorflow, CTNK and Theano as backends.  
-If you have Anaconda installed you may run the following command
-</p>
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">conda install keras
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<p>You can look up the <a href="https://keras.io/" target="_blank">instructions here</a> for more information.</p>
-
-<p>We will to a large extent use <b>keras</b> in this course. </p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="collect-and-pre-process-data">Collect and pre-process data </h2>
-
-<p>Let us look again at the MINST data set.</p>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># import necessary packages</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
-
-
-<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
-
-<span style="color: #408080; font-style: italic"># display images in notebook</span>
-<span style="color: #666666">%</span>matplotlib inline
-plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">&#39;figure.figsize&#39;</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
-
-
-<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
-digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
-
-<span style="color: #408080; font-style: italic"># define inputs and labels</span>
-inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
-labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs = (n_inputs, pixel_width, pixel_height) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels = (n_inputs) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># flatten the image</span>
-<span style="color: #408080; font-style: italic"># the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64</span>
-n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
-inputs <span style="color: #666666">=</span> inputs<span style="color: #666666">.</span>reshape(n_inputs, <span style="color: #666666">-1</span>)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;X = (n_inputs, n_features) = &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-
-
-<span style="color: #408080; font-style: italic"># choose some random images to display</span>
-indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
-random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
-
-<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
-    plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
-    plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">&#39;off&#39;</span>)
-    plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>)
-    plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">&quot;Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential      <span style="color: #408080; font-style: italic">#This allows appending layers to existing models</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense           <span style="color: #408080; font-style: italic">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> optimizers             <span style="color: #408080; font-style: italic">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers           <span style="color: #408080; font-style: italic">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical   <span style="color: #408080; font-style: italic">#This allows using categorical cross entropy as the cost function</span>
-
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
-
-<span style="color: #408080; font-style: italic"># one-hot representation of labels</span>
-labels <span style="color: #666666">=</span> to_categorical(labels)
-
-<span style="color: #408080; font-style: italic"># split into train and test data</span>
-train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
-test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
-X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
-                                                    test_size<span style="color: #666666">=</span>test_size)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-n_neurons_layer1 <span style="color: #666666">=</span> <span style="color: #666666">100</span>
-n_neurons_layer2 <span style="color: #666666">=</span> <span style="color: #666666">50</span>
-n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
-eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_neural_network_keras</span>(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
-    model <span style="color: #666666">=</span> Sequential()
-    model<span style="color: #666666">.</span>add(Dense(n_neurons_layer1, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;sigmoid&#39;</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
-    model<span style="color: #666666">.</span>add(Dense(n_neurons_layer2, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;sigmoid&#39;</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
-    model<span style="color: #666666">.</span>add(Dense(n_categories, activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;softmax&#39;</span>))
-    
-    sgd <span style="color: #666666">=</span> optimizers<span style="color: #666666">.</span>SGD(lr<span style="color: #666666">=</span>eta)
-    model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">&#39;categorical_crossentropy&#39;</span>, optimizer<span style="color: #666666">=</span>sgd, metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">&#39;accuracy&#39;</span>])
-    
-    <span style="color: #008000; font-weight: bold">return</span> model
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;">DNN_keras <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
-        
-<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
-    <span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
-        DNN <span style="color: #666666">=</span> create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
-                                         eta<span style="color: #666666">=</span>eta, lmbd<span style="color: #666666">=</span>lmbd)
-        DNN<span style="color: #666666">.</span>fit(X_train, Y_train, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size, verbose<span style="color: #666666">=0</span>)
-        scores <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)
-        
-        DNN_keras[i][j] <span style="color: #666666">=</span> DNN
-        
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Learning rate = &quot;</span>, eta)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Lambda = &quot;</span>, lmbd)
-        <span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> scores[<span style="color: #666666">1</span>])
-        <span style="color: #008000">print</span>()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># optional</span>
-<span style="color: #408080; font-style: italic"># visual representation of grid search</span>
-<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
-
-sns<span style="color: #666666">.</span>set()
-
-train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
-
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
-        DNN <span style="color: #666666">=</span> DNN_keras[i][j]
-
-        train_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_train, Y_train)[<span style="color: #666666">1</span>]
-        test_accuracy[i][j] <span style="color: #666666">=</span> DNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)[<span style="color: #666666">1</span>]
-
-        
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Training Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-
-fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
-sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">&quot;viridis&quot;</span>)
-ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">&quot;Test Accuracy&quot;</span>)
-ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&quot;$\eta$&quot;</span>)
-ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&quot;$\lambda$&quot;</span>)
-plt<span style="color: #666666">.</span>show()
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-breast-cancer-data-now-with-keras">The Breast Cancer Data, now with Keras </h2>
-
-
-<!-- code=python (!bc pycod) typeset with pygments style "default" -->
-<div class="cell border-box-sizing code_cell rendered">
-  <div class="input">
-    <div class="inner_cell">
-      <div class="input_area">
-        <div class="highlight" style="background: #f8f8f8">
-  <pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential      <span style="color: #408080; font-style: italic">#This allows appending layers to existing models</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense           <span style="color: #408080; font-style: italic">#This allows defining the characteristics of a particular layer</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> optimizers             <span style="color: #408080; font-style: italic">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers           <span style="color: #408080; font-style: italic">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical   <span style="color: #408080; font-style: italic">#This allows using categorical cross entropy as the cost function</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split <span style="color: #008000; font-weight: bold">as</span> splitter
-<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pickle</span>
-<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span> 
-
-
-<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Load breast cancer dataset&quot;&quot;&quot;</span>
-
-np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)        <span style="color: #408080; font-style: italic">#create same seed for random number every time</span>
-
-cancer<span style="color: #666666">=</span>load_breast_cancer()      <span style="color: #408080; font-style: italic">#Download breast cancer dataset</span>
-
-inputs<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>data                     <span style="color: #408080; font-style: italic">#Feature matrix of 569 rows (samples) and 30 columns (parameters)</span>
-outputs<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>target                  <span style="color: #408080; font-style: italic">#Label array of 569 rows (0 for benign and 1 for malignant)</span>
-labels<span style="color: #666666">=</span>cancer<span style="color: #666666">.</span>feature_names[<span style="color: #666666">0</span>:<span style="color: #666666">30</span>]
-
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;The content of the breast cancer dataset is:&#39;</span>)      <span style="color: #408080; font-style: italic">#Print information about the datasets</span>
-<span style="color: #008000">print</span>(labels)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;-------------------------&#39;</span>)
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;inputs =  &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;outputs =  &quot;</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(outputs<span style="color: #666666">.</span>shape))
-<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;labels =  &quot;</span><span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
-
-x<span style="color: #666666">=</span>inputs      <span style="color: #408080; font-style: italic">#Reassign the Feature and Label matrices to other variables</span>
-y<span style="color: #666666">=</span>outputs
-
-<span style="color: #408080; font-style: italic">#%% </span>
-
-<span style="color: #408080; font-style: italic"># Visualisation of dataset (for correlation analysis)</span>
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">0</span>],x[:,<span style="color: #666666">2</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean radius&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean perimeter&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">5</span>],x[:,<span style="color: #666666">6</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean compactness&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean concavity&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">0</span>],x[:,<span style="color: #666666">1</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean radius&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean texture&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-plt<span style="color: #666666">.</span>figure()
-plt<span style="color: #666666">.</span>scatter(x[:,<span style="color: #666666">2</span>],x[:,<span style="color: #666666">1</span>],s<span style="color: #666666">=40</span>,c<span style="color: #666666">=</span>y,cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>Spectral)
-plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">&#39;Mean perimeter&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">&#39;Mean compactness&#39;</span>,fontweight<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bold&#39;</span>)
-plt<span style="color: #666666">.</span>show()
-
-
-<span style="color: #408080; font-style: italic"># Generate training and testing datasets</span>
-
-<span style="color: #408080; font-style: italic">#Select features relevant to classification (texture,perimeter,compactness and symmetery) </span>
-<span style="color: #408080; font-style: italic">#and add to input matrix</span>
-
-temp1<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">1</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">1</span>]),<span style="color: #666666">1</span>))
-temp2<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">2</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">2</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((temp1,temp2))      
-temp<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">5</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">5</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((X,temp))       
-temp<span style="color: #666666">=</span>np<span style="color: #666666">.</span>reshape(x[:,<span style="color: #666666">8</span>],(<span style="color: #008000">len</span>(x[:,<span style="color: #666666">8</span>]),<span style="color: #666666">1</span>))
-X<span style="color: #666666">=</span>np<span style="color: #666666">.</span>hstack((X,temp))       
-
-X_train,X_test,y_train,y_test<span style="color: #666666">=</span>splitter(X,y,test_size<span style="color: #666666">=0.1</span>)   <span style="color: #408080; font-style: italic">#Split datasets into training and testing</span>
-
-y_train<span style="color: #666666">=</span>to_categorical(y_train)     <span style="color: #408080; font-style: italic">#Convert labels to categorical when using categorical cross entropy</span>
-y_test<span style="color: #666666">=</span>to_categorical(y_test)
-
-<span style="color: #008000; font-weight: bold">del</span> temp1,temp2,temp
-
-<span style="color: #408080; font-style: italic"># %%</span>
-
-<span style="color: #408080; font-style: italic"># Define tunable parameters&quot;</span>
-
-eta<span style="color: #666666">=</span>np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-3</span>,<span style="color: #666666">-1</span>,<span style="color: #666666">3</span>)                    <span style="color: #408080; font-style: italic">#Define vector of learning rates (parameter to SGD optimiser)</span>
-lamda<span style="color: #666666">=0.01</span>                                  <span style="color: #408080; font-style: italic">#Define hyperparameter</span>
-n_layers<span style="color: #666666">=2</span>                                  <span style="color: #408080; font-style: italic">#Define number of hidden layers in the model</span>
-n_neuron<span style="color: #666666">=</span>np<span style="color: #666666">.</span>logspace(<span style="color: #666666">0</span>,<span style="color: #666666">3</span>,<span style="color: #666666">4</span>,dtype<span style="color: #666666">=</span><span style="color: #008000">int</span>)       <span style="color: #408080; font-style: italic">#Define number of neurons per layer</span>
-epochs<span style="color: #666666">=100</span>                                   <span style="color: #408080; font-style: italic">#Number of reiterations over the input data</span>
-batch_size<span style="color: #666666">=100</span>                              <span style="color: #408080; font-style: italic">#Number of samples per gradient update</span>
-
-<span style="color: #408080; font-style: italic"># %%</span>
-
-<span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;Define function to return Deep Neural Network model&quot;&quot;&quot;</span>
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">NN_model</span>(inputsize,n_layers,n_neuron,eta,lamda):
-    model<span style="color: #666666">=</span>Sequential()      
-    <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_layers):       <span style="color: #408080; font-style: italic">#Run loop to add hidden layers to the model</span>
-        <span style="color: #008000; font-weight: bold">if</span> (i<span style="color: #666666">==0</span>):                  <span style="color: #408080; font-style: italic">#First layer requires input dimensions</span>
-            model<span style="color: #666666">.</span>add(Dense(n_neuron,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;relu&#39;</span>,kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lamda),input_dim<span style="color: #666666">=</span>inputsize))
-        <span style="color: #008000; font-weight: bold">else</span>:                       <span style="color: #408080; font-style: italic">#Subsequent layers are capable of automatic shape inferencing</span>
-            model<span style="color: #666666">.</span>add(Dense(n_neuron,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;relu&#39;</span>,kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lamda)))
-    model<span style="color: #666666">.</span>add(Dense(<span style="color: #666666">2</span>,activation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;softmax&#39;</span>))  <span style="color: #408080; font-style: italic">#2 outputs - ordered and disordered (softmax for prob)</span>
-    sgd<span style="color: #666666">=</span>optimizers<span style="color: #666666">.</span>SGD(lr<span style="color: #666666">=</span>eta)
-    model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">&#39;categorical_crossentropy&#39;</span>,optimizer<span style="color: #666666">=</span>sgd,metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">&#39;accuracy&#39;</span>])
-    <span style="color: #008000; font-weight: bold">return</span> model
-
-    
-Train_accuracy<span style="color: #666666">=</span>np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(n_neuron),<span style="color: #008000">len</span>(eta)))      <span style="color: #408080; font-style: italic">#Define matrices to store accuracy scores as a function</span>
-Test_accuracy<span style="color: #666666">=</span>np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(n_neuron),<span style="color: #008000">len</span>(eta)))       <span style="color: #408080; font-style: italic">#of learning rate and number of hidden neurons for </span>
-
-<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(n_neuron)):     <span style="color: #408080; font-style: italic">#run loops over hidden neurons and learning rates to calculate </span>
-    <span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta)):      <span style="color: #408080; font-style: italic">#accuracy scores </span>
-        DNN_model<span style="color: #666666">=</span>NN_model(X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>],n_layers,n_neuron[i],eta[j],lamda)
-        DNN_model<span style="color: #666666">.</span>fit(X_train,y_train,epochs<span style="color: #666666">=</span>epochs,batch_size<span style="color: #666666">=</span>batch_size,verbose<span style="color: #666666">=1</span>)
-        Train_accuracy[i,j]<span style="color: #666666">=</span>DNN_model<span style="color: #666666">.</span>evaluate(X_train,y_train)[<span style="color: #666666">1</span>]
-        Test_accuracy[i,j]<span style="color: #666666">=</span>DNN_model<span style="color: #666666">.</span>evaluate(X_test,y_test)[<span style="color: #666666">1</span>]
-               
-
-<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_data</span>(x,y,data,title<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>):
-
-    <span style="color: #408080; font-style: italic"># plot results</span>
-    fontsize<span style="color: #666666">=16</span>
-
-
-    fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure()
-    ax <span style="color: #666666">=</span> fig<span style="color: #666666">.</span>add_subplot(<span style="color: #666666">111</span>)
-    cax <span style="color: #666666">=</span> ax<span style="color: #666666">.</span>matshow(data, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">&#39;nearest&#39;</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=1</span>)
-    
-    cbar<span style="color: #666666">=</span>fig<span style="color: #666666">.</span>colorbar(cax)
-    cbar<span style="color: #666666">.</span>ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&#39;accuracy (%)&#39;</span>,rotation<span style="color: #666666">=90</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    cbar<span style="color: #666666">.</span>set_ticks([<span style="color: #666666">0</span>,<span style="color: #666666">.2</span>,<span style="color: #666666">.4</span>,<span style="color: #666666">0.6</span>,<span style="color: #666666">0.8</span>,<span style="color: #666666">1.0</span>])
-    cbar<span style="color: #666666">.</span>set_ticklabels([<span style="color: #BA2121">&#39;0%&#39;</span>,<span style="color: #BA2121">&#39;20%&#39;</span>,<span style="color: #BA2121">&#39;40%&#39;</span>,<span style="color: #BA2121">&#39;60%&#39;</span>,<span style="color: #BA2121">&#39;80%&#39;</span>,<span style="color: #BA2121">&#39;100%&#39;</span>])
-
-    <span style="color: #408080; font-style: italic"># put text on matrix elements</span>
-    <span style="color: #008000; font-weight: bold">for</span> i, x_val <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(np<span style="color: #666666">.</span>arange(<span style="color: #008000">len</span>(x))):
-        <span style="color: #008000; font-weight: bold">for</span> j, y_val <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(np<span style="color: #666666">.</span>arange(<span style="color: #008000">len</span>(y))):
-            c <span style="color: #666666">=</span> <span style="color: #BA2121">&quot;$</span><span style="color: #BB6688; font-weight: bold">{0:.1f}</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">%$&quot;</span><span style="color: #666666">.</span>format( <span style="color: #666666">100*</span>data[j,i])  
-            ax<span style="color: #666666">.</span>text(x_val, y_val, c, va<span style="color: #666666">=</span><span style="color: #BA2121">&#39;center&#39;</span>, ha<span style="color: #666666">=</span><span style="color: #BA2121">&#39;center&#39;</span>)
-
-    <span style="color: #408080; font-style: italic"># convert axis vaues to to string labels</span>
-    x<span style="color: #666666">=</span>[<span style="color: #008000">str</span>(i) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> x]
-    y<span style="color: #666666">=</span>[<span style="color: #008000">str</span>(i) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> y]
-
-
-    ax<span style="color: #666666">.</span>set_xticklabels([<span style="color: #BA2121">&#39;&#39;</span>]<span style="color: #666666">+</span>x)
-    ax<span style="color: #666666">.</span>set_yticklabels([<span style="color: #BA2121">&#39;&#39;</span>]<span style="color: #666666">+</span>y)
-
-    ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">&#39;$</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">mathrm{learning</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121"> rate}$&#39;</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">&#39;$</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121">mathrm{hidden</span><span style="color: #BB6622; font-weight: bold">\\</span><span style="color: #BA2121"> neurons}$&#39;</span>,fontsize<span style="color: #666666">=</span>fontsize)
-    <span style="color: #008000; font-weight: bold">if</span> title <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
-        ax<span style="color: #666666">.</span>set_title(title)
-
-    plt<span style="color: #666666">.</span>tight_layout()
-
-    plt<span style="color: #666666">.</span>show()
-    
-plot_data(eta,n_neuron,Train_accuracy, <span style="color: #BA2121">&#39;training&#39;</span>)
-plot_data(eta,n_neuron,Test_accuracy, <span style="color: #BA2121">&#39;testing&#39;</span>)
-</pre>
-</div>
-      </div>
-    </div>
-  </div>
-  <div class="output_wrapper">
-    <div class="output">
-      <div class="output_area">
-        <div class="output_subarea output_stream output_stdout output_text">          
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="fine-tuning-neural-network-hyperparameters">Fine-tuning neural network hyperparameters </h2>
-
-<p>The flexibility of neural networks is also one of their main
-drawbacks: there are many hyperparameters to tweak. Not only can you
-use any imaginable network topology (how neurons/nodes are interconnected),
-but even in a simple FFNN you can change the number of layers, the
-number of neurons per layer, the type of activation function to use in
-each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
-know what combination of hyperparameters is the best for your task?
-</p>
-
-<ul>
-<li> You can use grid search with cross-validation to find the right hyperparameters.</li>
-</ul>
-<p>However,since there are many hyperparameters to tune, and since
-training a neural network on a large dataset takes a lot of time, you
-will only be able to explore a tiny part of the hyperparameter space.
-</p>
-
-<ul>
-<li> You can use randomized search.</li>
-<li> Or use tools like <a href="http://oscar.calldesk.ai/" target="_blank">Oscar</a>, which implements more complex algorithms to help you find a good set of hyperparameters quickly.</li>  
-</ul>
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="hidden-layers">Hidden layers </h2>
-
-<p>For many problems you can start with just one or two hidden layers and it will work just fine.
-For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
-few hundred neurons.
-You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
-neurons, in roughly the same amount of training time. 
-</p>
-
-<p>For more complex problems, you can gradually
-ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
-as large image classification or speech recognition, typically require networks with dozens of layers
-and they need a huge amount
-of training data. However, you will rarely have to train such networks from scratch: it is much more
-common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
-</p>
-
-<!-- !split  -->
-<h2 id="which-activation-function-should-i-use">Which activation function should I use? </h2>
-
-<p>The Back propagation algorithm we derived above works by going from
-the output layer to the input layer, propagating the error gradient on
-the way. Once the algorithm has computed the gradient of the cost
-function with regards to each parameter in the network, it uses these
-gradients to update each parameter with a Gradient Descent (GD) step.
-</p>
-
-<p>Unfortunately for us, the gradients often get smaller and smaller as the
-algorithm progresses down to the first hidden layers. As a result, the
-GD update leaves the lower layer connection weights
-virtually unchanged, and training never converges to a good
-solution. This is known in the literature as 
-<b>the vanishing gradients problem</b>. 
-</p>
-
-<p>In other cases, the opposite can happen, namely the the gradients can grow bigger and
-bigger. The result is that many of the layers get large updates of the 
-weights the
-algorithm diverges. This is the <b>exploding gradients problem</b>, which is
-mostly encountered in recurrent neural networks. More generally, deep
-neural networks suffer from unstable gradients, different layers may
-learn at widely different speeds
-</p>
-
-<!-- !split  -->
-<h2 id="is-the-logistic-activation-function-sigmoid-our-choice">Is the Logistic activation function (Sigmoid)  our choice? </h2>
-
-<p>Although this unfortunate behavior has been empirically observed for
-quite a while (it was one of the reasons why deep neural networks were
-mostly abandoned for a long time), it is only around 2010 that
-significant progress was made in understanding it.
-</p>
-
-<p>A paper titled <a href="http://proceedings.mlr.press/v9/glorot10a.html" target="_blank">Understanding the Difficulty of Training Deep
-Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio</a> found that
-the problems with the popular logistic
-sigmoid activation function and the weight initialization technique
-that was most popular at the time, namely random initialization using
-a normal distribution with a mean of 0 and a standard deviation of
-1. 
-</p>
-
-<p>They showed that with this activation function and this
-initialization scheme, the variance of the outputs of each layer is
-much greater than the variance of its inputs. Going forward in the
-network, the variance keeps increasing after each layer until the
-activation function saturates at the top layers. This is actually made
-worse by the fact that the logistic function has a mean of 0.5, not 0
-(the hyperbolic tangent function has a mean of 0 and behaves slightly
-better than the logistic function in deep networks).
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-derivative-of-the-logistic-funtion">The derivative of the Logistic funtion </h2>
-
-<p>Looking at the logistic activation function, when inputs become large
-(negative or positive), the function saturates at 0 or 1, with a
-derivative extremely close to 0. Thus when backpropagation kicks in,
-it has virtually no gradient to propagate back through the network,
-and what little gradient exists keeps getting diluted as
-backpropagation progresses down through the top layers, so there is
-really nothing left for the lower layers.
-</p>
-
-<p>In their paper, Glorot and Bengio propose a way to significantly
-alleviate this problem. We need the signal to flow properly in both
-directions: in the forward direction when making predictions, and in
-the reverse direction when backpropagating gradients. We don&#8217;t want
-the signal to die out, nor do we want it to explode and saturate. For
-the signal to flow properly, the authors argue that we need the
-variance of the outputs of each layer to be equal to the variance of
-its inputs, and we also need the gradients to have equal variance
-before and after flowing through a layer in the reverse direction.
-</p>
-
-<p>One of the insights in the 2010 paper by Glorot and Bengio was that
-the vanishing/exploding gradients problems were in part due to a poor
-choice of activation function. Until then most people had assumed that
-if Nature had chosen to use roughly sigmoid activation functions in
-biological neurons, they must be an excellent choice. But it turns out
-that other activation functions behave much better in deep neural
-networks, in particular the ReLU activation function, mostly because
-it does not saturate for positive values (and also because it is quite
-fast to compute).
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="the-relu-function-family">The RELU function family </h2>
-
-<p>The ReLU activation function suffers from a problem known as the dying
-ReLUs: during training, some neurons effectively die, meaning they
-stop outputting anything other than 0.
-</p>
-
-<p>In some cases, you may find that half of your network&#8217;s neurons are
-dead, especially if you used a large learning rate. During training,
-if a neuron&#8217;s weights get updated such that the weighted sum of the
-neuron&#8217;s inputs is negative, it will start outputting 0. When this
-happen, the neuron is unlikely to come back to life since the gradient
-of the ReLU function is 0 when its input is negative.
-</p>
-
-<p>To solve this problem, nowadays practitioners use a  variant of the ReLU
-function, such as the leaky ReLU discussed above or the so-called
-exponential linear unit (ELU) function
-</p>
-
-$$
-ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\  z & z \ge 0.\end{array}\right. 
-$$
-
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="which-activation-function-should-we-use">Which activation function should we use? </h2>
-
-<p>In general it seems that the ELU activation function is better than
-the leaky ReLU function (and its variants), which is better than
-ReLU. ReLU performs better than \( \tanh \) which in turn performs better
-than the logistic function. 
-</p>
-
-<p>If runtime
-performance is an issue, then you may opt for the  leaky ReLU function  over the 
-ELU function If you don&#8217;t
-want to tweak yet another hyperparameter, you may just use the default
-\( \alpha \) of \( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have
-spare time and computing power, you can use cross-validation or
-bootstrap to evaluate other activation functions.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="more-on-activation-functions-output-layers">More on activation functions, output layers </h2>
-
-<p>In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).</p>
-
-<p>It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.</p>
-
-<b>For the output layer:</b>
-
-<ul>
-<li> For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).</li>
-<li> For regression tasks, you can simply use no activation function at all.</li>
-</ul>
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="batch-normalization">Batch Normalization </h2>
-
-<p>Batch Normalization
-aims to address the vanishing/exploding gradients problems, and more generally the problem that the
-distribution of each layer&#8217;s inputs changes during training, as the parameters of the previous layers change.
-</p>
-
-<p>The technique consists of adding an operation in the model just before the activation function of each
-layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
-parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
-learn the optimal scale and mean of the inputs for each layer.
-In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs&#8217; mean and
-standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
-mini-batch, from this the name batch normalization.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="dropout">Dropout </h2>
-
-<p>It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
-excluding the output neurons) has a probability \( p \) of being temporarily dropped out, meaning it will be
-entirely ignored during this training step, but it may be active during the next step.
-</p>
-
-<p>The
-hyperparameter \( p \) is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
- It is viewed as one of the most popular regularization techniques.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="gradient-clipping">Gradient Clipping </h2>
-
-<p>A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
-backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
-networks).
-</p>
-
-<p>This technique is called Gradient Clipping.</p>
-
-<p>In general however, Batch
-Normalization is preferred.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="a-very-nice-website-on-neural-networks">A very nice website on Neural Networks </h2>
-
-<p>You may find this <a href="https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false" target="_blank">website</a> very useful. </p>
-
-<!-- !split  -->
-<h2 id="a-top-down-perspective-on-neural-networks">A top-down perspective on Neural networks </h2>
-
-<p>The first thing we would like to do is divide the data into two or three
-parts. A training set, a validation or dev (development) set, and a
-test set. The test set is the data on which we want to make
-predictions. The dev set is a subset of the training data we use to
-check how well we are doing out-of-sample, after training the model on
-the training dataset. We use the validation error as a proxy for the
-test error in order to make tweaks to our model. It is crucial that we
-do not use any of the test data to train the algorithm. This is a
-cardinal sin in ML. Then:
-</p>
-
-<ul>
-<li> Estimate optimal error rate</li>
-<li> Minimize underfitting (bias) on training data set.</li>
-<li> Make sure you are not overfitting.</li>
-</ul>
-<p>If the validation and test sets are drawn from the same distributions,
-then a good performance on the validation set should lead to similarly
-good performance on the test set. 
-</p>
-
-<p>However, sometimes
-the training data and test data differ in subtle ways because, for
-example, they are collected using slightly different methods, or
-because it is cheaper to collect data in one way versus another. In
-this case, there can be a mismatch between the training and test
-data. This can lead to the neural network overfitting these small
-differences between the test and training sets, and a poor performance
-on the test set despite having a good performance on the validation
-set. To rectify this, Andrew Ng suggests making two validation or dev
-sets, one constructed from the training data and one constructed from
-the test data. The difference between the performance of the algorithm
-on these two validation sets quantifies the train-test mismatch. This
-can serve as another important diagnostic when using DNNs for
-supervised learning.
-</p>
-
-<!-- !split --><br><br><br><br><br><br><br><br><br><br>
-<h2 id="limitations-of-supervised-learning-with-deep-networks">Limitations of supervised learning with deep networks </h2>
-
-<p>Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
-</p>
-
-<p>Here we list some of the important limitations of supervised neural network based models. </p>
-
-<ul>
-<li> <b>Need labeled data</b>. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).</li>
-<li> <b>Supervised neural networks are extremely data intensive.</b> DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.</li>
-<li> <b>Homogeneous data.</b> Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.&nbsp;some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.</li>
-<li> <b>Many problems are not about prediction.</b> In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a <em>wrong</em> model. The model might or might not be useful for understanding the underlying science.</li>
-</ul>
-<p>Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.</p>
-
 <!-- ------------------- end of main content --------------- -->
 <center style="font-size:80%">
 <!-- copyright --> &copy; 1999-2023, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz
index 9e963fd57b9c86ab59a4d98f205a5b4ef44ade8c..b6a7c17e4418b5889eb9cb979a0813c5796051f1 100644
GIT binary patch
literal 34323
zcmV(tK<vLCiwFR|g(GDE1MIp5a2!FCE+}TQ$QCm*jhLC4SsL+(nVFd^vd9+8qLIZc
zS@H-imd0Xcw3z(-@9w+1*u8ylvG2uQ+-600R%K;pR##?yT@}%`4tCxiHl7^p<}NOt
z>~1c0F#qwy$;k=e<)!=w=Lhip6X)dq=k`B{i<5_+l8cv*i<6&|8^F&=$;rjX&CLlz
z$@#xvEdPeQJ-sYF{tm_2-qFUz)5hiB)BCGtYy01@^Uv=>`G4^LuMi?3G7KCHjH;zI
zrMfny0p-8YKV&ede=lwr7}&GF{BIkM;(yGfq>hf(KiB^0iPgUrr2mx)+`q{Gdf>Nq
zb!NA8a|7A1JKK0!0xi8Pg`8AnR4lD*K;~Wn0e-w7M}BAi0AFtxPe-o+FMdZ}hX7~G
ze=-BR_xC0Q14HnC+Hm)_^ssdCa&WPMxsEWrWNh+=0qD4ywEyxF&%Z0K^rJ{+!!f-7
zb)FJWZ?{o=o$?`4?}1Jc8fG+f8`ndTHoR-3+a0r^PUa%oQfm`b-SSMtKysrK{Kc>c
zvZ&LQdQ51p2r@zK)#Wf@@;JG{Xbf(&LG7%=3-8UOpC1`Oh`{mfrOj9%7o1SD@?pS(
z++7BQnu?(c`0SA^A1x?!)n@feYzGSz?fHhsm#o(+m$M`F%ot~Rm7Y!*caKoetO#D2
z$7u_*z){#5#58g4@W2{Q1q@1O&*st)7&7B#M&7Y$*cRGkCpYsRUQi8^Z6Rc3PO%qv
z?$rSC^lhc+yIGu^XI#m%n^Vz<2dzlH#lus=P`Yz)atiPW1GxCCt@v$igjFpaK*EB&
z0y5IvywYr(Qv7^uyqtpEY*M`ZvTQOk{G4)JlCr!~Qhfgr3HG1#|7NoN7yQ?-^i#00
z1lo9T{KtO>{(Jb(58(P&_z&O*{KbF2|G+W)yY&Ba{O94MboO#^wh`q52m%DS0Ro%?
z>|C6He?*F(7mbUHpHf9pN>WQ&K~YbZ-Otj?%Y*&@4r@hKWh8l313bA^b@Ut!9f8h9
zI$mmq`XEpB04Gj$SuGh=M=co@9VuzG09`LN8A(1>2QFzf8K9hsu9mD?fRwzduAH2j
ztdg{vCYm4*rS@NcDu(|pfBz2w4ULzF>%a9z|1)18Q2|#aBLQg#KcJ>0x1NHOwYQoZ
zrv{$~w+jHstH3MZB`fE`!|e@FchEL+;o;-(ceHnq(etqbX#<@l^*uGX98@@Mom35+
z^g;Rp_FjS-3i`Z)UY1@UTV-c17C$E!c@8aITT3;gzh3qARJ=6+_C~ha`hM2ZR(yIY
zT)uAB-aec%DpG=;^14di?wmSyRsbC(FCBSbO>Jpe1sfJ6B^5mdM;BEcXC-bWKMh4e
zA1Qz>7blCfjy*S@mX9F-sHf-+@YeN}m2t4uaj@Ytu#s}Gl(BK=^aiQeSqgIMdFuj|
z6+8pvyaoL11su5pT>K1p?RC`sJ*5@6jSTHQWPN!9RHgK-`2fC7>Ux}Rc8+p-PVTa9
z>N0{3T$)C1Tp*AkcYvC@qmQDsB9K>8Th~c|*V~2P1LW?ksiVhjB(3D(>tV^wqh;j>
z(DURKP?q;pa**fOR+99WlvI>N<KYqb_mbg%Dpdcu75JB}0NTG=|Nj;L|JUvQ|2z24
z#mU9}Z}6XwlauE^_y7MpC|*NN9utlDA8CQ9s37xq-wp@;dwoZN|0_9OSv~s85bPvX
zC1GHiQqf;55&yPfJ+<VeVCttx|Ivx+<0knxZIIdiR>NS-2>`W!h07W$I<kNM{K?D9
zv$eIIoSf|H>cYXnxwyFa+v(%uBO@cTwzfu1P0h{C9UmWGTU)E6qeD(kJ~T8G8yjn6
zWR#PW!^+C4q@*MwB7%#Hdv$fy($eDR=eM=BB_}5*ARvH+g*7)fS6*Ju$jGRtr}y{7
zVsCE`2M71%%NH~>G%+zTGcz+JBqSFXmk%F45EB!lpr8N%01ONaH#fJ2h6a3m{D6P}
zT3XtWkPrj}guf9J5)vXMB~49DRZ~;z>+3TyG5PuPClCk(fj~`7P0`WO8yg#1T3QAM
z26u0jL@+RvFp4sgI)1QcJ*aIrRq1T`N|R%!6Ae0hHD=_UW9)4|TvpDCp4Cp)?-v9A
z{GnJQQ-MQ3Nv;+_uJcw?ilahlji5x?^ib5_miWJv{&B&}p(XQB9_not{Y^$aGzC%t
zF67>~>peS~>yXF2arGKa`@cdz3ap^pz44rek=?i5@th`F3VCI!czC<L{-3=;k5iHb
z49o9$5b3Z!rNa|Md0tSSR^sf53k+&&Z1)Gch2gwtzpqB|rCKN8i~g&V3O_B>3n|^k
zHV?kd?(Va&IGWX((vOtU+zv~G-w1xZW<&q;__HKu^XUnhyza;APg5o>gYAA{H<*Xl
zz|X%Aq8xo*qjrdV#M|<(v09XPLjthUCR3~D+8@vDg0bJRe_c7AtUS=P>KFEq*lK%v
z%8?PzQ|h%Pc$BnyBdg7}f4d>2bpFMeY_a(e?TN@wx>H=F7!T_Tft6~_4}TE^(QMEI
z`z(3rGvrA>q?w|wK#(Bk3Hy&FpF1mT`o36c|DCoeQVqzPVA~_?Qf*86zS)vD)Rf&n
zexr~j4XjS;=CPRr!`+-boixdxTcH^M$b-!joz7z?H84GSLbKR#^(INlTc^i59fGwX
zflYz>X;K;Y80uWs^_re9$tr4p{vM7gAd!*hZ-);2^9kb@tGNfG;rS1=(${WR3G*>(
zcMB@yve$eN^{f5{r*nZ#Iq#(`k)1}%Z&{{)JgX{jc`qfG8CQ{5GRggs<72kgg5r?b
zNJo|<4OS35El=c~x*~S?GE%$YV%WnhENyw8lU_P=-W$NNE=yGeSz(4SWVsXjBpN9V
z)GI2+B5I2qNpTu-dOQxKobk^*Pu+sS2c3^N63RQaUM~R5heb?Xy$eU<CdD9A!|(l%
z{1j6|5C<|AZ=Nj$E|n#gw_-8ZfZd-(J7;AdDHB`U?<9QYKI`LH>$pUZTBfAaRcw5R
z-Ctvf-ss07*5JMkM9Fv8UwcwE96J6nT$6HIhe3}y>E+Rx*h&9O0XARKI^k(c5T?ia
zYYJtW>MCoMXk%dX(^b@G(>3U_rnHZK&QkJ7W?|NhM$m^mf9@Zw_Vrn`AK~vidotz}
z)IIr3sAkekZ6sdgHsus$biSw@@R$Gjj$5!zhb7|~)?KQw7p9P%-x>qzUdC8u%Um#x
z--v*)Wbwrm&1MTUvLTb7WfnYxTi*%t--$c}He(v)V9)RZ0Mf0@t}AHetGLJ_q>TA;
zL@>Hp%tI)RphXPE@YCaKMq$c=-qK#6oVbImK3#di_wY!;>06k4XTkQZ0xg}keK=2w
zG}CNe?z8$)F%LJMift2814NfuseBT!4J3qnt<nh>Y?`EI`NPvfus!@V$zgdncwzMs
z0hpc^97IxqD@tmB;zHR4Y);|)L*OCCO|>$#i~mB5<%PL#?DG0`i-M|v&i=%hsZtXU
zY(5uOts{=?0e2sa)!w0s5F>6Q$nd4LFY1QgeN&b+7uDr!3s7GhosL8u70Th*rE`ja
zydPGeK5Y>a4C5nphY&h<^4`M&tKFHq%PBwpn}0@;`uFe`w48Z>G7GBOJbiptvcM&!
zn>b!Ozm?tCz6z<rrkJ)9tr1{P0`9)zR3)!49v*eo&M{eLyn@uvf=aDtfw@_gNjZ-W
z>+4QGz=>dSCQXY0>vS+2;oNjNpIa0j8MpoCG+KO&sX$tP54y2-h23U9k}~5@jbs_=
z?fzVN^d;sb9;JWa(av8NP$SxzW+G%Uq;~XEBQBFv>W*+{q%_~D!{mX&_rY^kg3}M3
zZK~Q|q49f##=JB20*ye_a;2C2g8EP^n@{a7{a0?H4)?&2Z0Fy(Eu1%e*^1@okZ?XH
z0w3D(Jn!t8`bV!8=Q&|no+Ab}viFHB11qD$>i7@jn^Vxok=xf_@>O%Da_q-mlP2?j
z;`3iUb#6~L&jXA;VQc)f$GT#oG&PFkigc#nIQAc4tH>cdoYXLHKMYB^e|_;|%XQq{
z>u1W*Al!|4HVIjd#93>La(3+sO?Yo-Tq{As+@P1;ap-bT#a;ZlN!*z3C;T~cq|X}r
z`z51sT`xDq0*p0cWwE_8rhZdCx|BV`64R~%jFm`h_ZeGg2fAYE!9joRfNFcMbKsk!
zTJuNM{BUvchr5c)Y-PsM*sPV*M>LBrmF0C30oSxYwTsLmCFbJ3@9N7sE-pvAC5`EE
z*|54{R#m_*+%Dga2g!L1+A~q7hx2~E_u*_GO=wE(-&d6<eyK+c5{v3~PfmsRm1lrE
zr*EWdrE1MYyiC4oXFB@9S{g)3qwX0bevOl<+L6T_sEl>mrw5x4S%)j|?2YgaRL?$O
zTe;W9>tShE#1q7D`JhOjysK^Puk*OpZ)!2+Zii{yfDtUeQZR9gJNvrgZt>pj9vKZZ
z<!(>A)V32$?PrGt>W#W4;2gU8ea4O_CCnpk)5n3_YEr8q$>$gTg77U9+{2-?&U;_i
zUpTaU4`*S{-pEZ}z0&8v6RHMogW(@ZX@I@kliJWBmJd-oR9gyQYd5EFs&+*EHHZ0<
zXT7_d)O3kImIL?fK}8_0+7fDeskRV&WE72-QqD2cWD^Z@#w}^=LG#)vzzKUk*rInL
ztM{((1;M$H<Y;{|9H;%{Szr=1Y!W{s5bTsn!VsNHdKuB>Zco;bTf#M3&Jbjd`pdet
zEbSdKIn37`{4u9B1I?|~do42K40wC-z2GNRQWUrT9=*~Q&=1}oGRSVfzNz5TF&3*R
z!L1&tTNue_&5dfNAG>#kH!Je_P5vxiJ-N&8+Pk!P8g+AjiKTX&1<+fWt-4e{pkNtn
zPK|3RIdCVE9;S-QvOok?k-24^0z$E{bclQ!JNQgQ?8eZ-H7i6*f|z=ylSyqyb275f
z#6^+6wV>rSN=Y<{MjHY*O+&%z&LymCg?aF3tNJO2E002dTJ>sX^91wkg~^Z++AqG*
zz+r@R=I!Etqo3Pg%HI-=x7dsc$Mi|hGyr>xNl~~zwjOHE25*O@!V4Duyb_;^FTDeG
z_NB%%y#PrW147DnF8&0ucf|qmZqgOe1I@Z`lnN6-Uz=Zi42Q6?TnbqGc<sn)Z!?u}
z&#E40Ph>n{>;@K34y%ZLJb2;xG-nEZnY#*!ms@Xpg15Yd?d<M_Ks~Ly-FhXqQ6>y@
zJL89E6$%AWZ@p#wsL*F_Ki%`(GvUHvb=^b9N1B@V+w@c``w9Vir1q7&pA#M<3hH|i
zl~o~F-~;VG)~OzY7b3!6?;ukElHKIgM5SMl_`bqSlH|n3HCtS$Y1i6V^LEUU(T)-X
zu<TxgJW*S`LcfB9?4S*q8fPQx>Fmljmb$C%2jMi&yXX?+xOfl`GdgTZ8`!H;cCFwq
zbEX+R(g8U@3$K|rTT|=G67C%C`8!n8SRsLUkaZ1$t*?;D-z<3G9u)uEM7zcYY>oSF
zbBY#|G`zUBH(f&R)Ts2etO6&a<SnCuw4UDJ`%&1rRP;UemyQB!!X6!}z(5?RZZ!v9
zO+s&%SF$L|r_tr(irQbOP#&nS!Hw-v##dxQ)phUHQw6XA=@u(h97wv?*5D92lQ)1a
zZeti!aj%04&0w>8?uo7&4)G`;Syg?q-e<RTzpoLM1a2g$H`KTelf^nW(00!9okfAf
zWz3%Fqw+W{8&3&iop<pc-lQbXRB*(PSw0Tsc(+^4xi;kEeoPacY{>bJa{D_IVc^_%
znqWFl3UaO-DF(9CGP%Aj-jRL=;DRv;i}$Gn0K%Br6O)UTq_Jshxdrg+_R(t1Z=|P6
zCuq1}bp*WREO|&##c<%G^U1I^3bd}&_F%#zbi-mqx4>E*a%VwvQOwU|;3q6nH7;`O
z;mk|QF6{JaX<yGe5$(^fd`d8;epQhSKra3Lh5bD9{PQxFPhS*>KL<=`XNO|LxuZ>Q
z@Jd^HuGVaG=zJGq_szU%=hw4!6naPHIu+2#2wtlVcpf-ExzOLmuwvoH^Y!;%pB`w5
z@ACRHpQ&|_JJ&7qZR=k?ul<>>J=BjZ=}8ancuYW{Hoj0&Fv`q#77Xztp}(Z267DCx
zMXUEYg^@v#W=FN4)8p{%9VKKHEcx=|v<J6hu?M7)e8ql8JqH6um!9&)96jz{+iV|=
zI?W}FUM+K1)3ko(BK%|dK=&QkBrxSYDvyBY=UmBSCqp=gr|@t&nJq^noc6e?j|$Z!
z-GTJZk)yKq3YMgfSvi!&t%;BtS|89GzuIHaMA1whQ!X>CC1^|CIj)~)!_G-P86$c}
zB^;*WK_8_GD3YmFDLQ{3nZ6)|8vnH5M}o3PP7x<$P88X*zCB^tw6RcpSeaffJa_Y6
z^6B1ZXYL|O&m{uM1rcP+rLV9JXhD%&gXH2q0K0rvW^5smd!1(Vz^-;~v@h8^>KNn$
zAqF3!*gahJCIeH^+>svox0Aji52kpQIcE@<`pI>{0z<#ZrF|p$SkLKg(P~zjci;EW
zqSTlh%N&~;2w(pm_%H<d+J0k#zi%H?dvwzX4-Z$8%ZRyegU5__#qW2fT?m)`KoV^x
z<gqeuOML4B>l}LD#;hsTYBhX_WW>V51_M<sB?~X}_W#1Wg0Pws$ULOyf0N=s&O{>7
zYI{p#Ts~r)pXi0H@I|1OZxRGNaKe9KJXXU)C8H5`MDe&Za;!LgK4-nOpiAH>>m=5Q
zS8Acpu$w7Ih6+w?&7`RcZxl#J`V8;R5(D2WlWi|up7S|`zY|+SU&nq!$uaBNh;3Ax
zI^h^A(pi$m{&Ov{Ok}cFtXg;isj!@|bwRI9TRY_d6Y?G2#S;tiaz`ok_MHvz(<jza
z7uZs~Fl^)2#qZ@sdr^_}_udeKw*X-1v;`Ts_9xt#T&M0j-@Dwlg>r+l4m>2SKDC%8
z2cP04ul8M8S5e|RbZA6*%tUXJe0n0JMCK$d_)4P7VTDEejpsD~#e@sPz@#A=j~T`~
z*2kX!sFxX+6YdoHyYie;##U|(2O#eIshNMOge8hh%=N}y>Nd^Md3a8z2T&Hryt1>q
zE#bo;W{3yPnPKFvG0-o|hWJPqJmISeeA*^-f!m>8eD_g2{~)=vu5SZ*UeL;ge4fj^
zGr4&fXCe0$^@15->c{i(5j5n#AEoSA_x(IF1tUE{c_<mzPyCHF!iMlxywhYTRMZD9
zE6~IULY_(;xrb;*40#iNulv@>qPs%X@+?G?N~(7pw2u(y-ED;oKdnPBNl$UWb5;WU
zQi6rd>KNfg2j2R^d(4#>frH7GU{Fy~z~=?QYd|3F&%HTE{wlwiqSq=|-<1sd6;x_z
zvkX}W;X6=1%Oh)+Ql2-9%(kE#VDyXJ-GvS5PE$k6nZmCMz1Ja&rX+R!I=#oiNQZ2r
zgbb~<pKfmK3mNjOem<MQi#_Srr&JZ3fGoHE2GfBiw}(L9H8rhgfpJ#iZlYfPa*V14
zTKb8#t>F0FYsw`68`YByRNCl(DsLB42rN?qqd_^7+r@q27KJFFAga%KE^uom-L<xD
z>l;EOR#6}WS%l>gNQGwEd^-LC3)f2;4eXs1mZH#`Kk<cLGNnxuyFGfUZLfv4-0IQ2
zUd}I%sX{bC-jy1rXr*oYT|cFl71-HJWYJpx83Y!o8j@<X)9j|X3?k@n{KV7Bm;Dil
zoSjv5Hs<09R{_45{}HWjNh)DICV_cYf_0JCXFwWM&6I>CHcoJ?m#GD>=d|v;l!Ms&
z7xRYx{+#KF@F-<9C2OfoR2mf??~mnoIdTngv#VK>Hg8tZ2EZJT?VzO7?r%SkMC-hV
zVj-)DGIU17D4x2?%c+#d-V086z^<5djJaG%ynt0_`ODHllkXU8_ps+4Ut7M0fqGhO
zb;tPvvWO%^l!fCk2074AkCE58t2_EJMRB1W5kn}@g*7e!B_<+z=rgjh>WHFTb8~6y
z1Ldei6(%uYMoweefXXstSPAHKy7ES7v@;#7+oSh*HZH{d<40Z8ao0DQB=T}{X7F@d
zqLvTNof>5XRTtu};CVEO5b4a0L!3;jRx@VbJH4b??IfvzIF&Y{8D*;J5<~Za7*mWE
zCz3qX^FqDbCR8yWWf2)=i6x==>K>;^J#Mky=W@QqPsT~)lkW%s!XJzxpKiCj*|XM#
zf1OHVj;61n`E&iq<Hd!-2RY}R<>g0MEPj7)hros$o}jfZ^T!&i6(H5xGAe?GP$hH}
z9-rBfGhQ^h|5eG}^1AEz@-bt6OaFj>q;G)arhgulqpQgJIYNdR6F0rdHWa5-T%j+a
zdeNSPH9j*1q0rX6n3^6JI+T({2&SNUNx+rN09h--3oGCh)F;3r@>D-w->*;xVt}5`
zExT}KBviHRGrgSeo`baX%Wdq=@GE=#D~)qowm5`cezpq<06yY&!Dv41r$~OD-DExz
zmUtD4<Z)?qD#JPM$oEQR6vBc&ZgM(McabEM+vMf8q_th&D>B2vtb*CQI*&eBZ~$f0
z#VO5^LrnX7CDO)H&8lU>MF{%>vdB7_lRiCuBD^M2C1aa<_D1C*7zO?6oaKznpE+y_
z=#QW+ATJqSjsn@L7}#Bh!DNlz;;wpK)4~}ztjf7F2)aYgV_ExVV1<cA#j-vjJw`r#
zfBs-!s^N#8Pk_KCC8rN&`DN9Ti?^(^E|K8n>f>SUZ&73|lXS;?wP1TKJeI|2RCJZ7
zt!J^G%~w;(E2BMZ@&coMf_pZ=i-l5Ta;>tB5w>s2VxhxaE;84|FDpd~c2KNtk6Exn
z`EYjUnTa#_5;qGoy?#nUCo^{*i~)_Fmi4Iq25K~#zyxK*N)02TmXZL0m%q#*#|)YL
zhc!jl!qgkH{C)Z0kF|<7cS-dGp5v_=s;Ll4mWH*$)8R1bO!uN{o_w4U8(J5n@>?N!
ztswJ_t#t=;p>?i;Dp#Zin!v`~xEew<0Gk4_81fsj664AYO2f6|u8}3>^AbQ7@8{|G
z70k*ZHhe*hJbw-iHD)eRO6=pNGS*CCt=L5R;f<ug!Fk2gA6ZBY#M`tiByDiuYxsBW
z(xR#z*eo$L*W!Vs`i7JOV3BduQS(V#fhG0gxAYli^as>LxNr-N6lCgJd;d^48bs~9
zOJhi(K94u+Ph#h?64=k!%X^nIc;b5oS{j`*>h!OP&$agP-Q6v9PqX~Y`w)<uOd?jf
z>?coWC3s^TRB8zS)~WCV{`#(>*)eNbtk&pE|7{W~93v*X?GeOfOfR@CPB@9ODLKlr
zF<>x+<~y$<Fk>R8=$1k~g`P6sFz`y46-5&qnCu*tb_V0K<fW1OE%gm9KhMusH(;kR
zBG*&Ll4MQ|T1t{JrNMeH3&EO+E{$(Tt8QUZUsAjC&Q6dqe#0^<S2=!VTjlv61g`rA
z4=#O7WyddU!xVPOuXqU*d!z+$Fc_+KY<X(gk7w%`$Agd?xw^1dzAz^wAZcdU-Q=>1
zBP*~T=4j#E^e10T;>zcZV_Vz|zQgIOkfYaw#Hf&)rxoH)kt*WX0CCHE6k5>yvHZx-
z(}q>DOYKeiZOblg*fRnIcE(n2gj3;JmL#WGK*1pFm<Vp;AOkf5v|!{c=iYUkN`&9_
zW$yyLs{LIt+v}!vyplL1?&gX{?Y<6BE7b9PDS^M&G~W<77?T_=_e|gWGSW=v0Rw&S
zFn4Vui1mIP{s#w+hxSJ<=M%%9_xlrJh;kSF+n!<fc7^LGSYKCMcijL1l&7e+yOZ)h
zCUxJEOAcs+J<9eqd$&<uPMcOoVwyen_Q^G()105Uob{UqJcP_i;y-^5SU*|${(+A9
zi#oBxDcILm&aZa}2VCq)A7MLhdV+2{ep+S4%=#`RA(vWEaB~H;M)bH4J`@3$nSlc?
z2<6#vxhbR9OWzQKtjA5O7ffF35!qTbeZBth_Bhf-1w0tn$BbrPVC>6@I>jkAyI1q=
z%ab3rKtH*d${X;94u8K3j-djY?{l;!QOW*Xkk?ZYW12D}?ROtQZzc>R&^Q*XwIcjD
zlJTfhNeNDtHJfGBtn&ivx4M|U=JD`3`K8UudBC`2ES^f34A#%k4`WNUE|v4dfnGtX
z<@vz;FHR=WlLoWW3DNX<&fEvI>gb~Koq+}D`jan=d6<9H_~#-ZBX4FYs6oyitA%!)
zH>HMh8AW8kg%h(ov38X-uai+$S$%JOaE~!T)ZbHs;?TxUwNSn;WNI&ha_S3hGehHb
z{?g@}XWTbMqv-~UlvCM*`Y-ey*DF0Xmy;#Rp3eDSj~S)B`JYpwPTwV}y>m0(g<IX&
z{4?6{<=eZUNb(p}2xt#|4O2Up1qcXp9*c+qsdse?_(4lE>%`i8Cs1ZDas;AJm8WtB
zZ+sz+K6Po($2GA;N~-5!oICu!iF03FHCKa|hxye=k#Z1D&{7*G*L8smKyPMFs^Fj{
z=jh}3>52W6FqN^#tx2CjdKpK9(0g}cK7(J|zp)b}0OM7<jxqzjA`4)1_rE9C2>(F(
z!animqItjHSgboKvaS3V-W<5#r7PFAc>|DLN}LaigYHOpx{K5u^Hj*{_j3JkQO!aw
z(#w!gR49XE6;JwX_mI%6oken>)Lq`D7qyWEquH*o2asTsha0R|b&UrmV3|!G2sf4>
zCw^ysKOOs&9d)|IH_@Riws|_xf-!x!Lt&SlCc3YPMupX{6Xbzy4JzIx6Q6A*4N=_-
zBUIDtgU{`5L^-8z<J^-R9W)-J1yamoTxh6ly#H18?w}O1Kwn8qZhZQ<PZ7T!b*kO^
z%PG$4ng8SR1Pcoa4+H;8v@9TP%~Sts>E-?@xv#=OJh)o$E_-;Z#cS=C_)Evz(r*uh
z9hkaEqB+GbpS~NRM@E~8rk^;o1i^C|2!9s*odUB-Oam1r8U}S=WW@!z0j74l5|XhW
zg8mpiOT}^EKsj21qwJM0)W*Y^hwj`3D6tAoi*&E!;EB}72W6H~1*c7$g1t0n6z9<2
zH(i)BeTu@L2j5pxW;Z6lL+#Bkb)mzwBUj(}g%HBeG~mZKn4W{Cvytq%>imBR2<UBV
z0KL8Z$eXPa1$o<TKWS17EcwcT)e^t_e&@pqrcftE{b46Q-y~N#FMp^r@|#cZ^Wi7b
zswa||qzFI-u=$zin5JY@Ed1t^h^kIc5j^L0_H~f5#uqh)5x9%r-Y)`r+`J8M0WS1t
zYJ;%0yC;0Z3j*Lrue-kqGk1AKJ1xjZ1l;AhABQAQy@7n<RLGhG9b4(Q%_jaT&wb~E
zRJ&Ac@GLedzdjFt*SazosqjG&gcXdB()K8Ea=G;c+DvDG)c898%&8=nAahMQiqV<l
zX*TRhaD906k^-j_8m~px>fd}1+=$EqUal03UKENGJwAL&GEFr(^Q>I|Bo8lf&qc33
zljkBPitNFCJ=yrl(s4Hcslsyq7~ktt`1|;ncgPTk@6QR%9mWg-7(;(J1xwZfxm3OJ
zGWk(h4@|D)plj93dw)0@4>NiTWRjy~Gas+*@sIC`9&ZHm4!a!=@2pnLy@fZt?0Nra
z$z;N>1jumFY-nu^EJ>4O9#?XS<xgf6*?L+i1BW0#NE__%sBkUX9&)JBBi9j5%aeUO
zy$qDZh9cfkZn*BE(}K+;C)vy^(?E~u#stc#uH&%W*Up#mFuxjRTUe4T%<px3oQFa|
zgB|IvnL_&?<a&Xh%Z6C(wPXPfL)e+9NkN@Ew_^di%f<VMK4)<Iz0Tjj*|+MWKL^8y
z<-}YO&A8A}`WC)%B)7?mZJ~2tv*l*yZQ^&xyE<7HGd2#s*cmx+4wU`8$tQZe3!}_a
zkSl6<b|<XlB{y6RAy5=_9r!z(WU_`f2$6pp*qLIaTXXRXh}mBJp4W%;YA+KB1l;}Y
ziR~sVmm+1sK(MlbReXX;^thU`L~;!~lJL~g)e?blhkg(P*wr6%<U`I)clCw1Czq{4
zzp26rJ5bkBT~Tp|rMuSe+-?U*7cHKpY~Y}tmc_HW6g4w^YPQ}JlM@q<^}wLL$6KLo
zbX9)EeU+)?XV`+-)dQYnq`Ri?+$Q7`_TPzF6*w#lvk0>=7iHA%OVEPz(mAg23{*3V
zB$2FtW5KA<zbhOqz9*OK)lFHd70AghxF>ipIR-!*_4&q;6@uc@Ilfn-oiLBvHjk<u
zha4Mw7+^NN@O0DAY#^m9ofr()eJIFPJmgVqH(Mg5pyK1XE-a-c`}FbnU?Q1(BubpI
zfVSaB(xV9#=e6^pU(ZKhAl&zT5f+hh4-jgBTTd(5y7((gQjlf+UYUZ#jv4O_jq%GE
z2aSqpz%wCfiRR+nUnjM17FKPbf*%;=UCj9m1H`{30?@RuF8;A_Rwf9W*cfUsSlo}O
z{nV8%)FO!AsA{usoLMqFg!AEnyDO7d0R?0Duh}V}%9BO2f>`UE+U*%$HoSTKBdEY<
zkhJpHo~nQw8jU%A1Fs)^_gS;%QVU)1Cv((!&54)GbaDaoqdG55ymj0u)OiZ>>Skt0
zQV<W4Z8P@<kH-v)yXY$It+o2cKJ9gZdamTA_38oKIxbRy+H{VXJA4NB{vWUtlVs<9
zXJ6=|8L&XZf0L$X@G5}iNn0!)9t*1<axe7530ahh(cJ<atGh7WHBqTjL`>-KI=uJz
zagsW<tJ60m*q;;>o0E^GLb{y&7j+6DR{={k6%*p`kTcRR@c8Z+YM7IP?7X_)WZ2Sb
zI^qAO(nVRylCTcH&7-0K49&(US%bKubeW^eX!zgx$S7cHXm*2<>%I5r7DL~`C9~Uy
z;koYlvU8i4mv0{^h-|T|oU+8>8ZFbmNU9*$TPK)*Ee!h)V7N(HVyN<=-iq`LO9~pj
zdWz(~KBy)Cr|NIM=hy~!?(1NKd)wy!`Y*r({aX?NEseQH_gfO%pQe+Eq#%l}gleDS
zG<c|}pjqgk?Vm$kHM!o@^}*ui5YB75A16PdHxwO<6ObIz25}F((6r8(lzoh<;aj1O
z*GELT-d(>}Nu#$&{YWg58e<*qp$i@pC~z13H$#u;dt@Qd@gJDRtGOt~$aL529dD{Y
zgmd#*o6!}%@8>rzqR0uIy*{_!0cVEBW;xPauG9Q+7Zp``iKtkD>c5_zMwA7k1j6P<
zR-72&l7dc18xoNYLNcZzPw3MpL;WqDiUk_sk;6KTB|^lCj6Xb5ej_?ko$QJ|HbWj;
zE&a$e{__Vc26+#$52AXnDrv)W{^%i=JmJ+~{>s6QSIt!u>0ya|m-2U_U1LEr=>mq=
zr+6MhL9lC2*93mPKqCP%WpnVgr!YKHTC$|0<~EPj$S`XejI(ZY*3Zg)3Gc|5#=fa>
z;B8$;7KqU>J5_!1M&}V;VEWsK1*T8BvI}S6>%=tGPCl#%nermMW+~)(VS=jx!L%ho
zUc|#*BKVt@bJtss@s?sko7Yn%{DCCe)Uw0}!R&QVfB3Q18tidrulY5}kf*sBx=Zoj
zbFnBo#F4(r(z<&W;mV5+>vR}4ck}6)tYZ$ta@3Bh*k&P4FxxrN`6S~OQ6pIBDgSWH
zMOB-D1l3ONsgDe~BzoSLSa@)mwQ}hya0AiFBZLfMYmQfobO*Cq`Ryz>%OgM@i}x&>
zL^fX!gNnjNci~?2=2TC`Yul6A<}J1J$nl8jHU&e4w+|CJ;Z#d8%T?Kn<&;CBb<o4J
zmb;iKcx6`GB_2M|y@cRbAp0#jew07t!1Mi@U4v0s5FRoJ%gcjXLGe^Mmda)vKUml!
zK#f%4Kg<76(IEA!>hj7IxC|F?a}w9E(b6R)eb_o_=##v}HV-cEdr62rmUIpIQ>Du2
zlv?K#%5-d0>tNl({LT01Zk6=mX*{PGcG<i)=!j1$^dEocFrPi1x&GF&;C20F=hMM^
zTX!T=w~h;S{(a{Rc17%kE2FAAl9#{RV=(VcV@&P+Md+62uBbDq0H$};TqK418@9GR
z`8%9|CuLKRz`yl!&KT05(Ot#U!kK)Z1OaDX9LAsCy!4N_F%ynH)QE{cU+)QvB&D$p
z>ULKbbQaM_*VJMaS9k08HRKb&X8~7-9}uF&#$dohZ7mz*?sldab2dukRwcx57%-XY
zW^lswjm+fN^weuGO?`8urnB&(ykh22<?AU66!lWmUbRuKC<Yyp+kU4x!;C=ea*%PC
zeX*i@U3l)cupuzhE6VHqqUa4}NfQP8u;tOA5vGeb*%2B=eAw)wou9jOu+zN)e;p+v
zTzO+kt1S}#{HQ1liPj?lFoF#p8RP)GQioxwpF2wRc2XBex7u@wa3sGRB5W2@dqtfl
z))ggX<il3V`IrJQdMP^`6|G<x>A?*M6!d2Zp`P@t7wY86qc6DW3IaY^h}4<O4#zP)
zJP75Zc?9ZY3W#vzHl})vHN9VeL5d%^!-N)2@*q>hs$u3+w9PCq`CQ4%dK>aVP(rAN
zP_G6$D$t1#H;GkIHNJ-zs4wKyb=4NnTF#g>8En%{uB)>VxDHRz5f6f-F>DdU9lRh0
z9Jf+=;f28*84q11)p%bM^f*vzjej(oB(eRji_+0tPxa-r;3=Tv#{1`+7gjz5;3<X&
zhPC6%n!BJI76N;&(~aqG>$j8=oCmy|kLS2DeWH4<^n1F_boJ{dzZdtVbB}cgwt1(&
zBQ1)u>*pV`Gcu1?QUo|5A#8itebcYnR2sU<HH5f?zmj?>OAYu(FjFQ!ln|A4!kE)G
zbKRBw;`&yaqGF@-F~_-DpG|Sqx#|**-x<@2McdXlcn*jn$5uiyv%sesd684QNzfxF
zB}N#*y&z%$)aofwi7#Va3;E-6t7?NaDk)_=CjDfS0lynDhBWx48#j8BLs`1d9?4`r
zgzNq0^F99GBw>3y+@@IMaKE3+W%VPC2XV<sT%gt{G*B0!b&FwwPn<MYshTg!ZEvM1
zQWLAmKTT&?K6-S#V`$)`_!ZWJxK+$^sDv3!`SFH$^2+ZcTp{AcRY`W*gq&6iXv~^1
zO-#hi=&&BHSE^?lGPQj2Tu4=bU3rqRZ;_srWuBe|Ub~Q@<E4Q=gx>+C#e+1wD#c?;
zqalP!DDr?33<8Arcnin6CZ=@Y6X7x#NX&QX+MEYi<BnBpqVOCXQDi3?9dKHx%1^?1
z&6U{&wDMHV;WW&L>m1fYc8e+d*sr$RDNz93sBx;GVUY&Yx5!L?9B4nlL0JY}9-y^K
z9OLWu93EqDzdTYB?Z+0lU88R(ZZm6Yub&qYn6ZTduUeua--;hnk}4!S4?_X>){ODc
za6F__u}*?$-uZZa_LGAE_1u;54ZB^_F`Xa&R>~te>UK{HttR&xs@=uezlkZ0jFNJu
zqXX;Yim!kD*@Acf1^IDIA08~2?T4s6)*rgq7Ww0ty8S8vLTI2|&Li5(ll2*Rm(8h(
zL5k3%A}-~ND#Hns7Rz|cf}e~-=YzRUS8?e_qm=#J+&Sk<nHdqZE-Wy<L65@;!*(q-
z3WHoJ`$8SJ4hKyljQc?Vp|^Re%p#+DHg<I5{urc6_1=4^sNzi#D6KubfNNb&T3hvp
z5LgTY-N{Etn}nf#6qoXGM*){|xKSaEDUuvb6I`dH=_7u%SrZ9?C5r)3W(W5i>gT?F
zr^WaMdj&*BycDnEGtdeWCO#e^{Y8zy>tIs?gs6@h-X&5p%i}_7Xy}msWMdLYVo^P_
zBveN6!#){*^dJHgeUTu;<2ETWRp`S5_K|?|RBT0Y{NOloKKlQG_<13Kxq`=Jz!}gS
zWSJ(;pLhfqa9%?tLuqVOFr4oXeG!IF3CjEj*ro6?JTM0&$o5)Zqo*Q*Ekx|t2i73_
zAEdkkhR>4IUZV+m$-u_Tr0eKVlkRX#5byy%Kp->BMDjaJE`BF`TCx;TqhSFgw4+{F
zAYF9~g?-03>WU1Iz13+%fv65%3Qc38RHlQeN!#R(dqs<Dyh#15QIbW&AlsS-orvo%
zkgTcH36ErF8X+AX)sHN~<Uu9jbi;hVN$mi?fL}*-s{4LIm~#a(6#&Th!T>d6sR3C!
zDsQ=?K=X&|35lbidMZ-NX0C@>Tf25tH#jH|XZv*$1pI>=L3Er_p3EuweBd{X5kXH9
zto}*vu?N*TY6X*$KDTJn4<fsMl#U6Ll^XHYCdx#jlggONDfH&cy}TM0Sy<QCI1nsF
z;Kvy24|`rs)4x5bjTs|knm(T?N@+*L?^)0<@T_f-dvSi%Tx_p&RmU0XQoxaJsEVyj
zoS}C(=+i=Iy59;_>Zu?>vL}{&OTw+z-y+>LR{IhSI(9Q)6Dpsv3`^;k`x9hC`22>{
zPHpbMU7$J+Gx|bM>=^4Z&Wym`oGA2Cn4bK(2*mhdI~i4d5#d93Gw_t4SF2v7`6@zZ
zn&n!OD>DWSx=9uwY%3$Lw`v}iHu;Sn1$~qBds+tK3d+gajB|!_g17~6z$_lyC!ZL1
z0_}%+Z#_GKs`Oh1F7EffyeY{MH<~VlgqfTjn(uc0vE%Ti5ma3vn+K5ED`DPoeInU(
zv#)EnX0558Dr+`_pa9)*GIR8R?V?;()SC3kOR2;LKB4xw3f`^JG)zuBAjA(Fu+hGH
z@F0oa7)~03Ee9YXJ+agDMpQj$N<A~flM3)kCWT_MHkIUkvXP)@K{{y`QVM(ecB?vJ
zs`?wJK14+)Giqem)-hyI%-05xHx*^oo}T;&$CW3OH9NdVzK|Z8<lYfFYXF3QC&VTF
zr2rEPm}%Vqdw`TIjMVM9h)V#LWjDtw7(WiCZb{{MDr89GI^*CyEb;pxSo}Vc(p*fJ
zQDm6+ak~_JfmV%@UV|E^Ny{7XQ#uG#1>Ii`t&q+WgQQ()*crgaPsC39Ffic5H`MED
zEg6~y6Pfj5J~Sss&II;v-51<jqXbf0sBAY3UIpz+6?W!Pj>Eg{dGq@HK3fC@s0$x?
z(qWrjJk@wHopQD5r8yh8Chcg%P!I_~aDo~6u3@_kIQPiyQl<e4mmC}J7IUBTiQpCV
z-A$YqE?MaRn7C`_M#AsT({dyy`>4oRCmcn;Dse3SXm);9>OS6_2|h#H+Wf`uXH^)M
zH$UZ9Y<KjB-Q(w3{>*g6Y*va=yWeD448Aj;MUReo@NFa{Eb4*7&em2o=L-vqfG9my
zlmO+A-=zXv7n=-&m85l$GF$bOu{z#GYpK@Zw3`n@;dN=R%9Oc}cZSh}a)U|qdY7Mm
zM4bw*D!C&fw@S5|pg3@S1cj7^HguH9f>g<w&MJ4pz75pZ3(RB13PotxVww~Nk5Rh+
z+p^j0PN^Cd(n|P;^;Q+~#>&>yUUZp8gZ<sNYn=#M$fx=$)*Z^y$hee$zr~^XGzE!S
znGWPK)_m5p@Mp0iNF)F^nU^mEs?*;11%1bu8Z+tcp#!aeWyg3yTuh?wOLnSfoc%sN
z@Z-6V+wUkT*+8ecXhh0Oe7pkf)?H;}J6M6}m{mo?!J#)gAjRM@EOfbQ<cqtn;|XyZ
zuZNjPlu8#)eKJhrk{l=tp_?;Dqn)?eRQ(ExeDK>Hn(N`U&eW9upM%t}Q~SoRUQ@<1
z#BfjoV>5|@r&%G2bx-RYtkui4T^CR?&dUTUTdVbX!gdb54P;~xYafDMwfn6SCD1S9
z6e(MVwiPTHeMx+>rY7;N>Z=R@Pzy2!-6j)oCmR$~uGU}eCz!GvMHJudj0S_Dzm@+W
z_D2j8GQcNN63$-43~?J2ColLWg^Hf(%Lo%^eY=kJOI=e@d~)}F%(>2`YO`+{o~2BI
z*cQ_)Pm~TyM1=;Go+>kuqrHcWV0SfsCo5*GY+Mp1YPI2WlxkAz_t4p_>mQ6qj8ojS
z6c%)ba;;pvo5gmuKf;b8uiFX&O7&u>-bM(sXBrOpkK`=y@F6%=Xu|8%*sU;?nt?nc
zmu};+*4==StAUv_)U229yVSvP{JoE)>o-;N1eORVU4AE!lVwUQLX9^pi>>ctlUX0;
zsP0g@LFzCsI}*0=yp+$hg`JeN)~^`E6hO;E6fGh^f1vlw0QmtJ>>ki~MH}pD(3Yrs
zj`A7Vt*gW=c7W8fir_fiw8k?IMAh75@vgtr*-Rr+KQfwzgtVfe*vs)n1X|w)qA+7e
zc{j2yK~ieZ&@S+CbXRkuh!kF<-ZcvIuo5M0g=uS-*Q^>3hfQJghF85ltg)vr4@=g|
z8l9$l-xT9nfvV}J^rLaOEUxkpQH4xoY8cPv0O4xn<nd=RI%=O8fikKO;8N<QNog6f
z%go=-O-C-2JD(u7Hw4&KG~cy*Rk2RIIn6eOwYgo!#XPH~&@W6Z20m<k`a1_;x|^UX
zg(MoRNUNLhZ+HpahutySRK{4|S9IKTs2khKwP7ZBQG;vXNv5Aw$tGwa>#!R|k+;d8
z_P)|=i^ya9O9A^(4FtecuWrmn?Ds9@nBN+{l4Kb>cuKmf0p|#LFrYZgg9%}5Nx8V=
zAwQX#!)<2EIJIaUV4c){V$(^Q$7hy)_g4C6Bg4I7T6A>IEkHfz5APoi>3+f#Wm1ul
zDP<m<HZoQ;OBoaBA<WmDCL$woeb6bwg_dMrWHb1oNe-$G5@xD1{tbX78TP0|*xqdo
zi*)O^KN&aHvyW;8h4C{&(4t(XXD>*F_2pp%7Ii$Zm<BRDm2mp`Qk&X1ud$;@b&P`F
zHqxZmVQ){#7_SH$dI3ks)O;k7m7*g8V2`aLw2)kiB!btf>)-zk0rf`X-hKJbY8KnG
zKH9JJfv|*<2fNj1+nA-ANR05{_Rn$-eLAMv?<VvtoGA~l+=*jkR6ix4HU1r;#TEpq
z`9v52(CNC^u+xR#Y{iQ_%D#Q)pLRCFAZ+vF^;o791A6E>mUN^%n4zTFFNeNM>~=x%
zpdL$Dq;9KC@qlIVZfX}4I~M5UpTHK}p;#~wP8JtvT~AwZ#@K{CMTyL3y&+};O8Ywl
zs|U|mrIT?n+nCk>)9uwCPjXw8Qg;Or!!U?Ewmf2U;$m?+sg>QZOEy13l%9}2NBPW4
znp1~X$${iYlb|Ac>k>cqT^z2@(q8a=UC{{-885<Yd(sudw_^)WN-sSB#IU}f2&Nco
zB9;v}?8fG#d@i#3x$aACG?Q?JLBd}X#^2%3(kNjX>~wTbpF2Golwk)?A$~mI>h?#^
zVqpErYh9r;Wd|mY#9}*eh$VavljKYyqv_*58{J(vtCB1o7VZD0;)b<(19b0MD5A+O
z>zu}Z42+c<e&^Lb`}w$VrDmEx(oP=j)MOI?^?g|u{|fs<7i`_pI_h^|Qga&riVxKl
zdv3#}6ZW*-IfdHJ`!;b1I_vq5Ka?_&<TShcoO<ez)I^?pG|yF`gS1AfduDr47m}UZ
z+Av=U3Mpd}uL@va?!-2B0jvUy?_*9*rr8LWaquajavAtN0-jaY3Zrg-Svmc1vR6k?
zTQ$2J{{~PynyuTi^1d;(%R)H90(-L#fV5RPzP=EGA-F<vll)W=%_=HJRLF5Pq55uZ
zk5-p=nw7pbN@Y9O54?6@+;MxjO==?*6vA(;d}i-y_=&1>VEEqhV=S#@>RNKzB%q-$
zpH`37c*;%g8YxIvoe%~fZazpIZ)AaPZ+-9V14Ei7HJO)frEOZ<-jPax&>D^XQ8xC{
zxn%ObeAuPmgavviVX>=kPS#`DO3-z{eLXaS+-R&F2C13dzciL0G49v=n5G00FFT+Y
z=C!(K6UrL!wp4CXElHl+FE!tZbGfs$T?tA%v==d+&`f^VqODgdZ|or5w3GMKDkTDh
z0d#aE8pB!XfcgbqodWIm`)mA30VEg7>bhhD`oHLI1NKNd_4z|2s%h3gQM%1H%$JlQ
zoC&UC3?nv5FBkfejTr{Y3!Esyd`fVP8kP*~+awJ9-H8YoM@KGmZY=kK=PP4mvDYMq
zJ~D?%6I{WQ*m+(>t`T0+&@~C*WG&<S;#I3cVvdrI>ei6bXuzaTa(ol>Fw`k1dV7`R
zsL=Z&EV2ufEs+31sePRk2s3Ne#;!?5bwKZ%=#)ZUMv2B7=}@76*~W2F+Q7}?$3mo!
z4RbB-ST7czb-s!4EyMhBqJ&%{Ob}NsTWD!AE!aH-=yY81>nBU4*NlIV29m1=n@_sr
zE!n*-s1^Kff<-%XjDwj?3zoRMo&C*G&67W;-^yWf$ai<5CoT%(z9@x&`!N|c4CF4h
zfEY{G0BQL!{tgA}H%(}&^(<(y_zW8PE2cPrxhgnrZcPsd8xN58e!A<x<o(e+>+olj
z2A52~`boa@K`SZS5JfHDFz6>kdB)fuXy~X@nPjO>&OCy1!TWRhS|3OuG4N8imx~er
zHAL)WuG^^IKuZEibry_+!nv?>#dW9}tAHm{1hp2mh@c$O`v_5i@204ZfI0G<bse6x
zn47y9@0My6sle}F9${6k$QXER0CE~iR>?bZ(I|YdG2tw!8+jXQ0(dv9by9?~ldIG#
zQ#LKr?2h$3Ex#Nxc|f5*pIm~B9w?LT)Gj7%5u;j7GjBvP;VoLMNdaTPp36XwJpQUY
zsTFd9J*-2Ek{LUaV)n_tK;F9Qkgs}Qb~*Ju!D9y{DQb>$VxzGqkzYrwzO34V=O!oA
z-yH*6zU-&G>!H!3RJa-M;Fc7RPSv4IzG9B$0_M80nt~hpd1Lullb_v^BPz%PW{y;b
zcP_dZAbOgGpdg5gitG@Kr2<b>MHdmN8<;pVD*Ep#5wCKvZ8LR`G-qodnK!qilb7%r
z^Rb(S^ndMe;1-~Nm(He}R-jDCze6Qm1}4ycVMvDWSE_I#rL5#3@%7c0VVk6CbKBkt
zaM}R`c+vFV*S}jKHtIqg0hMRCQNVZk9bRGz)OTe_D1Lf;UC!I?V3wh|7M%Mry(35+
z`i-*ql;p(!j3_O*y2N!_V^&%f0QXX|)CWE)$>3aKODZM<P9ONSEb@NkmlL(1e_#Bg
z5NZvo?TF04#@}`FI|tS1&qvaUj#1qQF&@dFKpWWjNaV<3v(Ip(CZ!bU{QLJ0G(LNz
zJv6?;fbiP#j3=~nc7(04*gtP{7DW+oeQ!wJad@y2iE=%mDDX6COd3l4X?VW^nT~W3
z-Q?Sw09Hy=(bmnOK$ld-X4jTzNEX_83c-=ew6$|<i=4hSM@-;b$}XbR#g!-VH)YoJ
zphxUZLk92<$Gg324*W-v9C(wZCwCG={@aUs!z57v0u6h)0|D1?4M1Z}y1ekD5E**r
z_YF(&u^?0+`hmrz+}M2hCfQ-qcG7Su^yQUEh~&fW$sbZ>45MUslDW0{KFJrb>0U%X
zy-1?g*C%_q3BmV-ZRI18Ai<kyG^dlk#f_YgY#M=~^K28pa+};wK3MaS1l*a2+>_X_
ziX=eX1%nPxW}8bn_^kIDs{d@g)rR&-tuEfv8{0FK1~nN)ko32Asm;}&p^SJ7+r+H^
zwl$hZiPIN0*2*LwWm^Uw((_)8a!fzWy3T_hynbK%h7z-mHBg!4vOLa`W=0_++y$4J
zh@SLz4yv{deq%<xu)AoS$|p)LhH)_34R=M3e|&md<%%L>c!8m+>XOC~o(GjwNVtYh
zx;QRwJsexJz|S}doI%<73Qpnt4c}mg+7PFgs}dX&Q14DPKVqH={!CYFp)GHJr)KT%
zLx*R!BC?O_`prfDF)g`li>I<!p3Xh|!*vJ8S0*4QsuG|C)2j!KCRZXO6O%ozeI8_i
z6v-p;9Ev9WfMW2BeCPuEMNo@-@&MF^JhZ1W&p`ksK-$0ZH8<DC!4kLA^jWiSw+@at
zSulI3N383LYqqDaPlMN=X>BWd&Zg>X;0gxaVhzkMUSq4LwKIv2G~?S?!&1?yl^N`6
z`o>sv62f*dm;T)9*{$0nUV9!YcR4sa(|IaO22ciqaw<!tEi%53$Y7$9Ejy+~3h&#E
zqzO(IX5Zs7ET)L{@kPzelOLM0)4e4vCS5kChU%x3Rv6}{?9$~Xl>#`_;wBMyvQr3H
z4ZeL}>Po!0IPZ{4{fTGaW29rQsWZkuPOh^{qg(_=Y2u&(3TnbD*ujzfZ<J&{4X;k<
z)9<mAl!#Y;Qjqn08nd3JPgD3WoSIcr99)owaS84YK>`T`hrwNgy968DW^gCCyL%v5
zaF@Z|-C=;>8eE6q`|ZX4fvt0O&Z&AYx~u!`r>cj<8Q1Pfx*GNKu<%CfMn+e{7(azg
zn$LCvD|x0&oOGkT4;S86jik2i-EfSOckKMj+fUyZk}cRenc{*AmryvbKzUQvK>(c^
zIk&m&pd2ZRj{^AgX<a=Z)9H?-wy;scb;^<(?yoD`(p+s!@)uIOh|Q1Im6(#p+rQx9
zF}E$5Cq)gqf7_wr_z>+bZgv-`bRxX)!_KJ9Kfg{Rx>-EeXJeZ=->)xr48tAZO2zPW
zw_Lw%5CvMqnzb7LGF-sUV<btgx^9E}?VXO&xTc~y^eUeBLm}wFx{&KdoR5QC2_5W3
zeN37@+ue&)*3w&`6vV%5c{#wVpJ5+Eg1**&j@EJ1H7&aRAG<qGZ_aXaLmg$THY+Ru
zSztm+%ey`VIK1__$qL%@bL-e4q9%4f0dw{IwDWN}%oxVOgs!c~U;xF6a?ErZQhSQ|
zb-UW@#wFedeRK5KPM&)wISsZ>Ycgc2wjWg=ZPrYU2lo1Q@8Hf!vRg_43I!sClj$0>
zI{ur7&r*3Dq}Yw?mOaN81oII|Y&?!=j#9W(8r7P_T=lQ3dZBe+DD=}lP+N|Sirv_c
z4+imQ2?FG?nmz`KG$3HlGEN*wluRI4JOqOZJ%cy$*OTeBRx0je>Hq$AcHW^fC*(=?
zn?y}s6ODyyWmCb?c;>hTESAh2Y&9aeyV$UhH{ozQN^!-qp?V-Ub+-0AoeD~RBH2ul
z?Ne`0Ac6A`{EDG9gDnVYBy+Pd8HIPWBW)*I`Dbd_q(pV$qz8ZS#_)}<Lt(MNugW`T
zLD|05KX@k(JVe-(thilgnV=uExq)lNaQ+SLGEiMD=Ec+z9$cE0F+Jpju&yTGn{Ga}
zu9VT+hQ3(Pnok7<ZUn;S4#AYLz<Y5SnbP7+ZdFQsr(Boih#tR8&2vs5VZYGwXVR2|
zj-j!Sir*)2S|>(><`F>@YaH;@0Cy=!oOPc^G6UH|9i(ykTb|G-Kyx%YIMMH?cTU<+
ziL%YsQ_?{aI;fHjCH`hU&yLUdnYukT-t(zbR>DJG_xV`*(pVU6xi91oW*8C(8PfGD
zDB%>NKWs|n3`K^%?`c`L9|5X;=873->uSI4o$8%a0;?zBqgi&TukLI0ndL&%MJ=vJ
z(>zy96QzVh<9?%5m8kyX*~6;lX$Z=axP3PIzg+<Oja1DJ{?%EOL)fqOpzWd|)Gn?c
zZcX~A-{X$E=XSStz6yHASP(Z2mF~uOYM0$vYNNi??+x)cM!5_-Vg0>uVfov1<S&9(
z5!&JlGAm7&oW>5Aum*6@GfLsaavZGAhUd2OwAlQ~#=t2Mb{nqCKb_mSsf^{vSP6%{
z9fA)gDt?*w!)B7)Rt|Vi3|A{nL}K@N2{zSb`v4YNaV%#_^`Se>k-fa5)Dk(v9<<s(
zBkgjjqYtX~D*amTmo4vlEAN!P!H;Nb&)F^Cf({|{KNroNwsUCSt7`(>!_qb}`RKB6
z2sF>FEM^D@HWy?iezB)Btt1c$hXYol?$6~~Dr?VXlgw+^D`-eRZrH>Unk<s8J!iGJ
zTgv6;z-hP&<~^Y(+m)QeE}=QE4I&A(USX3X6Ev|>OEx(;ZpvEc$Fk?%a81_3!91n}
zC;x<Pgnz}Sn%x}hjDxSxn5CYdgG|(X{I}|vZw^EM@CfGS4Z?8jQRk3Y@5fq=GJ<^<
z`u=h|TcDYpjz!b)rjK8N2S>8h!8TmV?IGcddJ8ets)F*+r58!5Z9$_$s-`cDpNm`Y
zmp0arXDPT(WaeHX)5I<g2GG~&i5=`VOR$Yjk$oU&Cv0kQg|_;A;6r~z6&pv;CS+OQ
zDr}FGTT+iZ93GnH8W4piGzABOa@fjpYAT6v?woUduv#94TbSoD>a7`K9Q|T9eY06$
zPhjh33uV_)i^)>+(^^YYDym6^eCxe<aplfS-k2@1?VqEJW{QXcl3$*P==L2=Wz|`I
zg9pX-Y4b8Ra2rSH(39dF33E)X!w4s1zeStXN#eDZVph2-h?yJWG|}3&a5xr8tvD--
z`X6S-1Y(6w=6$*mH=wz$-iC19l?$Iy(UXnoE{qn@scv|AiKc;WL<ly+)z$XCW}5}N
zs&MD1Ra)KpC5IEpT-B4UU9Y8~|L`^Ls)4M=z*E?JE4QA0`SCz)!+i@=xW)EvvUws<
z7iTItA(4wE$2hGuyC6|%9<lJF-4P*0{?x>oA->Q>{;MNa7zogQDmoV8NXA*SpISej
zmE$VlAhN?gn{YDRi?w^s?^^Dh8teuabZ2WKQL=jvHpr`n-5hA1=TXK1**de-6I0fN
zka^sdZ#xu=ie1IL3Y4j9*0A{-?cgW=70VO!URbTZ5E2>N)N(l*>>+4p`_m&8xC<6g
zo8O4~^Y&Zlbfe*J@|}HwO0A#-%SncR%CUF4Pjg>(5^EY$0l81Ar_EwPH<1vaYK<9W
z7%#KNpiKa#pJ=}&&J51l)ll|_9`d4&awx{uf_|ub+!td%o~k36O`}`o3?J;HZX9un
zBY~M=09l=16(uQc5LOfy(sVn9curQ!QGCMaEMKlHLI)S|_6*%mnPG4ATY@Pi^|>m1
ztuyiJ?TuzJD(<mO4j;5N*RB>;L`Z@H^4SFV8gfKn;oA<xk%1J<5GW&17{yh``D*+N
zZhWl7`=Ku|&_P`?VGX+^&>)*gN9^@Hd{GIKh=r>RcjnXM<hrw!ax(b?58e2%O2Uvc
zxo+q3M=k=hbZ*g2Ov>K^A4S)d=PP&iwLNQbW|E5_vvT@oP4S^JTuueyoThSy*D|ST
zEZq79ZKP3d68w7n$2?SEZHixtg#pYBM*v&um(U^HnPOXOrMkZ7x=QoJkIGmb=166@
zTwnEx%$JovjT1i};Pd)PX${tRAY-%EYa_z3bAfStgd$0W!Azr!X#w+{=R9`;iYtM+
zEIa41WF(PGXzDM~6Uu|ZQI|DeSK7KCb0@=5(zBT@!A+md{W7=EV{st^enOzog>CDz
z`5-^<8j2jbL=jG;l8w;4tKaYx)2(=dD?I!<wX&on>c<lwFXC}HMxH(maP~Sw<8!NF
zw9wk=ci}L+hTo60;^Cs@!77VOa4_jVe^*j0%-=5pKTug7?`a+P=9RE;4Kq9@J+>B+
zJ*Q!-le=QnpcPQ@dXazNU)|tIF(q+*I9JmuLd{x7Sa0SZJ*?b1gGUX7UaC_FuRYfK
z_@@%shpg<!4;UGaMHsa`uG?A)k0+iopHP}E%rffET`yvNso)Gp)|sddqCy0ND$gYA
zXs+X7P#H5#0lm1oD6_qMGA1VeB{abl#h(2~e4k8r)s*ohzlO9M!UVioMuY4yvq|}c
zxr@pfSZ{nVWWbjKY`HQh$J6hL2mriwBv6(khE96lJH^qg);#O5<+1lh%8ZxY3UQj{
z-qWJ2rF*Y`8S1)UM}#`hhA*^<A~7O*ygalr)xn>4m8{*L-#0UoVS6P7#qa&zXE`TP
z2BY(yAFMi>=PTC~x>zE#d%u^t7_RySQlW#bHug3+I&upT&py|ar5sy~-IK8Pau+iv
z9*-~SG}^AZ{~@<`uP9|%Pl=F`FvxBRS;m)>c|h=BXi9tV=qyOkHFE@x%wGb_sR!e8
z7pF0YiCa}H*6kpQVsp1k7pBZ;>;BEH)*W#nFl0#6dpH%56&5oL(SwIa9{us6XUb5k
zUaYR(u}rwmf{NM(A4T!>v7K{fOc3s+3|DUUGd7O+fMa8GDAPMth<U*)D<GLO0!ko#
zj(%oiW8(jO)c!V9g&ZpfN5P1HMCMEI)xBq$+r_wxPpth~P^WRMBNT9%&;ezPxJXu|
zCPxHoG*fuBPLr};Vw}m_q!e6^Y>iNC&1sFm<z&fQp+|E>+c2oFY<1AZErBFqRV>4Q
zr?l@X2N(;gC+WE{>3tD}P}W=o=fRsEjjx}9MR!vrYdZuoLP5&`E^H^Hm8+p3&-0N2
zo110~i;*m9VlH@aW>Z=P@?8Ff)ZIDG**BZha8bKoWt{y%y(L9sKk2)1O-3$Bt1d+#
z&>6$|tm#CmPbVkxzYOgu($!^4anK~zlNS{HQCZxP>IP;RTg18uc&FAycX5yb%yk|W
zM!oF=cQ}|))9gdPt*rUw1$LADjHMSBWUnQ2F#XbLGk+E^e*HHBCj<w!>IE$6$<3At
zV50!QS<Pm9V#xN3h{TvDh?Ifu=q^u~jO@u++>9-Zx4!B~d`%)KAfOPmg$5bK-t_H}
z!&w>hyTrxU3%!H}8zjhQzoXTGZLhJ4z7VE3=2W(N#a>^W1`K@T_QV%g^{wzlZ9?xg
z|G`lK?=1#NKm<&cNa*Ni{7KpTFkDpQld#nd2mSYF4LMJau?ih(c(*tCx#N4W)w&$2
zg~L0($X{O^%KPltA)1n3FWbNA$A8?O$sy#BF9ie8FhG9zYE+GxY-{_gjH)YH2V8m-
z6yo<c#d;B9Xp3i)Yd0T+bI9AeYq8ds7{V7zf_6jM!e`bFq+Gt%fo%}m2twe(e~c`~
zsr!)xcwppcbCLbqFO-v*bCL&KoMlTOHr`W3UMJbKFh2~}sOcy}i-vbv4^Azxn}{Gq
zLCUi__@Ar5qk=+H>)#J7I|uw}>aEX2Y{c;D({<G!W&_u-J8>^N))%g_B{>Am)BdIW
z8C?$J$>Z=x3s3Co8-!i=mZa~NoQ&ZW+R4t-Dc?e>+M`{Lv6L79FT)2yrpkvz;B)%n
zM93_)B~lW2*^h7!5v2|mCeJ-bUh%o$+w%uwco6(KY2~IG!9?flcVJ^VjH5}U&Z<|(
zh`54jBw0#X&U1E!aZXI(4hFCP3Z{U>##NwUiQ7VI;Q=9lsGTq8>ZKe1aE-A*;?HEb
z#m0I?l9P!dT2%u+F&qvnj!Yt{Fj>!y+7q#<KGaI`xHY|Z7&Fj>ZR-(2gLUPf8RaEX
zCsE>xfVV@dx947>2ofGh<3&3+ZH*fJbr}2){BCD3XJ$7uwa$(Rh<-O0-}kovi1M=(
z4#3?JU(DLq-Wg6r)abME4yzIa&K=vo{ZIhdDGyXwo?{-osJ>g|WU#qr4}$W^R9~*X
z3B>+P!c{NtcpN%5pq^gJ%2*$V!t9W+Pme{iIttt7=T-wk7hxRK)jSJc-3<!X^4MLP
znHcF3&o{L_<?m)+xG2C?gcb>0cdQ|>-~Y5<<-ll@<L%!?ek2BBnd~m|B995wrDxWx
zv#`$2z&ToD<(GfWy!ulYTsz9V05dT{0+-PR^mdztdkZBP%C~p+htvp@z8@ZhbKZ_3
zeU2y2$(SRquQx>`m~#cjO@%pc)B!O~WZ>D)<6k`^!6U^}#L~Y6aZcq&k!P(0VWXWJ
z{6;%`=3%B=dMIF>r6liXvX&;_#%N43(~2%&yL%PBhG~`_`?CTlZzkR<KOR$4Fni6o
zAZn+*Gz`<y|Bp~>hJkh-7FfW`;`zbutmD#FkP4hBVnUll6y|O4KzIxh&Ep{#e%z8U
z2U3{IF_F>r!_A_*hnRMNJ~-DmTU<91N~K@^Kpp6B9d8(-XJHJAI|i0>Ihs@VRbR<i
zQN#l9n<7O>)}(k@ueV0cS3(#-uUsQ<7YCm1l~LhjGFldE%kk#OAx)JaOc`g{Zbxz=
zakrW^sx{$4Gx9pToHoiatvKIOfk&TfcB0)YXf`UiD7mUby*j`ZyN@`9A}JAQBxd@v
zU#o+To-Q|bzWxBIx*PV<e(8Qy$uHEtyyzQtlu6N2wdZmXI;a!rEY)Cm!=s=HpWSCa
zTvT^nU=~s-FVcF}9i7`PPcnr@P<PnxGiQgygP?VLLGZnj>gy+|Ujg}MEgkfT)I11)
zU{r0|t<Au3+~d>wy@K<yeAo~re<Tl3Svbs?Q+GJJd}tGjkk>NP@md>RBfE9yFUy+>
zzn;SA5*3939mN|E+N9z+{Mg`Cb4IINZK~w^wTY@%)=DX@u(qTzjq2xyOh%WerBZKa
zB3_lg%M>>sv(VY=wCMc-YKsh!t9UVXdcn?jF2O*e(SlO)VmXO*L=YNvTjGUinSY%K
zmKM9y{j3Y`XI|>v{BsllSF-(7EPFEM<cSYV@eUXwd%Ydb+cn6{Z&J%MjYcE1<siDX
zGA_vGeTsV$Z(5T|OeHz7P{igN?f-`dF`Nloxk=R{+89W$$r)FO{gtUoD#ge=D%jn@
z2X%<l#y1G7J&f5sWn%W<k-UKvaadq0xvZBr-)ldPy+Y1Vt=&jGrdemln@x{x!$XWZ
zQLp>zN#=yjN<Z18Y(}NQ^=p%gq!1FwVq(Ms%b;JPwVSLFHRDW0K7k}K1a`rcwW%ca
zzu&#>Il;B%!I-7Ww~I}DvcA$Dz)*a~_#Ui*?A`qX@MrGS-;g`t(SVkeR#iX*PLszZ
z{r2Kcl6gWYhZnp|kuAa+K`#ot8?&?&BM<3s%3lN?0CLua+eDEgq?0c;s&EFMMfjfR
z^dHJ%@aPiUvo(sJkMAQqEw(A*+i#^)|AUJau7BzkMgP*NZP^=r{<6cbdxxr?nO@tZ
zZs$?xoCnY5p#WT@P~lBb(1=^?2u>&S!##+DE=D+-n1bSzZ1ba4@dky2q@L6=rpnwz
zl2!V%D~Jr^KAvva$NSBUicDFoalhi%`5}q{Ei_`q@9)X#M3ykc*D=m2K6BNFux1g!
zov=^GhZCVs3O#*{A;(X4{*Wq98Vn(U=1TC{(u!RZD~3cqax-7oS>2O@1x)M1+EYz)
zC3rX9iten@;p((kxJ%H<zmcw~xtETZcyt^8mpH#ZYUvj`!@YJhYinz{f$_pxUg14H
z%j3pxCDvk1UA@B%#QqxstLI`b=8+pt9@J_FbbZ)4==KWn8<LWY_;o7w*B5@Q^hjs3
zWeB^fm1R?UKfmpt>B(JJ!QnK$_Cn^RSce`hio{vr5LU&^2d5$m?Cb?>TSRT1gQ`Cr
zwm5k}ctIlZQ7o@<d;aiMUr4HCuM+Aw75-AmbeZC&cH@@*C3uh$>R|2Xh3<#KFo6p}
z<)#h;G2`q0=@iEN^snbj0urK#&{x8a>!I*<spLl7<y1qHxQ~qlJu;8#%Xq+V(rR}F
z{l}^s;pe;5=DiR-CK{^QJoqz3=~}FJBK^!q3j1Sj&Fj+2qL3VT&{vFK{nbYnq$#8*
zr|V|ALU))BT%xnzH!{2YFozVKTWxf=$=hP{MNpwn)UwfZrzmnk9o_O}4fcccq-(B$
zakk<y<JUzGF{$dvRYxqET!>b`ozZ#x&iJMMCpyZy{9Hy*8tT5tQPhe2tk1^^6YZ|9
zYyQoo%Su`q)6C>AG&9Sm5D!Q3pAwe#KqYN?L%+d37|vcNdGDv29@4A1i_B)upH}h!
zr6-&952TaJyX3z~XHn`ls2J$Xs)GcwarnBc;6fp32b~MWmZHnG=;yd~6t3t;HZkNZ
z=8wlDYX*JU3jNF+G36KC`Wp9EJ)(XSRpHG-{Fr{zX~<TEkxW=?v_JwnC0!P!Ptq0%
zc)_%*V}V_LsJ8O|DcOda{Yd`zu~Y^yDSBZ;q&yM}OOM?N5sK$sqIQ7yBtF8rznW!)
z;`)c$XS~@lMSJVa=9#ewOxVhH0;ai;dF^l&_sz{FW=YAZoKecDkN|=dZ3{#*NM`d%
z>-?j%j{i_rvP;MKm(CJjYLqR2MM2fy-2t>h43U<+hVGVsIqqz6QwkIVkef;a<Idcr
zsI0)dS~u&!1uN`tiepU+-~KCMim}n(hI1!Si;d|lk#*~VGkAMudF~`}R&5JGTc28S
zFOF8((LDum$gPhUF-6@-Rs$}|IU2BtY_<i%Xm^K{T<}(xd(7f^{VIg;!x&GUz^cY4
zXA1k_REjm45O?nlx%>|(rjpAV?IY4%+Gt9poIW%si(H80WKWQqkNH%{pw!CYfFiu=
zgfdj!CevdB!J_>!GErB7Vc-IzaIr2kI`n*O8e7MT4*rx?OrfzVZ<4l=QFQ8#$&cC0
zl19p-dp_{oR8r@C8er;WcH^#_`P?PPOb?YNlp4Jc7ey5bg!K%=Q$!6hG9J583bY0P
z<K*rb$c`Si3w=k;fpwI5?Qtw$so%BSJJ!cKkjBe=ijGNruX!_Z=KMme$x?`?(d!=X
z@q0b!>O}GNT_$CU=gSMl`pSlWzNR43vX<b#_by9HcY=GB0@p;N_rE=%Eo%_5fX9R<
znp2aZBlHQ98a+16KaK$`E6L!;=ar}DfG0*#Da^o$EjbMDABawKM(x#_UtPUAtKO!L
zWMP;7w4FsQua5)2PEB}vdQI+-i`;}8#%P|TbwQ;#T?Z!J@stQt1J|Dc!|~o#Q&y?}
zn&4e6U?we)XpWl5zM=+u)rJp6|A^DG$Gn6kb>A<3ojh+1#w<fK1FCyiqXXVryvpM)
zuVH&Yq1^oc=CSx?f3SVGM;iJmsd5<$MKa5@Q23OD2RxCdIU=H)u)zC&6gmk>yk;AI
z4sJ$e9%ZzF0J6|4(Hav6d4wPOqiH0PuOnEWhO7`cAKBv&tw9c(WXjF#Q`ZcCf#K1V
z2F{gR$p_p870f_$3sQ5MzN;iG6e%VQvV(e6PhSv=s|jV(!ME}&H@>o4F*Fn1L&k=p
z_Uh|*JlEm8U)zmc5>r5_nAT#c&*BZMlkJT4QNgx013pa4cyjG=iS_~pd39+6k=v;V
zfT>-9muBT~?1^~}cf0Nz5!%93eyZGB15px()dQBmhygNRj#P8r{+bP`%+#r^wT0Gh
zgkHpJJ1LMoehAmEKtPq}36jv~wT1&f!ds_<!1_-RUZAQ}jytK#W;-S0bBLRDxOd%n
zjo!@5B!pW1JHg4CmTzL~gBN;6!DmT#%{Lt=B#<f`0P1_XJ7Sgf<jg$;3_#$|zSyKZ
z`r(GvBR1Dp2ZQN}n$IP*eChCi)>osuyRp?na-fpNr0goudW}~(KwqRY+$K{j0D#B2
zb*dxNM(_t7WMyVmuNPY~$4cz}TVlk?li^|oN6P@oI=mgXE4p%7!jp*GP7#O9QqIP|
z+&}lxgfK!x>0<@Ib$F0i6K5JfnkXG+%#hiW)%uPi_Q75IR3{C*5Yl;)Q2*kMu}Z9G
zjYg(ou?u&Kn}{IvGI@q(R2m^}X)xB<&)Z~Um_HcKFa*q0hd7&FJZeI@$L9Fj;}lab
zGy9zeAN9Z2>GK(!j`ioUG)E4@2lk1rVM7Et;x6{2PYdB#DfGLvH*J?@p^%PClMpaY
z-RmqQ3k@+lj$A0(O|7Y4*K{2ngmP|c7a3p1N&<voD^TpHH}o6MKeuG|*&*>AvI6qn
z@g#lgURuZQJ^r)Bg4gyGA<3@Z_y`#5#}(<Qtj2+Z?tqE!Gl-yI2rs#-RyHy+$EOr=
z3kv%`7Oigv^>_yw1|Jz0UuozFulN<~RA#<ha*m>(f&&XO7J6a9VPWt<IVp<QztV`}
zRnN>QZCw_EOBiQlvw0SLhHkv}3UFKrEXutP{;@vu2<?3vGn3^ozRg!m4vcXM4mzY)
zWGFVmI?g$zDAjjvm2-XsV8^h0VcP*MQr@yB3Thw~4meg{@{Fe?#x1!3*~RtS<#S+j
z<^Wh!L^^52t4`XE{h2n#vkgPa7XezvkRCgkyiU*ylc2C8=&?DK<41vsop9UGTS@8P
z!+5^loZXvC0G|t{65jCgV)=KrDOzpf%BT>4Ex%9~voO}27w?)F!<W#aBW0)Ouz0$s
zsPYeuHlo;nWG@{5*{c^p2hlXKGc*MyvLwa{w6(u&_K>lFUqo!%Q6Scto{WL#VGv;|
zg@;lqgJfjA$P3$T{n36-6(@#R*Fyl~V)c(EIdDe}pN?GSdw5%JL<RlhL}jgRSiIwI
zJrz8Ny(z-QcDjA~W5&j!snEpdz{y@1LtWP_b(!o?;N?5~-R4F(mSDB%oMDt<pgD!W
zeZybs4zFPzrda-Q_8pgmy4tmapNC{lu2|(|RCEFoF<9pN9)xwJ`O2-s9Fc|P2WRB$
z5(<=cN#Ouxv}~w!H(qaEI6$<uVF;z(O(EJoRo>(zwLqb=)SJSTjZC#HNB>xL0D|&L
zQAwY;#(=17vK>4vG6x%<tvv5c=<{Hx3z=m!Y>^p&;D$FKfhZ8J;FD~oZH3Cyg1+*B
zfy@I_Tc3fd;uJ4^-$MBsjuW4^Y<?(G#px<@j$Mn0ludpA)rREarSit%ThF##y@Y25
zFg0jb)WM|Ii{x9~M@iT*q0w<D3ObJf%tZ~MLjNX^D8EK?`5%b%k;7%5q`5bI8HtLP
zMG!&)ho7FzN6yhJDHM|{3eq~MC!jOC6?Q%J8}8chbBF=Ix>%r@n96P~h|!<OyZ3)Q
zwnK98X1zflRxgFunu3F%u3@h547TGmE-I;gtI(gJzs8@~{{oUjT*m+h*`MJTXwgK%
zpK8>a{e=y>I@+U8fiIwhOtDEic{1&ozM}$i96z51n^ULYK{E&tjI|Fddg{PgIU)9|
z3HyEYTfWG1vR)4<gWV^MIrV@Ovt8|Zkx|Ec{~=|$+?DFNR9M@xv<|-#!)PIS0KWGh
zem5Dc>CDB5^Q>NKDT;7%PFLvAKdGrbtaG&83kNIsb$GCbu=Mos&f?GRley2i7Suxo
z^}pLfWkSrv^vam<Pih~vp0zdQpx>^K*A=10N6L7AAG=Sx2-X{ez!DT6nzQj7C)z&9
zA+ZFBXE5^;KmkqYD;fc8!~N{RHAL#go@4u=?Jby@nl)Es>1S4`3=?m7J+d_WQ~UH?
zq~^Zxb?!E&t=)H;Z1foVfBRt|AjRymGnNs9FiU&li4&LA?vwfJdxZI_V@HGr-*%J<
z`h!>nmaz8zG!c(Parbwj2%Klkp(WhFn850M@b;5NS_f^YsexlEaxe9N_v~~Ep|8X~
zra~JJ0ak%O)%pKmnTm7*kjLz*WLr}N`<=jfk6I$j2F#unck!#_P-8R@Hke=~V`AZS
zphd!c)?eXo!7hn%FK6kp>69Q-Wu~p$dOp`z4E|}+%r=W@g~q}jgr;H@KC;yip|zer
zzbk9Ajn*on<&LDFhj}G2LCVl4l2e9+k!iG`h>Ss)Brs+1Z6=_uYJUmb)O(bOl0<L<
z@<ny!a;W%$?)dTL!&K1{mJl;5w0V+V{NO5{uNM$xfmLUE{~AWw!$GWdaDf(-7H{gy
zS;xL4_EPc|vifQeJ|2=ykg^|=z?D;<*};hu@!&Y@T2g^B+wG}|qfJ}dK?{1fkKHRy
zaXF^7yxmfckrZn~8O>Vi6z}(_Cxw=|y*>v};0}%F86)x}43<HgI<ja<*euwx#Lypl
zPG|G|`WVy*@0k*`UJLRX#UMHqYnTa`eFO$)HeUdp$dFS<k^{Nc@Qk#sz$!kOe}z>?
zjKSOxsSqHybYGlz6B)G*1LIZI<zn$ZV(yE*&ewlP+WSJkTT#rj*_Q@4V)Embqg@es
z#;7|{JGdfISXQYM{Po8no>w(Il?twZV%^6GwEINyVJmfVVJc}%3-!t&QkZ_^HF&Sc
z8aW=Dfn0<KyA|5<s>wa1Ynf_%KX7@toGG|6_8FVpjW#BwMfwuv7JU|T5`5@U6K?Ck
zyX2h$lxt>+WN_LlABUR~<Lw6nzBG-I87Q~3Mp+YFO&4jp`c(>KVlk=5H2=Wv8ZkPH
zKuhbDwcM>2>F3p;!34vDnE9#KiEEPh*?AK%glW7?CH*8rfP-}#98`z+aa*%8bmAC$
zy9_TlRd1Uzw!O0IPYxAt$wBsi#f#<U^{P=XUB%!i9_7p^=7vu#svP`i?2OuiZ%b;<
zq8a)#+XfB03u5p;^WtU3L0g8Gs^A-@U&fIDpONjD2zC3LJKn``*HG#t03};r*H<G?
z^9@ukNiU6&&i`)cVKUd3e7Hq992V~4fyDEL^Dab}{*5y%h`EgtvVQXXKnYJV7*T&d
z!kbKu0;YgwYBY5M&vdSK$ghnNz>%wMm6&V-;LH3R#TtaQlSS>0<wMuOw(-SkKO}-^
z3@9+1W}Qd%G=1e~1geBDCWw%x-ve$%{t>}u0^NNY^6wqw8_{voemLaY3I4xbfZ2Fa
zEWX35j}9jZrN^n9BE2Zp4j}VTvO!*aYJ~sB7xh-0byGw7ow?a~N}*t`5z9<PsQTC(
zQ_!tz1eN!RL{l$4L)Tp^Q`Z9`1yVviRgitL5Zsz5u*j_+vN2{Z=TPcj-7;Q3JM8i%
zW03widKHEZGN<-Hn-Ll2l8ovTNR!(}q4Yg1u)u|u@xhiBhLfFJt^PC?5N+=y>%s%+
zh1Okc#e5Zn@z<ae4E?f4v|7ju8a_~j4&O(yBCMsl__Dh`iZzb|a740Zuj~}+sUuI#
zmo&<HRPUC5mZZ>8oe{2!xFw=Djf4<LfaHd1?lTs5#-Edb?9H(6B?&SB`)58X1z9+X
zvyUK8P&tafS*=mQf0nQIrJgFmo<Z=#+P`H4f|GjT^lLR4!>x&taufc>VdJI>Q`sWb
zv>(uCeqabC@L9KPw`v%Qx5n9vvSf1wYni*rc74UQ`EPLEsUbAC0V<2Ykh}Ouuj5SG
zrSe+_jiKDTR7A-(KyGiN01EaxGC9wX|GZT~`mORw%s)0Wa{<>0uBpC5e=@XPHuVT{
z{42B7_R$0H<<3e~iJt$7qAYt$NyU#$;MYDaIkZWwj+Mg*uz=Y_&{Pzp3`nL45NoVl
zHKB#yDq>oERrBVVC47SkP(DzD?_;PX3WutTc3<{?{#E{37leb2-JTV34odjcUEj-S
zxZs1<U^smO_DEkN6><O)wgFrECY|tNWj$zGtT1n#FUull^}dYz6gVl<9DqR9^}aMC
z0XIp#j2Z_z8ojV8b{Ab6_AIr8-S}sqfZY7&cl>u9!+1(nB0^LU;)?XUA;=N?Tpnn!
zF`)#yL<N8LSPwX()WduGC|vm7%kq}@w%~Cfo50a2_-I5}C)Zk{Ff|Apo{dKYC<#Ck
zhrcTeeV&!}mB?XSw6>LnCqLJ{C~7#@bE~@9$G$0!JQH33l-TDak^%`ra}m+P&uv0N
z;Lpu|qv{jZQMEGG(CSq%Ehzm#T?0P1qu+nr0o$xTCAxgKzrH{hv~XGOKm!u|=!z$9
zWE69=X;G)fFoq1lt%ul;yLLtuv@k44YJj;%V$tuhV(E;tpEC)X>pyMm|Fb)cTvwdE
zW-ke%8sx<jMqMXwqG;AXPW~<jClqMU?2H0&FYlU?4S0QF-C}CHJ8qtK<&XW6h+VmY
z=(t+hb@#j6MYrb{iif=M#I7~aHxOp7VLW<9)w9qWuU4{GUoGLAVieJmt48wD$JsDI
z7y5&|iIZ-<@96<88ws$Vr;9UJNtysgKDU$c6h=R*pQu);?v(Q+)D{|uuz2lO|1XPx
z0=7!Y9#9ca+t5MNhq%3ej53hCSD_76cp>5yts@%{i;exr@}$KYVcnjSa+s{SZ{pf{
zv4CkI0J;InL3h^`aD&4Y6DO)fHpC0B!|D9Ch*Wp9o&68_;qEi4I{Nt#`6*J}p0l#n
zh?&9p;at^ErJ)NeO~r($ngfz*8kXAC9#wInEr}%)mAjHJ1HE}M_H<a8(S49W82vOC
zZ-px<^wvo^ivU6|OyX*~%u1ZhdQv|Z`V`3`E1qVGH~A}gcEhSUMEyjcI=C05hGg}K
z48mkTT42Pwpcm3R4EuchVoz3`DH<>TsetOpZpRdXa@^zB!?r_!`)k76Cu29Am(YgY
z^4`Y~cu*}j_Ku)G<3JTlod|UrZ@Sr7fuBiXeRv?2kr+Y&W7o?U=HG!H6z4C(%;<q@
z_L8|7GSq8&AJs%DzM3)hOgbB=N%M5~+{<b8=)#>GK3lD9XJ>Br!j4A~QO-Y++rvS{
zkD?EhR6qtC`?JQ~M#2K-m)0MreFZg1F>tt2`?|t8Lyj|rWVLaQ1&r@R{sqNZcE%hu
zS51<xtKOz+I#K77?HH9jygvmgw;Hd^&_|Spb<AfoS<V{&@W9I4v`hK&_I=u4lgLo3
zWnQKOPY~z3W{V5Gxrcb9`Ao~IWgd4B2~<gSGw>PquW^^u8&hZwer*X9w}b(5>tu0O
z#zF?=)M#{VSaaKqoLg|%d4I3r{9s6vXnpqQcgIB`YfH4`g$+6aUxe^_@Oo)s71P1J
zz`wvQqMGt>Vn`cp=zYAI7`^zjsHXWnywI`>-;}o(gbeDSn7q5>WWgiRVYE*Y*j{_8
zNx4WeML^Cyjc+b@Fow3YMEXk7B(vs3c4RWaK(oJsc*g3i{FAHx)q=E*u&EAR6aAQ^
zp&{4vM;t12eJ8sNg@nh|bp%2=-0A*$)>eXG!*@GcV#vN9+cn>obWrr>PUehXrA6^;
zOc%sX_1GIA<iWu)#7OlrVC(>NjI%fU6@RV(R*ZX%wPCgUHN$h>9@NEJ6LnI!ghKvu
z_F081!y|!Q5MwCE6)aT1HtAy~P%9IG2-S&AcsfP`IEf6H3*_;Jf@v$LpFZO6k$MgH
z|MivCsS&W<UOV>QYs45@tlt>FeIp%XXup=2zHjNhuu9+H*7|Xr%W=AqN3YJRaq!!6
z7X85E#uEobdi}M2B9xL8^{+d8juW^nqmS~iE3`Jhj#KQDtQX4T`{QeE)<Lta*)pBL
zCl5#JQjvrPwKMqjTz^?vm67zV*}cirg)uVU5Ysk}Qwgo-n0Joe8XqM_EZxUNKQ|-g
zBRE)ID+)SpCFOcrI9^F2rrTcTT2#6xywE1b(oj?shs9>&=dJ^$o2i6%ot2(;D;d07
z2@PWIW;PPj`KpJr0e`rJ1iJd2(8-}q#4bV#CRL^AgrF|&n({0L2#v<#+;U#isp7(s
z5&JQAhXa3L7fVgl+1W5w)VULsRftyh{iAtrmxrWL_B|966OXvnf#Qou5M8{NOy$9p
zR2zoic3&GXX@MI7pe%R(JHqBGF4W&8b~_*IjI09VZ?I2Z?~gZ9Xvbc5*%9<2l@X^p
zxO<d1xv|f|>yE?r>c7ac5So}SF962oT2ETf-D(6kd*l+=S>Vh8m$MrvFo{-14=#`x
z8oh(H?ulS0_Y6IdCfJ@X$F$?9lq`t^uSIvB!Vn|}i*j$@kENoAddEKnm@ac$0q-7Y
zVALKA2||`QvHW627*N#R?C(K7nGIF)Kp|A|CaFe)sHPwh%u8&Wjc!m3v5u%x0Dp-<
zR0M4jX8M2rc4Q^ui*gvamREr07Tf)WHCE5i*p0C+q2Ilay==irtn##HyYY6E<4MQp
zB7(S%cB6%ZN4ka9C&$1@9LD^6Q2qlulcUru)Vb!bLH7MHH-~IJ%>(@BYxo?j;So&U
zKtJ4?dfGzg=J$IVcrw{p>Y0VC|IArP<_%lfD5@5JIQLh6Df*AZyj-bSMdxP)2Imm{
znbjlN!<)^y;9d_&1606{PXYa{5(FJ2{}rKQPt{=*zgF%a=l=>jBU~Jl<CToU?s@lP
zMc&l3xo%}M?_an%tnP50ku2Ju&I`H|iLH>Zpd&jz`4L+M77u9RvPkk(Xo7WM&rex)
zw_DKw#mnYeu1-W63Ew|K?V_4|zx%65eg-+0zBm;iQ@SxSBWs2brV_fs!A_n0+IsH|
zn8wwSd;7|=P(Zl;nbc<LfH^5+!;dH<)d`E^sbUI^n}e9K+~=s6p;2xMcmu;a-+O8U
zS(%iL`w3%9hYP0)2ZcZUfP-N;Q*lP1+2qt9HU<4%<k!u<H?snMW%FB<iv(#dIcXE-
zWhjU$0DA~m7#iq>qh_xwYjxl$S|J%7ttXwDq}aLH<EswI{)JJY`s_X^RuwNeH`El-
zmBSCZw$M8RxM!fD=lqyza^&(a!qv=CG7j1xEJGUZ&9v0)o@2W|9v&j(R?bcGwD&TZ
z_o7>vvD0ibet*!bPtL&DXCcm5^S1F)wGuS+4eZbqMVmDQsIvj%F>-ZB&w6+}T2fpT
zrPA1_i;*2)z{r%(FE<p~hLLxS@{&8ufUnIow1}|lRuoG;xx$3U3+G{@GowKt25b<n
zgFj6Wx#R7LDN&@+avaX0<_Ct3WBLCY0_Zv2O2n_Nkf~CY;*Z?Z>8*$bIyCeYi@10C
zh~H{5?U&|0>bDdAFRYl*kv;NHMJcnW0s4lzu1UV?=APW^pSd#c>6%6#4c|Dv&^<IV
z6&NM9UE<XC^Ey-7BT8RVKT8;sYsjHN+533$a?QI)UmNU6Ye#V(YG(`2(A^!DUBBwO
z9o=L}apK0P%l6X!^h5`~AI0o9+t9DE2Tctc(PIjJRfdlv>b<~Tk>b&^{7wEoOsOIL
z#Wc1*AeJgE@4voliy~_dI@XE9luCkjK;BBKdII983-M6XLbWVUBnsI+QuMxE@`Y8G
zX;H`(I<sh|n?v-DED=|E3Z~D+{!~Sb-Ha+YD!mz4$<eW1*qw^?7xDMc6K8DP=v^6h
zNlwL2=9j(ye!Dr%Scq%;K~67yR>yV{)}8v;CEsYA9TWv}A_HSLoQk~#8-6qn;6$a@
z4=LXgTj_b+l;caM39l)yvhJF(iT(J1m!qOz6!5atvk*@J<=~1su?=74JpB0VTV*Ra
z#EY?N`5H!W?$k!{xIKm5it2q$i2;YH?vGa~ZN&d6GboA@WfklB@2{he&kV^S=@Y$H
zxT|itt{a2=^=`jlqgR|N_lefH`-vSl(bUv%t6QpUeKad{VV|)(sTxeFXyde}EXuOI
z9Clfb$AFDM!Tc}&s-`OqQB(4dm}Gq?B(|3kOP5HPeOL9r-`e7?$!UFNf<YkfARL$Y
zJgQYTCYw~vWH<u%;x@#I2V-Nz&vgAhBaA?^<G)35Vw`QPaUre}nZBm%8+we~(K;=R
zu1{5XX6I`BY;Q}F(x2(7esAO8ub9=lRynt^#xA9rtrPzp=E}l(QDodJMw7sXtM7Zb
z`Lx_WzE1M0AQMVrBA9+Dh=wRR{3adm<%>epVp{OT8V^#+k@fm@`RxvMRH)`}kw_wJ
z7wrpOyC9)J*PEAsWPaplF}7^>1OnWorS+I~8oDiO!dep%+>mPrCl<Qyy-6tP`SF1f
z){C}VjN4fT@#-f(G^7jdDE?n%83wXGR5aVjKX8=GgdR1OY7jga*V(SD+;~0;O^f9S
zNrUBvk0!s!R^<-#1DjMAnOiavXygn+fz{dSSVvNStWTA)5m2$kZB)OB{j)d~yhwzF
zdD^^=abp((-9S_vz$~kS1>;WtuW>XbJZFsf5Sm^c%d>Uo_-yn4n1CJMY-8S3MO+|#
zK@|_#$b$Tpscii3w_fu3x;u*VX193eoe&KFBqug?|L3Fp%(7l98-geK`TvlU_4@|t
z=vT>cB#p~|71xf*aT@iZP6(1h$;6{RCm|G$#RQ{YyFR$WRY0r})nX5s($btZeY?X}
ze(5rsKa*V^Uir@R<npB!tC=8X)E37mxpH!$6hq9<#Xb$!!1Z}Veg*!rX%g%7t#tzs
z^r#k`y2>yeyDtX~=8=n5F>P}nh+{6b+~T^Agew#o!K(}rx6TUa1kVlQUHjfpo6z12
zx@Vk_a9;fT;+BmV=mE~N2yL{&BR<mM%6(VigbRDJQrVj&qfz%QI+xavhRilo(d>sI
zMd`A}jsj$yT>|4-C6CzDH0C&e4!&m!fUNydk&}O*<i=$Zn99`)J^u<hj$#S-OC)x5
z#b+>O(KbB%l0Fp_Oq?3rruQt3Pibw^HKbNzN80>}b(OB+)R)_m6JqcG!u|g7@?_A|
zLz?!d`81n+{L!N9rv`!)0pTCk&W>91C97BFlTjBrXKsM0fP_hAL5CC{3d5He5JAlY
z@Gg<bVBg2ToDX;M(xp(yLZy+Q^R0~{9<fJ+bz}d2@zV!<)MckY_Vj>Sb{Azo&9%Xx
z|L;i{3+jg+*=bWf@835^2OOywdaXpW7Wlp+j%2$?IUb&M#^7W)1w8X;@98`8wv?UP
zP?WX_Ovait!^%6kx$6pbJzo=nrw7CdE9)h1-_h6lWXBu}{1}42RcC0Q8q<Sh*3atO
z!B~SQ?dPRt+Ra<vM;UDlD3@tm-yJZ0np&Uzf=uVsHawvY_It*?AKPwg8~Jvpyjn{>
zIg2{DFQ*T5dnN7oA`CF@v_<@o5d&s4uDW)-dPpy&yVy2H=g}3#wHa*y=2uIzQo{H^
zIn%vVMNsf<u0z7l=M=}tZL1sj0{z93D{eIJvk*R1WMA}^Bjbnw!xoa&rkH<o10Hgk
zKRIoD@zyW$U)uRImV~O=H)NVLupBne4-D^PuzW8{MTU}{hP`rP&L$4(Np`=!ov#!=
z_|+v}4rIz{$bGe_8#%&8@SvHq&@>sM?zUz6=|2tY8#kw6EmgEeXwP=T@#2XfEsTaD
zrWh?@M`-)T-mzs^p?;AhT3wT+wwliA+(4!tqLs9mxwQUed}J-8AGd~ebh?GqvS2~%
z>y{`KC0q%Z5*_DYv7{mD-dG&61j=<RkZ`A7g?mm>dz8i4+b>RKHCp>#3=lh4zz%;x
z$4{+Y`;D`9M3q(v+@cM9VSUdwg3mUWvs<2cs?l?o-i<gY4`WNaSH}DWX4D<Ags?(>
z+Vx-*5lF`*)tMy{2Xh^XN?D@caBuQ%yrWpzyUD>?b|M%qL`>dF$haC!Fm&lTaoWC%
zy}?;N#IJgu3ETAH1AiyOU(qZ6(52+NSU`PCP4jSYZsKd4jk#dLPDRw&QTqa!|Mh7G
z@aRvNg4ecT{aiC8bp#zn#NAENC!}`Un19-IxU3L@hIiS(p!j3165VWVpVzI|^EL?>
z{>e$e;gC*fL=NL)@AJV`>)S!#_+k%H*sQMY<i*bHa@xVW2hseGFh$P(X=;E<NUctq
zXXM9g?USmVW<q+m=%k&QC%vFEsnAHe*Bhj!IBct2G--p^Ddyrfa-j&5-9V7rhmw80
z`!9?ZLj0n=A>q6z0lHVGq!x-<eg)GTP8!Bc2z@2hmLATI;`!lcijDzs@xxJA-{&;y
zWjalS<5m~TPYISET_Ref^s1Al1t~Y2Aj{T3a}kOd&Wd)vM85*{ec8tIh`Q&vx?Hh9
zWGreD!4$=#dlwO3Yn_LO=W1^GQuTfC0QU_<#zWgBs(?GkooD#7i-4Pl)q_jFadLwX
z1dG3#J136LeXJ$3&QE{iPX*?$d=<gdKN$g0MbQp1XR=klxIcHhxdCq&P4>Fas5y`J
z;h`csrnJvo<T7m412t9ta7#ol0Og-*M-H)c8S~Tg%JdyR+wyvl@8Eg-lnhA$t}oPa
zbR?RcEFC18_l2QZ+YMtNav2BOM-R*UsVD+K^rCyYc5JS!$AaWO*9y{SXNh+k>8}Es
z=UzO~i*f+PZ<}cwicxD^xoQmUq+$J72^ev7e2G7}^;y5f3x4NKLHf>LFzrX*#p_T~
z!rVsam5TU@DwlSWz4@uNbr~z-4Of*%$eo8d4{dKIZ8b7&_PYygU0-lg$-^<K?e%v2
z*?J7iB-a+sm1M(fo0D9oJw-Ka+SFb4Xrb{?I)SsTM|?B2ng8$lFnU&G%bdYpB9^hU
z`U`1#^}&GQ3fz=V;YtZ<%xW7n<K=5!or#8|Jn1K3#hDLD>|dxuh71_&BM`CCP}cGY
z{=OZeJXjD^N*Im#(&0M2I$5H=?=!|1v7{@=nTr=Py_%`u#wk_=c#s!A`NPERWGRT9
zvn-ofQr>3dHr`mD4v^1TbxP+=OR5pS=O`C%FkC9OzZx|-9dik!mG8?-5{=l{YtIwg
zA11>J)0Fd|^&OFgLm>Ja_!j)&LvOY4S%cHv_V-aec^4*2y<J>5BiYy5phUwZ2Wdh0
z{&}y;!lPGD$BJz`d$W=+CS26^pFiQRf=U{4paNxT%?;(J<JTA9K#$FguzUlx#Tx-0
zr%>%8&e%qpfCG51elpXu^aAgGtT>8bIY;-ua3g6^kr!*smb5c92vU?>S#5)k)D7$>
z2A!_?*LGUj+<yYwjK^3gm>8t*?=lp_Hv|26)WVu|v4($0##$*QjmjameYTMDrE9;l
zBxVC!^cKiCyR(BoQD~qdcrV%(@)agq99dl&aHVXdF8zm+0Q1FEOo`u;SFHv=K2hR+
zmCuZs{y0z#mzfzVPT7@Y)zD6~;WH`evq`HtpvMoNnC6svt|OA~NksBdSI%QA#Gi4T
z6RjABFhsR0M@pWsq5KGu-F;(|HyD&g?x6moIZt2-zq<kZE++Y^r)^7M1q@r8@tIF>
znjypbG-dZmZUf<}^}7g_2I2zWi@m`owl9$xE(blhCIgYS9uJ>=DGHv2(5QngMtEdB
zE{@l~AsMrP06C}o5<Le}T!fSDd4F_zcax@2-4XrvI{GL^q+M#=>Ay3VX{G-Sp%v=U
zvYg`*6jFr!_~)L8uo^}y!p<i0tns1C5;LLx58?uRU}u-zA8U!bST2En<;(UgN$HOi
zt}#AjqXtIIoutK;Q7MY<u6O4oHi9kJ06n5h7x{EoqN%r^MTt{YYAGh>oTG>#yArQD
zY36+YP8M}D`Cf?ch%-*L^~_tJZb_%hvc?=5JNdj2@bh_~IWO&ZXA)D84!2PeIpv_h
z4$7^Vm2I)Ut9X4hVg!i&ijt1&oUNi*X<l{tJc>&G#rK(D%6%~Z(G4&j^pC^|8VBHu
zm4qRemy)#cg%p!fvdpRI;4H|4vAHOwM%h384m?9xvN`VXzcRJvZW^VnNlPx#eQB_Y
zhEH3@$#x3-!#!A1g?y*M2~4H!3^{g`{No{Ys~$^AXQA?M$U6A!J}DD6-w8$yvPv$N
z*pv_IR3;$vooIL8*6*Y|Pr#2k>&P(?L|*eUmV@y6X!*CucvEQ6H{*SrRmnwkHaMYM
zla>7SUyK_Smj&XUOL3^l9by?&X|p24Bd>#b#DYNfs=u>Ft$uyuh6G{_l?&M}iU$UP
z*14+i;W}kK`%PvU6vI+@`J;z}`LI_(Ud#&P*?o6Bgn1@+D07K^&5Xav|8o79$&;e0
z<ydV1an94?DFg7!-~RYmj=Um``fJ!Ln`Dox2NcQBoM{THjoEJN{`DvE_b0V2Bdv5<
zecdjsOc`OHGIWVn%Kk%5Z0m;AGBm4c!t)a-ZSO~7C&aif;<i3oW=T{K#O~)0B6t2Q
zn6fM4psZY}?mEX`eC(ILWMq%q#*;9~-@rgjLU8q?RZ<$hd*in$`PjMhL3Y%YvV>)^
zA6!@Qn!<77kavX!d#ePu{+=)&Ufc3VL19o^sgHbirdIn_*vvm%(m4doRqZjXja=bl
z`x;ycEBdhBZ@xwMZ{&aJC$8$QFQ}SM$%$92IsHasi!fJ1h`JJ+W2GP^HrwkixGy%p
z6a=@MPb@RZG_Vp@r-&5vQ>>3&uDK?lK*8EeGnAmYF_g1pct3&I7Hf|rY)Mt>P@MH-
zLx<qz`UtjvvM#Fb^A__cM~`W@5a!cx0ow&6`a*M_fxR0-1bJ_sDLGhT&sXag4BN`f
zqzc+++xzp3(n0+I?k@EYDk6fkj7aiX$r_1=RtFqxA6BEqq$^5b@n1-2@mDADHjw~h
zPM`Uzs7aTuu&a6|o<6J)=$Tt=q2myE3wi48{l08s&&QfbrzID9mB8QN%}uTQYBMV<
z>r#u`ogwbH(!iE$VX|^FO+~BpL;MeH*LU1P$k57;o`VbXmxc4K(uK+CbX_lIG0Quf
z@dO<KUHl>NL-r!#6+gGdp*T+wbx*d@ensz199N9Oa9lu}VcJgnGVQ>bNjY7?;#?Rz
zHWfO-oIb0H6!`gY%ok6fow_QeZUEL0v1;uqoAX2>x<F2L?qbKNb`=XCO%^^q+BOUB
zFlECuCvHM^6Ju<7)Dq&+nzmFjQ54p?v4Y^*TK>=Ey`TL4GRa-@_Vn<eiF~~oC5<6#
z{8nV_C_NnFsn$DFsrX9!L8kpqlNZ?-FP-S$tFZa_QJbVKM!n8+oPO_r*_5zf*8X~>
zox{zV*eTN=UbrIfIzQ9xr-lDiR?e4~v#)J<&l|YzL1W?R7l$4^uR6ML+Zny-oDbC|
zC>&k1tVvt?YD@~hv&DrhNdeziU)4RWEYte+`d(O<vKk23Km8DrcCRB*)sBl-)Oor=
zV~$7oR8@7)S}Uv8Z-3)9m40}>(fg0Uc?-ikD~^B7bGT7qZgXP&LDlaJvRrDS?AweN
z{(kCYaJp0Qto{3{sk81+iGLnoclT+V<MRFKYiDgUP@d&4baUOlx%VE<PPXEErO;UW
zvh~e0ce`1_jm<yiIhff^+r36x{gsq*e{xCT=DtGzX}il}7JctD<!P}v>wH~mRr8w%
zTRnd`s;@i|-eIn|L?qwK=iG)rOV(XJRX0yfWofJTGbydey~$e5Rfh^fB6hVYe4p#G
z?YFMo47VT4?ArKU`%0~sbuVbXpkj7R_*BOvPUSi3?@twV9Thnky!7i5#}CO5rrKZn
zV;(r`!k?V;6O{62XO#qOzR<4zV)l^*B7Z)apRW-9RO)@;{Dnm`gFY|*u6OR1()M%O
zYuD~O9?kewwlcXVe#OKCKSbRM4qqshx4z}#|6q}(TF(5l`;xux+47bB-^F{3HSo@b
zt!<~atMNMrrtjIU+0!n3FfXucuSLoc_m$GBsViQWe_FgnIJD9u<JnXbIpw>4QXePv
zUtD+e$4{%8iI1nhmu1?TU+_mL^rZ5eu#T)7Ddvxz0!*@&9<bjfQq{<lJ1_QcfVNBu
z(3rC3LpzeRj=QZZ|7-C0W_<6&soxXLPTrgIMf`V-eif(w?)6uA4m&hm+sfDYcz3MT
zi61_@lLFE#3TN!jf2#6#MvU!}a5F30lG(pCW$O|(Z$&+5wX~R&$yRy%cE5N^#M-}C
zZ$E9=ZEI*A;<~ef&oTP0C-a4{pDSyUjtgix9_hVz-NmK;vqtWe+NoNavB$Tq`5^c9
z`2v66N~?Qoxw}HYrcYN|+q-#w_gyar21X;$S^(i+!e4&rOt-(~UU~U;_r|@Vua~yB
znjd>3KK1QUL+kb(EW01R-!xyM=kwLx)%M+?GDqG>s?1f%dOCZ5tl#~*4&%&I<(JwX
zCTr!+No$O@NwhzzyJ=$OlW!a``#knfKU*mATzMbs^R1DKbT<W8Mu?S(?mKyQ*#*4~
z6_&Abd>2l=^IiYk&N!D(YIiM5SpE(lt5dP!Upk7aJG34PS{DjTz1`m3!uxakn&ADp
zWsaAQRi9&7e0;CYw(IB5G;I%d-*)=sk`Ie*X8xO@o$b0iTE0AXyZHK@rkYhTsX?Bz
zixZqPE+?|M$Av#!vOjXBXYt*hsp%(Hbj-OvbIDfDv+@4R;)D{7FI7IgshgEn&$(3d
zthdF6i8Heo=cH{+xNKxDTXbfXTh|<m31Il4((*$+qnG5%bKI^my$nF$>FVdQ&MBdZ
XVHAvlQ7{Td0W|;swX0QW0G0p%*zAk4

literal 191
zcmV;w06_mAiwFQF&Uj=11MSbv3c@f92k@Qu6nTQtx~@A5?%+WX@dY}TxjNU*wnO*!
z?gR9sco`z}cli?%LUPE~n_U*Uy9*XW2;q#um?=-DB&)feP?`W`EasG@gmEA=PGS}S
zS?;8l&N^X+Q=00G%A$HVH;k3#hduKv@XSAPtfYnQzIT;YptQqW>l$u|HKUPidzC|>
t(T*)Jc<rP?5UK}J6w*np#3gKvJ{i&2DE##^p67Yq*B<6ri~s-#002JfTcrR1

diff --git a/doc/pub/week41/ipynb/week41.ipynb b/doc/pub/week41/ipynb/week41.ipynb
index 880d4e64c..83113fdee 100644
--- a/doc/pub/week41/ipynb/week41.ipynb
+++ b/doc/pub/week41/ipynb/week41.ipynb
@@ -2,24 +2,24 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "id": "fe50dc63",
+   "id": "2ab4355a",
    "metadata": {
     "editable": true
    },
    "source": [
     "<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)\n",
     "doconce format html week41.do.txt --no_mako -->\n",
-    "<!-- dom:TITLE: Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks -->"
+    "<!-- dom:TITLE: Week 41 Neural networks and constructing a neural network code -->"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "fda369ac",
+   "id": "bb11e4db",
    "metadata": {
     "editable": true
    },
    "source": [
-    "# Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks\n",
+    "# Week 41 Neural networks and constructing a neural network code\n",
     "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n",
     "\n",
     "Date: **Week 41**"
@@ -27,63 +27,1964 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c14090dd",
+   "id": "1d2fb3fd",
    "metadata": {
     "editable": true
    },
    "source": [
     "## Plan for week 41\n",
     "\n",
-    "* Building our own Feed-forward Neural Network and discussion of project 2.\n",
+    "**Material for the active learning sessions on Tuesday and Wednesday.**\n",
     "\n",
-    "* Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks. \n",
+    "  * Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation\n",
     "\n",
-    "\"\n",
-    "Reading suggestions: These notes, [Aurelien Geron's chapters 10-11](https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf).\n",
-    "For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "24b17eef",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Videos on Neural Networks\n",
+    "  * One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code\n",
     "\n",
-    "* [Video on Neural Networks](https://www.youtube.com/watch?v=CqOfi41LfDw)\n",
+    "  * Discussion of project 2\n",
     "\n",
-    "* [Video on the back propagation algorithm](https://www.youtube.com/watch?v=Ilg3gGewQ5U)\n",
+    "  * Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent\n",
+    "\n",
+    "  \n",
+    "\n",
+    "**Material for the lecture on Thursday October 12, 2023.**\n",
+    "\n",
+    "  * Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.\n",
+    "\n",
+    "  * Building our own Feed-forward Neural Network\n",
+    "\n",
+    "  * Readings and Videos:\n",
+    "\n",
+    "    * These lecture notes\n",
+    "\n",
+    "    * For neural networks we recommend Goodfellow et al chapter 6.\n",
+    "\n",
+    "    * [Neural Networks demystified](https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs)\n",
+    "\n",
+    "    * [Building Neural Networks from scratch](https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex)\n",
+    "\n",
+    "    * [Video on Neural Networks](https://www.youtube.com/watch?v=CqOfi41LfDw)\n",
+    "\n",
+    "    * [Video on the back propagation algorithm](https://www.youtube.com/watch?v=Ilg3gGewQ5U)\n",
     "\n",
     "I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at <http://neuralnetworksanddeeplearning.com/chap4.html>."
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "a4b68d95",
+   "id": "9dcb51d0",
    "metadata": {
     "editable": true
    },
    "source": [
-    "## Review of the back propagation algorithm\n",
-    "\n",
-    "During the last lecture we discussed in detail the back propagation\n",
-    "algorithm. This algorithm is based on a repeated application of the\n",
-    "chain rule. Let us bring back the basic equation and at the same time\n",
-    "link this with the basic mathematics of automatic differentiation."
+    "## Lecture Thursday October 12"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "27603656",
+   "id": "b2dde1a5",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Introduction to Neural networks\n",
+    "\n",
+    "Artificial neural networks are computational systems that can learn to\n",
+    "perform tasks by considering examples, generally without being\n",
+    "programmed with any task-specific rules. It is supposed to mimic a\n",
+    "biological system, wherein neurons interact by sending signals in the\n",
+    "form of mathematical functions between layers. All layers can contain\n",
+    "an arbitrary number of neurons, and each connection is represented by\n",
+    "a weight variable."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9dbb7580",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Artificial neurons\n",
+    "\n",
+    "The field of artificial neural networks has a long history of\n",
+    "development, and is closely connected with the advancement of computer\n",
+    "science and computers in general. A model of artificial neurons was\n",
+    "first developed by McCulloch and Pitts in 1943 to study signal\n",
+    "processing in the brain and has later been refined by others. The\n",
+    "general idea is to mimic neural networks in the human brain, which is\n",
+    "composed of billions of neurons that communicate with each other by\n",
+    "sending electrical signals.  Each neuron accumulates its incoming\n",
+    "signals, which must exceed an activation threshold to yield an\n",
+    "output. If the threshold is not overcome, the neuron remains inactive,\n",
+    "i.e. has zero output.\n",
+    "\n",
+    "This behaviour has inspired a simple mathematical model for an artificial neuron."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9b37b818",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"artificialNeuron\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y = f\\left(\\sum_{i=1}^n w_ix_i\\right) = f(u)\n",
+    "\\label{artificialNeuron} \\tag{1}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f0c18c43",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Here, the output $y$ of the neuron is the value of its activation function, which have as input\n",
+    "a weighted sum of signals $x_i, \\dots ,x_n$ received by $n$ other neurons.\n",
+    "\n",
+    "Conceptually, it is helpful to divide neural networks into four\n",
+    "categories:\n",
+    "1. general purpose neural networks for supervised learning,\n",
+    "\n",
+    "2. neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),\n",
+    "\n",
+    "3. neural networks for sequential data such as Recurrent Neural Networks (RNNs), and\n",
+    "\n",
+    "4. neural networks for unsupervised learning such as Deep Boltzmann Machines.\n",
+    "\n",
+    "In natural science, DNNs and CNNs have already found numerous\n",
+    "applications. In statistical physics, they have been applied to detect\n",
+    "phase transitions in 2D Ising and Potts models, lattice gauge\n",
+    "theories, and different phases of polymers, or solving the\n",
+    "Navier-Stokes equation in weather forecasting.  Deep learning has also\n",
+    "found interesting applications in quantum physics. Various quantum\n",
+    "phase transitions can be detected and studied using DNNs and CNNs,\n",
+    "topological phases, and even non-equilibrium many-body\n",
+    "localization. Representing quantum states as DNNs quantum state\n",
+    "tomography are among some of the impressive achievements to reveal the\n",
+    "potential of DNNs to facilitate the study of quantum systems.\n",
+    "\n",
+    "In quantum information theory, it has been shown that one can perform\n",
+    "gate decompositions with the help of neural. \n",
+    "\n",
+    "The applications are not limited to the natural sciences. There is a\n",
+    "plethora of applications in essentially all disciplines, from the\n",
+    "humanities to life science and medicine."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4d476078",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Neural network types\n",
+    "\n",
+    "An artificial neural network (ANN), is a computational model that\n",
+    "consists of layers of connected neurons, or nodes or units.  We will\n",
+    "refer to these interchangeably as units or nodes, and sometimes as\n",
+    "neurons.\n",
+    "\n",
+    "It is supposed to mimic a biological nervous system by letting each\n",
+    "neuron interact with other neurons by sending signals in the form of\n",
+    "mathematical functions between layers.  A wide variety of different\n",
+    "ANNs have been developed, but most of them consist of an input layer,\n",
+    "an output layer and eventual layers in-between, called *hidden\n",
+    "layers*. All layers can contain an arbitrary number of nodes, and each\n",
+    "connection between two nodes is associated with a weight variable.\n",
+    "\n",
+    "Neural networks (also called neural nets) are neural-inspired\n",
+    "nonlinear models for supervised learning.  As we will see, neural nets\n",
+    "can be viewed as natural, more powerful extensions of supervised\n",
+    "learning methods such as linear and logistic regression and soft-max\n",
+    "methods we discussed earlier."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6577603f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Feed-forward neural networks\n",
+    "\n",
+    "The feed-forward neural network (FFNN) was the first and simplest type\n",
+    "of ANNs that were devised. In this network, the information moves in\n",
+    "only one direction: forward through the layers.\n",
+    "\n",
+    "Nodes are represented by circles, while the arrows display the\n",
+    "connections between the nodes, including the direction of information\n",
+    "flow. Additionally, each arrow corresponds to a weight variable\n",
+    "(figure to come).  We observe that each node in a layer is connected\n",
+    "to *all* nodes in the subsequent layer, making this a so-called\n",
+    "*fully-connected* FFNN."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3f765006",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Convolutional Neural Network\n",
+    "\n",
+    "A different variant of FFNNs are *convolutional neural networks*\n",
+    "(CNNs), which have a connectivity pattern inspired by the animal\n",
+    "visual cortex. Individual neurons in the visual cortex only respond to\n",
+    "stimuli from small sub-regions of the visual field, called a receptive\n",
+    "field. This makes the neurons well-suited to exploit the strong\n",
+    "spatially local correlation present in natural images. The response of\n",
+    "each neuron can be approximated mathematically as a convolution\n",
+    "operation.  (figure to come)\n",
+    "\n",
+    "Convolutional neural networks emulate the behaviour of neurons in the\n",
+    "visual cortex by enforcing a *local* connectivity pattern between\n",
+    "nodes of adjacent layers: Each node in a convolutional layer is\n",
+    "connected only to a subset of the nodes in the previous layer, in\n",
+    "contrast to the fully-connected FFNN.  Often, CNNs consist of several\n",
+    "convolutional layers that learn local features of the input, with a\n",
+    "fully-connected layer at the end, which gathers all the local data and\n",
+    "produces the outputs. They have wide applications in image and video\n",
+    "recognition."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "20de12ca",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Recurrent neural networks\n",
+    "\n",
+    "So far we have only mentioned ANNs where information flows in one\n",
+    "direction: forward. *Recurrent neural networks* on the other hand,\n",
+    "have connections between nodes that form directed *cycles*. This\n",
+    "creates a form of internal memory which are able to capture\n",
+    "information on what has been calculated before; the output is\n",
+    "dependent on the previous computations. Recurrent NNs make use of\n",
+    "sequential information by performing the same task for every element\n",
+    "in a sequence, where each element depends on previous elements. An\n",
+    "example of such information is sentences, making recurrent NNs\n",
+    "especially well-suited for handwriting and speech recognition."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "48751fb1",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Other types of networks\n",
+    "\n",
+    "There are many other kinds of ANNs that have been developed. One type\n",
+    "that is specifically designed for interpolation in multidimensional\n",
+    "space is the radial basis function (RBF) network. RBFs are typically\n",
+    "made up of three layers: an input layer, a hidden layer with\n",
+    "non-linear radial symmetric activation functions and a linear output\n",
+    "layer (''linear'' here means that each node in the output layer has a\n",
+    "linear activation function). The layers are normally fully-connected\n",
+    "and there are no cycles, thus RBFs can be viewed as a type of\n",
+    "fully-connected FFNN. They are however usually treated as a separate\n",
+    "type of NN due the unusual activation functions."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4bc57cc0",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Multilayer perceptrons\n",
+    "\n",
+    "One uses often so-called fully-connected feed-forward neural networks\n",
+    "with three or more layers (an input layer, one or more hidden layers\n",
+    "and an output layer) consisting of neurons that have non-linear\n",
+    "activation functions.\n",
+    "\n",
+    "Such networks are often called *multilayer perceptrons* (MLPs)."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f194e3be",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Why multilayer perceptrons?\n",
+    "\n",
+    "According to the *Universal approximation theorem*, a feed-forward\n",
+    "neural network with just a single hidden layer containing a finite\n",
+    "number of neurons can approximate a continuous multidimensional\n",
+    "function to arbitrary accuracy, assuming the activation function for\n",
+    "the hidden layer is a **non-constant, bounded and\n",
+    "monotonically-increasing continuous function**.\n",
+    "\n",
+    "Note that the requirements on the activation function only applies to\n",
+    "the hidden layer, the output nodes are always assumed to be linear, so\n",
+    "as to not restrict the range of output values."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11dceaa5",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Illustration of a single perceptron model and a multi-perceptron model\n",
+    "\n",
+    "<!-- dom:FIGURE: [figures/nns.png, width=600 frac=0.8]  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer. -->\n",
+    "<!-- begin figure -->\n",
+    "\n",
+    "<img src=\"figures/nns.png\" width=\"600\"><p style=\"font-size: 0.9em\"><i>Figure 1: In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer.</i></p>\n",
+    "<!-- end figure -->"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9564ced4",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Examples of XOR, OR and AND gates\n",
+    "\n",
+    "Let us first try to fit various gates using standard linear\n",
+    "regression. The gates we are thinking of are the classical XOR, OR and\n",
+    "AND gates, well-known elements in computer science. The tables here\n",
+    "show how we can set up the inputs $x_1$ and $x_2$ in order to yield a\n",
+    "specific target $y_i$."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "605a35fc",
+   "metadata": {
+    "collapsed": false,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "Simple code that tests XOR, OR and AND gates with linear regression\n",
+    "\"\"\"\n",
+    "\n",
+    "import numpy as np\n",
+    "# Design matrix\n",
+    "X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)\n",
+    "print(f\"The X.TX  matrix:{X.T @ X}\")\n",
+    "Xinv = np.linalg.pinv(X.T @ X)\n",
+    "print(f\"The invers of X.TX  matrix:{Xinv}\")\n",
+    "\n",
+    "# The XOR gate \n",
+    "yXOR = np.array( [ 0, 1 ,1, 0])\n",
+    "ThetaXOR  = Xinv @ X.T @ yXOR\n",
+    "print(f\"The values of theta for the XOR gate:{ThetaXOR}\")\n",
+    "print(f\"The linear regression prediction  for the XOR gate:{X @ ThetaXOR}\")\n",
+    "\n",
+    "\n",
+    "# The OR gate \n",
+    "yOR = np.array( [ 0, 1 ,1, 1])\n",
+    "ThetaOR  = Xinv @ X.T @ yOR\n",
+    "print(f\"The values of theta for the OR gate:{ThetaOR}\")\n",
+    "print(f\"The linear regression prediction  for the OR gate:{X @ ThetaOR}\")\n",
+    "\n",
+    "\n",
+    "# The OR gate \n",
+    "yAND = np.array( [ 0, 0 ,0, 1])\n",
+    "ThetaAND  = Xinv @ X.T @ yAND\n",
+    "print(f\"The values of theta for the AND gate:{ThetaAND}\")\n",
+    "print(f\"The linear regression prediction  for the AND gate:{X @ ThetaAND}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7682eeab",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "What is happening here?"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8bbee5fe",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Does Logistic Regression do a better Job?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9ede7953",
+   "metadata": {
+    "collapsed": false,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "%matplotlib inline\n",
+    "\n",
+    "\"\"\"\n",
+    "Simple code that tests XOR and OR gates with linear regression\n",
+    "and logistic regression\n",
+    "\"\"\"\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.linear_model import LogisticRegression\n",
+    "import numpy as np\n",
+    "\n",
+    "# Design matrix\n",
+    "X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)\n",
+    "print(f\"The X.TX  matrix:{X.T @ X}\")\n",
+    "Xinv = np.linalg.pinv(X.T @ X)\n",
+    "print(f\"The invers of X.TX  matrix:{Xinv}\")\n",
+    "\n",
+    "# The XOR gate \n",
+    "yXOR = np.array( [ 0, 1 ,1, 0])\n",
+    "ThetaXOR  = Xinv @ X.T @ yXOR\n",
+    "print(f\"The values of theta for the XOR gate:{ThetaXOR}\")\n",
+    "print(f\"The linear regression prediction  for the XOR gate:{X @ ThetaXOR}\")\n",
+    "\n",
+    "\n",
+    "# The OR gate \n",
+    "yOR = np.array( [ 0, 1 ,1, 1])\n",
+    "ThetaOR  = Xinv @ X.T @ yOR\n",
+    "print(f\"The values of theta for the OR gate:{ThetaOR}\")\n",
+    "print(f\"The linear regression prediction  for the OR gate:{X @ ThetaOR}\")\n",
+    "\n",
+    "\n",
+    "# The OR gate \n",
+    "yAND = np.array( [ 0, 0 ,0, 1])\n",
+    "ThetaAND  = Xinv @ X.T @ yAND\n",
+    "print(f\"The values of theta for the AND gate:{ThetaAND}\")\n",
+    "print(f\"The linear regression prediction  for the AND gate:{X @ ThetaAND}\")\n",
+    "\n",
+    "# Now we change to logistic regression\n",
+    "\n",
+    "\n",
+    "# Logistic Regression\n",
+    "logreg = LogisticRegression()\n",
+    "logreg.fit(X, yOR)\n",
+    "print(\"Test set accuracy with Logistic Regression for OR gate: {:.2f}\".format(logreg.score(X,yOR)))\n",
+    "\n",
+    "logreg.fit(X, yXOR)\n",
+    "print(\"Test set accuracy with Logistic Regression for XOR gate: {:.2f}\".format(logreg.score(X,yXOR)))\n",
+    "\n",
+    "\n",
+    "logreg.fit(X, yAND)\n",
+    "print(\"Test set accuracy with Logistic Regression for AND gate: {:.2f}\".format(logreg.score(X,yAND)))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "110b8ae5",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Not exactly impressive, but somewhat better."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e5f94af",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Adding Neural Networks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f5aee400",
+   "metadata": {
+    "collapsed": false,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "# and now neural networks with Scikit-Learn and the XOR\n",
+    "\n",
+    "from sklearn.neural_network import MLPClassifier\n",
+    "from sklearn.datasets import make_classification\n",
+    "X, yXOR = make_classification(n_samples=100, random_state=1)\n",
+    "FFNN = MLPClassifier(random_state=1, max_iter=300).fit(X, yXOR)\n",
+    "FFNN.predict_proba(X)\n",
+    "print(f\"Test set accuracy with Feed Forward Neural Network  for XOR gate:{FFNN.score(X, yXOR)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "812e072e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Mathematical model\n",
+    "\n",
+    "The output $y$ is produced via the activation function $f$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64b04fac",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "y = f\\left(\\sum_{i=1}^n w_ix_i + b_i\\right) = f(z),\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7f8be757",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "This function receives $x_i$ as inputs.\n",
+    "Here the activation $z=(\\sum_{i=1}^n w_ix_i+b_i)$. \n",
+    "In an FFNN of such neurons, the *inputs* $x_i$ are the *outputs* of\n",
+    "the neurons in the preceding layer. Furthermore, an MLP is\n",
+    "fully-connected, which means that each neuron receives a weighted sum\n",
+    "of the outputs of *all* neurons in the previous layer."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a3a920b2",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Mathematical model\n",
+    "\n",
+    "First, for each node $i$ in the first hidden layer, we calculate a weighted sum $z_i^1$ of the input coordinates $x_j$,"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dacd264b",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto1\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation} z_i^1 = \\sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1\n",
+    "\\label{_auto1} \\tag{2}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5cb2ca57",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Here $b_i$ is the so-called bias which is normally needed in\n",
+    "case of zero activation weights or inputs. How to fix the biases and\n",
+    "the weights will be discussed below.  The value of $z_i^1$ is the\n",
+    "argument to the activation function $f_i$ of each node $i$, The\n",
+    "variable $M$ stands for all possible inputs to a given node $i$ in the\n",
+    "first layer.  We define  the output $y_i^1$ of all neurons in layer 1 as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c33f37be",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"outputLayer1\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y_i^1 = f(z_i^1) = f\\left(\\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\\right)\n",
+    "\\label{outputLayer1} \\tag{3}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8747dd79",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where we assume that all nodes in the same layer have identical\n",
+    "activation functions, hence the notation $f$. In general, we could assume in the more general case that different layers have different activation functions.\n",
+    "In this case we would identify these functions with a superscript $l$ for the $l$-th layer,"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "238b66fc",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"generalLayer\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y_i^l = f^l(u_i^l) = f^l\\left(\\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\\right)\n",
+    "\\label{generalLayer} \\tag{4}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ec349a1b",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where $N_l$ is the number of nodes in layer $l$. When the output of\n",
+    "all the nodes in the first hidden layer are computed, the values of\n",
+    "the subsequent layer can be calculated and so forth until the output\n",
+    "is obtained."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ab5abd1e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Mathematical model\n",
+    "\n",
+    "The output of neuron $i$ in layer 2 is thus,"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4cebe1e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto2\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y_i^2 = f^2\\left(\\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\\right) \n",
+    "\\label{_auto2} \\tag{5}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dd700242",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"outputLayer2\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation} \n",
+    " = f^2\\left[\\sum_{j=1}^N w_{ij}^2f^1\\left(\\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\\right) + b_i^2\\right]\n",
+    "\\label{outputLayer2} \\tag{6}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6e309498",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where we have substituted $y_k^1$ with the inputs $x_k$. Finally, the ANN output reads"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aad2882a",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto3\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y_i^3 = f^3\\left(\\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\\right) \n",
+    "\\label{_auto3} \\tag{7}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dc802cd2",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto4\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation} \n",
+    " = f_3\\left[\\sum_{j} w_{ij}^3 f^2\\left(\\sum_{k} w_{jk}^2 f^1\\left(\\sum_{m} w_{km}^1 x_m + b_k^1\\right) + b_j^2\\right)\n",
+    "  + b_1^3\\right]\n",
+    "\\label{_auto4} \\tag{8}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "57e22c56",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Mathematical model\n",
+    "\n",
+    "We can generalize this expression to an MLP with $l$ hidden\n",
+    "layers. The complete functional form is,"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6e91389e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"completeNN\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    "y^{l+1}_i = f^{l+1}\\left[\\!\\sum_{j=1}^{N_l} w_{ij}^3 f^l\\left(\\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\\left(\\dots f^1\\left(\\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\\right)\\dots\\right)+b_k^2\\right)+b_1^3\\right] \n",
+    "\\label{completeNN} \\tag{9}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8755afdf",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "which illustrates a basic property of MLPs: The only independent\n",
+    "variables are the input values $x_n$."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f1dc0077",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Mathematical model\n",
+    "\n",
+    "This confirms that an MLP, despite its quite convoluted mathematical\n",
+    "form, is nothing more than an analytic function, specifically a\n",
+    "mapping of real-valued vectors $\\hat{x} \\in \\mathbb{R}^n \\rightarrow\n",
+    "\\hat{y} \\in \\mathbb{R}^m$.\n",
+    "\n",
+    "Furthermore, the flexibility and universality of an MLP can be\n",
+    "illustrated by realizing that the expression is essentially a nested\n",
+    "sum of scaled activation functions of the form"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "71fb8777",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto5\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " f(x) = c_1 f(c_2 x + c_3) + c_4\n",
+    "\\label{_auto5} \\tag{10}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96280b34",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where the parameters $c_i$ are weights and biases. By adjusting these\n",
+    "parameters, the activation functions can be shifted up and down or\n",
+    "left and right, change slope or be rescaled which is the key to the\n",
+    "flexibility of a neural network."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6be3363c",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "### Matrix-vector notation\n",
+    "\n",
+    "We can introduce a more convenient notation for the activations in an A NN. \n",
+    "\n",
+    "Additionally, we can represent the biases and activations\n",
+    "as layer-wise column vectors $\\hat{b}_l$ and $\\hat{y}_l$, so that the $i$-th element of each vector \n",
+    "is the bias $b_i^l$ and activation $y_i^l$ of node $i$ in layer $l$ respectively. \n",
+    "\n",
+    "We have that $\\mathrm{W}_l$ is an $N_{l-1} \\times N_l$ matrix, while $\\hat{b}_l$ and $\\hat{y}_l$ are $N_l \\times 1$ column vectors. \n",
+    "With this notation, the sum becomes a matrix-vector multiplication, and we can write\n",
+    "the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d1eb96a8",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto6\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " \\hat{y}_2 = f_2(\\mathrm{W}_2 \\hat{y}_{1} + \\hat{b}_{2}) = \n",
+    " f_2\\left(\\left[\\begin{array}{ccc}\n",
+    "    w^2_{11} &w^2_{12} &w^2_{13} \\\\\n",
+    "    w^2_{21} &w^2_{22} &w^2_{23} \\\\\n",
+    "    w^2_{31} &w^2_{32} &w^2_{33} \\\\\n",
+    "    \\end{array} \\right] \\cdot\n",
+    "    \\left[\\begin{array}{c}\n",
+    "           y^1_1 \\\\\n",
+    "           y^1_2 \\\\\n",
+    "           y^1_3 \\\\\n",
+    "          \\end{array}\\right] + \n",
+    "    \\left[\\begin{array}{c}\n",
+    "           b^2_1 \\\\\n",
+    "           b^2_2 \\\\\n",
+    "           b^2_3 \\\\\n",
+    "          \\end{array}\\right]\\right).\n",
+    "\\label{_auto6} \\tag{11}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "044d5e68",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "### Matrix-vector notation  and activation\n",
+    "\n",
+    "The activation of node $i$ in layer 2 is"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "23a2c440",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto7\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    " y^2_i = f_2\\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\\Bigr) = \n",
+    " f_2\\left(\\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\\right).\n",
+    "\\label{_auto7} \\tag{12}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f6921f1",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "This is not just a convenient and compact notation, but also a useful\n",
+    "and intuitive way to think about MLPs: The output is calculated by a\n",
+    "series of matrix-vector multiplications and vector additions that are\n",
+    "used as input to the activation functions. For each operation\n",
+    "$\\mathrm{W}_l \\hat{y}_{l-1}$ we move forward one layer."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bc3e047f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "### Activation functions\n",
+    "\n",
+    "A property that characterizes a neural network, other than its\n",
+    "connectivity, is the choice of activation function(s).  As described\n",
+    "in, the following restrictions are imposed on an activation function\n",
+    "for a FFNN to fulfill the universal approximation theorem\n",
+    "\n",
+    "  * Non-constant\n",
+    "\n",
+    "  * Bounded\n",
+    "\n",
+    "  * Monotonically-increasing\n",
+    "\n",
+    "  * Continuous"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f2ff2ee2",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "### Activation functions, Logistic and Hyperbolic ones\n",
+    "\n",
+    "The second requirement excludes all linear functions. Furthermore, in\n",
+    "a MLP with only linear activation functions, each layer simply\n",
+    "performs a linear transformation of its inputs.\n",
+    "\n",
+    "Regardless of the number of layers, the output of the NN will be\n",
+    "nothing but a linear function of the inputs. Thus we need to introduce\n",
+    "some kind of non-linearity to the NN to be able to fit non-linear\n",
+    "functions Typical examples are the logistic *Sigmoid*"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a336177e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "f(x) = \\frac{1}{1 + e^{-x}},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c9a263c9",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and the *hyperbolic tangent* function"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f102c33c",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "f(x) = \\tanh(x)\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "92eeded3",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "### Relevance\n",
+    "\n",
+    "The *sigmoid* function are more biologically plausible because the\n",
+    "output of inactive neurons are zero. Such activation function are\n",
+    "called *one-sided*. However, it has been shown that the hyperbolic\n",
+    "tangent performs better than the sigmoid for training MLPs.  has\n",
+    "become the most popular for *deep neural networks*"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "17339021",
+   "metadata": {
+    "collapsed": false,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "\"\"\"The sigmoid function (or the logistic curve) is a \n",
+    "function that takes any real number, z, and outputs a number (0,1).\n",
+    "It is useful in neural networks for assigning weights on a relative scale.\n",
+    "The value z is the weighted sum of parameters involved in the learning algorithm.\"\"\"\n",
+    "\n",
+    "import numpy\n",
+    "import matplotlib.pyplot as plt\n",
+    "import math as mt\n",
+    "\n",
+    "z = numpy.arange(-5, 5, .1)\n",
+    "sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))\n",
+    "sigma = sigma_fn(z)\n",
+    "\n",
+    "fig = plt.figure()\n",
+    "ax = fig.add_subplot(111)\n",
+    "ax.plot(z, sigma)\n",
+    "ax.set_ylim([-0.1, 1.1])\n",
+    "ax.set_xlim([-5,5])\n",
+    "ax.grid(True)\n",
+    "ax.set_xlabel('z')\n",
+    "ax.set_title('sigmoid function')\n",
+    "\n",
+    "plt.show()\n",
+    "\n",
+    "\"\"\"Step Function\"\"\"\n",
+    "z = numpy.arange(-5, 5, .02)\n",
+    "step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)\n",
+    "step = step_fn(z)\n",
+    "\n",
+    "fig = plt.figure()\n",
+    "ax = fig.add_subplot(111)\n",
+    "ax.plot(z, step)\n",
+    "ax.set_ylim([-0.5, 1.5])\n",
+    "ax.set_xlim([-5,5])\n",
+    "ax.grid(True)\n",
+    "ax.set_xlabel('z')\n",
+    "ax.set_title('step function')\n",
+    "\n",
+    "plt.show()\n",
+    "\n",
+    "\"\"\"Sine Function\"\"\"\n",
+    "z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)\n",
+    "t = numpy.sin(z)\n",
+    "\n",
+    "fig = plt.figure()\n",
+    "ax = fig.add_subplot(111)\n",
+    "ax.plot(z, t)\n",
+    "ax.set_ylim([-1.0, 1.0])\n",
+    "ax.set_xlim([-2*mt.pi,2*mt.pi])\n",
+    "ax.grid(True)\n",
+    "ax.set_xlabel('z')\n",
+    "ax.set_title('sine function')\n",
+    "\n",
+    "plt.show()\n",
+    "\n",
+    "\"\"\"Plots a graph of the squashing function used by a rectified linear\n",
+    "unit\"\"\"\n",
+    "z = numpy.arange(-2, 2, .1)\n",
+    "zero = numpy.zeros(len(z))\n",
+    "y = numpy.max([zero, z], axis=0)\n",
+    "\n",
+    "fig = plt.figure()\n",
+    "ax = fig.add_subplot(111)\n",
+    "ax.plot(z, y)\n",
+    "ax.set_ylim([-2.0, 2.0])\n",
+    "ax.set_xlim([-2.0, 2.0])\n",
+    "ax.grid(True)\n",
+    "ax.set_xlabel('z')\n",
+    "ax.set_title('Rectified linear unit')\n",
+    "\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8c915f6d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## The multilayer  perceptron (MLP)\n",
+    "\n",
+    "The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of\n",
+    "1. A neural network with one or more layers of nodes between the input and the output nodes.\n",
+    "\n",
+    "2. The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.\n",
+    "\n",
+    "3. The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.\n",
+    "\n",
+    "As a convention it is normal to call  a  network with one layer of input units, one layer of hidden\n",
+    "units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.\n",
+    "\n",
+    "For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.\n",
+    "Hereafter we will call the various entities of a layer for nodes.\n",
+    "There are also no connections within a single layer.\n",
+    "\n",
+    "The number of input nodes does not need to equal the number of output\n",
+    "nodes. This applies also to the hidden layers. Each layer may have its\n",
+    "own number of nodes and activation functions.\n",
+    "\n",
+    "The hidden layers have their name from the fact that they are not\n",
+    "linked to observables and as we will see below when we define the\n",
+    "so-called activation $\\hat{z}$, we can think of this as a basis\n",
+    "expansion of the original inputs $\\hat{x}$. The difference however\n",
+    "between neural networks and say linear regression is that now these\n",
+    "basis functions (which will correspond to the weights in the network)\n",
+    "are learned from data. This results in  an important difference between\n",
+    "neural networks and deep learning approaches on one side and methods\n",
+    "like logistic regression or linear regression and their modifications on the other side."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cdcb2b32",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## From one to many layers, the universal approximation theorem\n",
+    "\n",
+    "A neural network with only one layer, what we called the simple\n",
+    "perceptron, is best suited if we have a standard binary model with\n",
+    "clear (linear) boundaries between the outcomes. As such it could\n",
+    "equally well be replaced by standard linear regression or logistic\n",
+    "regression. Networks with one or more hidden layers approximate\n",
+    "systems with more complex boundaries.\n",
+    "\n",
+    "As stated earlier, \n",
+    "an important theorem in studies of neural networks, restated without\n",
+    "proof here, is the [universal approximation\n",
+    "theorem](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf).\n",
+    "\n",
+    "It states that a feed-forward network with a single hidden layer\n",
+    "containing a finite number of neurons can approximate continuous\n",
+    "functions on compact subsets of real functions. The theorem thus\n",
+    "states that simple neural networks can represent a wide variety of\n",
+    "interesting functions when given appropriate parameters. It is the\n",
+    "multilayer feedforward architecture itself which gives neural networks\n",
+    "the potential of being universal approximators."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "566b0dcb",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Deriving the back propagation code for a multilayer perceptron model\n",
+    "\n",
+    "As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.\n",
+    "The unknowwn quantities are our weights $w_{ij}$ and we need to find an algorithm for changing them so that our errors are as small as possible.\n",
+    "This leads us to the famous [back propagation algorithm](https://www.nature.com/articles/323533a0).\n",
+    "\n",
+    "The questions we want to ask are how do changes in the biases and the\n",
+    "weights in our network change the cost function and how can we use the\n",
+    "final output to modify the weights?\n",
+    "\n",
+    "To derive these equations let us start with a plain regression problem\n",
+    "and define our cost function as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "66c76764",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "{\\cal C}(\\hat{W})  =  \\frac{1}{2}\\sum_{i=1}^n\\left(y_i - t_i\\right)^2,\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3d0bfa4",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where the $t_i$s are our $n$ targets (the values we want to\n",
+    "reproduce), while the outputs of the network after having propagated\n",
+    "all inputs $\\hat{x}$ are given by $y_i$.  Below we will demonstrate\n",
+    "how the basic equations arising from the back propagation algorithm\n",
+    "can be modified in order to study classification problems with $K$\n",
+    "classes."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "01cf3c8d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Definitions\n",
+    "\n",
+    "With our definition of the targets $\\hat{t}$, the outputs of the\n",
+    "network $\\hat{y}$ and the inputs $\\hat{x}$ we\n",
+    "define now the activation $z_j^l$ of node/neuron/unit $j$ of the\n",
+    "$l$-th layer as a function of the bias, the weights which add up from\n",
+    "the previous layer $l-1$ and the forward passes/outputs\n",
+    "$\\hat{a}^{l-1}$ from the previous layer as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1417eea3",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "z_j^l = \\sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3c4e10cf",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "where $b_k^l$ are the biases from layer $l$.  Here $M_{l-1}$\n",
+    "represents the total number of nodes/neurons/units of layer $l-1$. The\n",
+    "figure here illustrates this equation.  We can rewrite this in a more\n",
+    "compact form as the matrix-vector products we discussed earlier,"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d9dc403d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\hat{z}^l = \\left(\\hat{W}^l\\right)^T\\hat{a}^{l-1}+\\hat{b}^l.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "83b73b28",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "With the activation values $\\hat{z}^l$ we can in turn define the\n",
+    "output of layer $l$ as $\\hat{a}^l = f(\\hat{z}^l)$ where $f$ is our\n",
+    "activation function. In the examples here we will use the sigmoid\n",
+    "function discussed in our logistic regression lectures. We will also use the same activation function $f$ for all layers\n",
+    "and their nodes.  It means we have"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0516dfcf",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "a_j^l = f(z_j^l) = \\frac{1}{1+\\exp{-(z_j^l)}}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7cebeb6f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Derivatives and the chain rule\n",
+    "\n",
+    "From the definition of the activation $z_j^l$ we have"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e94a91e9",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial z_j^l}{\\partial w_{ij}^l} = a_i^{l-1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "343a68be",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1eb93ade",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial z_j^l}{\\partial a_i^{l-1}} = w_{ji}^l.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b4294825",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "With our definition of the activation function we have that (note that this function depends only on $z_j^l$)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3c84d05d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial a_j^l}{\\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)).\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "acd76b39",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Derivative of the cost function\n",
+    "\n",
+    "With these definitions we can now compute the derivative of the cost function in terms of the weights.\n",
+    "\n",
+    "Let us specialize to the output layer $l=L$. Our cost function is"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f0383586",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "{\\cal C}(\\hat{W^L})  =  \\frac{1}{2}\\sum_{i=1}^n\\left(y_i - t_i\\right)^2=\\frac{1}{2}\\sum_{i=1}^n\\left(a_i^L - t_i\\right)^2,\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "81c5b644",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "The derivative of this function with respect to the weights is"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "31fde909",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L}  =  \\left(a_j^L - t_j\\right)\\frac{\\partial a_j^L}{\\partial w_{jk}^{L}},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e4e2612",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "The last partial derivative can easily be computed and reads (by applying the chain rule)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9c9e5698",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial a_j^L}{\\partial w_{jk}^{L}} = \\frac{\\partial a_j^L}{\\partial z_{j}^{L}}\\frac{\\partial z_j^L}{\\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "421c9de0",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Bringing it together, first back propagation equation\n",
+    "\n",
+    "We have thus"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10597fa3",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L}  =  \\left(a_j^L - t_j\\right)a_j^L(1-a_j^L)a_k^{L-1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0cf3157b",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Defining"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "01128c99",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^L = a_j^L(1-a_j^L)\\left(a_j^L - t_j\\right) = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "097ee928",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and using the Hadamard product of two vectors we can write this as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cb2ff4ec",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\hat{\\delta}^L = f'(\\hat{z}^L)\\circ\\frac{\\partial {\\cal C}}{\\partial (\\hat{a}^L)}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ee3423ea",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "This is an important expression. The second term on the right handside\n",
+    "measures how fast the cost function is changing as a function of the $j$th\n",
+    "output activation.  If, for example, the cost function doesn't depend\n",
+    "much on a particular output node $j$, then $\\delta_j^L$ will be small,\n",
+    "which is what we would expect. The first term on the right, measures\n",
+    "how fast the activation function $f$ is changing at a given activation\n",
+    "value $z_j^L$.\n",
+    "\n",
+    "Notice that everything in the above equations is easily computed.  In\n",
+    "particular, we compute $z_j^L$ while computing the behaviour of the\n",
+    "network, and it is only a small additional overhead to compute\n",
+    "$f'(z^L_j)$.  The exact form of the derivative with respect to the\n",
+    "output depends on the form of the cost function.\n",
+    "However, provided the cost function is known there should be little\n",
+    "trouble in calculating"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49a7e0e3",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8efd3f58",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "With the definition of $\\delta_j^L$ we have a more compact definition of the derivative of the cost function in terms of the weights, namely"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dd61c9f6",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L}  =  \\delta_j^La_k^{L-1}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7574ecb2",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Derivatives in terms of $z_j^L$\n",
+    "\n",
+    "It is also easy to see that our previous equation can be written as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "103823da",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^L =\\frac{\\partial {\\cal C}}{\\partial z_j^L}= \\frac{\\partial {\\cal C}}{\\partial a_j^L}\\frac{\\partial a_j^L}{\\partial z_j^L},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8fffaff6",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "which can also be interpreted as the partial derivative of the cost function with respect to the biases $b_j^L$, namely"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ae94d046",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^L = \\frac{\\partial {\\cal C}}{\\partial b_j^L}\\frac{\\partial b_j^L}{\\partial z_j^L}=\\frac{\\partial {\\cal C}}{\\partial b_j^L},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3874588e",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "That is, the error $\\delta_j^L$ is exactly equal to the rate of change of the cost function as a function of the bias."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "296c98ed",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Bringing it together\n",
+    "\n",
+    "We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are\n",
+    "\n",
+    "**The starting equations.**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "19e9a098",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto8\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L}  =  \\delta_j^La_k^{L-1},\n",
+    "\\label{_auto8} \\tag{13}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "edb4be27",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f85640ae",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto9\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    "\\delta_j^L = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)},\n",
+    "\\label{_auto9} \\tag{14}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ef718563",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "69a17671",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "<!-- Equation labels as ordinary links -->\n",
+    "<div id=\"_auto10\"></div>\n",
+    "\n",
+    "$$\n",
+    "\\begin{equation}\n",
+    "\\delta_j^L = \\frac{\\partial {\\cal C}}{\\partial b_j^L},\n",
+    "\\label{_auto10} \\tag{15}\n",
+    "\\end{equation}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "84ef0653",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "An interesting consequence of the above equations is that when the\n",
+    "activation $a_k^{L-1}$ is small, the gradient term, that is the\n",
+    "derivative of the cost function with respect to the weights, will also\n",
+    "tend to be small. We say then that the weight learns slowly, meaning\n",
+    "that it changes slowly when we minimize the weights via say gradient\n",
+    "descent. In this case we say the system learns slowly.\n",
+    "\n",
+    "Another interesting feature is that is when the activation function,\n",
+    "represented by the sigmoid function here, is rather flat when we move towards\n",
+    "its end values $0$ and $1$ (see the above Python codes). In these\n",
+    "cases, the derivatives of the activation function will also be close\n",
+    "to zero, meaning again that the gradients will be small and the\n",
+    "network learns slowly again.\n",
+    "\n",
+    "We need a fourth equation and we are set. We are going to propagate\n",
+    "backwards in order to the determine the weights and biases. In order\n",
+    "to do so we need to represent the error in the layer before the final\n",
+    "one $L-1$ in terms of the errors in the final output layer."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fb567e6f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Final back propagating equation\n",
+    "\n",
+    "We have that (replacing $L$ with a general layer $l$)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "42773e4f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^l =\\frac{\\partial {\\cal C}}{\\partial z_j^l}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c01f5515",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "We want to express this in terms of the equations for layer $l+1$. Using the chain rule and summing over all $k$ entries we have"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b23962dc",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^l =\\sum_k \\frac{\\partial {\\cal C}}{\\partial z_k^{l+1}}\\frac{\\partial z_k^{l+1}}{\\partial z_j^{l}}=\\sum_k \\delta_k^{l+1}\\frac{\\partial z_k^{l+1}}{\\partial z_j^{l}},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c84de332",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "and recalling that"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ed77e307",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "z_j^{l+1} = \\sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ab04fbdc",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "with $M_l$ being the number of nodes in layer $l$, we obtain"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d0ce3d50",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^l =\\sum_k \\delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a217439f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "This is our final equation.\n",
+    "\n",
+    "We are now ready to set up the algorithm for back propagation and learning the weights and biases."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "71731044",
    "metadata": {
     "editable": true
    },
    "source": [
     "## Setting up the Back propagation algorithm\n",
     "\n",
-    "The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.\n",
+    "The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.\n",
+    "\n",
+    "First, we set up the input data $\\hat{x}$ and the activations\n",
+    "$\\hat{z}_1$ of the input layer and compute the activation function and\n",
+    "the pertinent outputs $\\hat{a}^1$.\n",
+    "\n",
+    "Secondly, we perform then the feed forward till we reach the output\n",
+    "layer and compute all $\\hat{z}_l$ of the input layer and compute the\n",
+    "activation function and the pertinent outputs $\\hat{a}^l$ for\n",
+    "$l=2,3,\\dots,L$.\n",
+    "\n",
+    "Thereafter we compute the ouput error $\\hat{\\delta}^L$ by computing all"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a2afecfd",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^L = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ffa0ea9",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Then we compute the back propagate error for each $l=L-1,L-2,\\dots,2$ as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "025daba8",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^l = \\sum_k \\delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "935b27ef",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Finally, we update the weights and the biases using gradient descent for each $l=L-1,L-2,\\dots,2$ and update the weights and biases according to the rules"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "50179be2",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "w_{jk}^l\\leftarrow  = w_{jk}^l- \\eta \\delta_j^la_k^{l-1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "90daf72c",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "b_j^l \\leftarrow b_j^l-\\eta \\frac{\\partial {\\cal C}}{\\partial b_j^l}=b_j^l-\\eta \\delta_j^l,\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "63e9545c",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "The parameter $\\eta$ is the learning parameter discussed in connection with the gradient descent methods.\n",
+    "Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e8fbaffb",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Setting up the Back propagation algorithm\n",
+    "\n",
+    "The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.\n",
     "\n",
     "First, we set up the input data $\\boldsymbol{x}$ and the activations\n",
     "$\\boldsymbol{z}_1$ of the input layer and compute the activation function and\n",
@@ -99,7 +2000,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "cfc83d3e",
+   "id": "59b2fcba",
    "metadata": {
     "editable": true
    },
@@ -111,7 +2012,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c52a5c90",
+   "id": "db7fd0df",
    "metadata": {
     "editable": true
    },
@@ -121,7 +2022,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "f87049f3",
+   "id": "fef9229b",
    "metadata": {
     "editable": true
    },
@@ -133,7 +2034,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "3b402468",
+   "id": "720ee2fe",
    "metadata": {
     "editable": true
    },
@@ -143,7 +2044,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c3e371c5",
+   "id": "df965f5a",
    "metadata": {
     "editable": true
    },
@@ -155,7 +2056,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "05459156",
+   "id": "29bef69b",
    "metadata": {
     "editable": true
    },
@@ -167,7 +2068,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "8ee0290c",
+   "id": "8f8670bc",
    "metadata": {
     "editable": true
    },
@@ -178,7 +2079,109 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c48f1301",
+   "id": "b37be623",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "## Setting up the Back propagation algorithm\n",
+    "\n",
+    "The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.\n",
+    "\n",
+    "First, we set up the input data $\\boldsymbol{x}$ and the activations\n",
+    "$\\boldsymbol{z}_1$ of the input layer and compute the activation function and\n",
+    "the pertinent outputs $\\boldsymbol{a}^1$.\n",
+    "\n",
+    "Secondly, we perform then the feed forward till we reach the output\n",
+    "layer and compute all $\\boldsymbol{z}_l$ of the input layer and compute the\n",
+    "activation function and the pertinent outputs $\\boldsymbol{a}^l$ for\n",
+    "$l=2,3,\\dots,L$.\n",
+    "\n",
+    "Thereafter we compute the ouput error $\\boldsymbol{\\delta}^L$ by computing all"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6ebda602",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^L = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}.\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6f2371ba",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Then we compute the back propagate error for each $l=L-1,L-2,\\dots,2$ as"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b3458652",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "\\delta_j^l = \\sum_k \\delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4a46987f",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "Finally, we update the weights and the biases using gradient descent for each $l=L-1,L-2,\\dots,2$ and update the weights and biases according to the rules"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2b898b0d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "w_{jk}^l\\leftarrow  = w_{jk}^l- \\eta \\delta_j^la_k^{l-1},\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1af5bcf",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "$$\n",
+    "b_j^l \\leftarrow b_j^l-\\eta \\frac{\\partial {\\cal C}}{\\partial b_j^l}=b_j^l-\\eta \\delta_j^l,\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8d7a0d4d",
+   "metadata": {
+    "editable": true
+   },
+   "source": [
+    "The parameter $\\eta$ is the learning parameter discussed in connection with the gradient descent methods.\n",
+    "Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "37f6a409",
    "metadata": {
     "editable": true
    },
@@ -206,7 +2209,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "4ef49cf0",
+   "id": "e7939992",
    "metadata": {
     "editable": true
    },
@@ -218,7 +2221,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "19f1dc57",
+   "id": "d754aefd",
    "metadata": {
     "editable": true
    },
@@ -228,7 +2231,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "16155951",
+   "id": "7711b16e",
    "metadata": {
     "editable": true
    },
@@ -240,7 +2243,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "0566b8db",
+   "id": "e831f3b5",
    "metadata": {
     "editable": true
    },
@@ -251,7 +2254,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "93b66473",
+   "id": "428cb018",
    "metadata": {
     "editable": true
    },
@@ -263,7 +2266,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "59609222",
+   "id": "303e8c9e",
    "metadata": {
     "editable": true
    },
@@ -276,7 +2279,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "541c0474",
+   "id": "42c3514b",
    "metadata": {
     "editable": true
    },
@@ -301,7 +2304,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "f1d7b488",
+   "id": "6675c087",
    "metadata": {
     "editable": true
    },
@@ -314,7 +2317,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "82c5f484",
+   "id": "73cfb814",
    "metadata": {
     "editable": true
    },
@@ -326,7 +2329,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c7dfc681",
+   "id": "c32b2b0e",
    "metadata": {
     "editable": true
    },
@@ -338,7 +2341,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "27afae96",
+   "id": "15c86254",
    "metadata": {
     "editable": true
    },
@@ -348,7 +2351,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "4c7b4937",
+   "id": "9da11ce8",
    "metadata": {
     "editable": true
    },
@@ -360,7 +2363,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "10004f54",
+   "id": "47275fce",
    "metadata": {
     "editable": true
    },
@@ -372,7 +2375,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "d188ab09",
+   "id": "22860f6c",
    "metadata": {
     "editable": true
    },
@@ -384,7 +2387,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "17ee12d5",
+   "id": "2c90d0da",
    "metadata": {
     "editable": true
    },
@@ -396,7 +2399,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "17ffedce",
+   "id": "12ae44b0",
    "metadata": {
     "editable": true
    },
@@ -406,7 +2409,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "b0f0cc69",
+   "id": "7d921bf4",
    "metadata": {
     "editable": true
    },
@@ -418,7 +2421,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "b384259f",
+   "id": "5e48ee91",
    "metadata": {
     "editable": true
    },
@@ -428,7 +2431,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "f357a71b",
+   "id": "6add3f84",
    "metadata": {
     "editable": true
    },
@@ -440,7 +2443,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "4a172fb6",
+   "id": "22759cfd",
    "metadata": {
     "editable": true
    },
@@ -453,7 +2456,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "52c754bc",
+   "id": "d18311a9",
    "metadata": {
     "editable": true
    },
@@ -465,7 +2468,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "211abd96",
+   "id": "9761e98e",
    "metadata": {
     "editable": true
    },
@@ -475,7 +2478,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "13f05813",
+   "id": "01a4059b",
    "metadata": {
     "editable": true
    },
@@ -487,7 +2490,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "9c92564e",
+   "id": "56bf706a",
    "metadata": {
     "editable": true
    },
@@ -498,7 +2501,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "8b15007b",
+   "id": "60613b46",
    "metadata": {
     "editable": true
    },
@@ -510,7 +2513,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "e317235b",
+   "id": "e97cc842",
    "metadata": {
     "editable": true
    },
@@ -520,7 +2523,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "f0bf3e1d",
+   "id": "9c431661",
    "metadata": {
     "editable": true
    },
@@ -532,7 +2535,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "2e7dbb17",
+   "id": "1903bf14",
    "metadata": {
     "editable": true
    },
@@ -542,7 +2545,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "ffea970b",
+   "id": "dab43f84",
    "metadata": {
     "editable": true
    },
@@ -553,7 +2556,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "bcefe435",
+   "id": "03d7a2cf",
    "metadata": {
     "editable": true
    },
@@ -566,7 +2569,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "36ca4b8e",
+   "id": "1813356c",
    "metadata": {
     "editable": true
    },
@@ -576,7 +2579,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "79c1417a",
+   "id": "debe1289",
    "metadata": {
     "editable": true
    },
@@ -588,7 +2591,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "2592db6e",
+   "id": "6fc65621",
    "metadata": {
     "editable": true
    },
@@ -598,7 +2601,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "da359e00",
+   "id": "8c86ad26",
    "metadata": {
     "editable": true
    },
@@ -610,7 +2613,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "12521ab8",
+   "id": "86ecd366",
    "metadata": {
     "editable": true
    },
@@ -620,7 +2623,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "e835a921",
+   "id": "d68a68c2",
    "metadata": {
     "editable": true
    },
@@ -644,7 +2647,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "199c93d2",
+   "id": "c9ec79c6",
    "metadata": {
     "editable": true
    },
@@ -693,16 +2696,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
-   "id": "5ac526a9",
+   "execution_count": 5,
+   "id": "b07f9bb8",
    "metadata": {
     "collapsed": false,
     "editable": true
    },
    "outputs": [],
    "source": [
-    "%matplotlib inline\n",
-    "\n",
     "# import necessary packages\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
@@ -749,7 +2750,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "9a2b47ab",
+   "id": "6b776a41",
    "metadata": {
     "editable": true
    },
@@ -769,8 +2770,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "id": "5414fbee",
+   "execution_count": 6,
+   "id": "3e399ed6",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -808,7 +2809,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6106cbfc",
+   "id": "2c165e0e",
    "metadata": {
     "editable": true
    },
@@ -852,7 +2853,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "67e9f13a",
+   "id": "94075506",
    "metadata": {
     "editable": true
    },
@@ -892,7 +2893,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "2f8ebd5d",
+   "id": "0bb70540",
    "metadata": {
     "editable": true
    },
@@ -912,8 +2913,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "id": "439f735b",
+   "execution_count": 7,
+   "id": "f3894282",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -939,7 +2940,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "a6bd9d44",
+   "id": "25c08547",
    "metadata": {
     "editable": true
    },
@@ -967,7 +2968,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6cfe367f",
+   "id": "96a5c9b5",
    "metadata": {
     "editable": true
    },
@@ -1003,8 +3004,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "id": "0a74c4ad",
+   "execution_count": 8,
+   "id": "948f1d5c",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1050,7 +3051,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "1a87eda5",
+   "id": "6a48b541",
    "metadata": {
     "editable": true
    },
@@ -1081,7 +3082,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "004473d8",
+   "id": "92ae85de",
    "metadata": {
     "editable": true
    },
@@ -1119,7 +3120,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6e78b93a",
+   "id": "50af9f08",
    "metadata": {
     "editable": true
    },
@@ -1153,7 +3154,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "e94ad101",
+   "id": "947d899e",
    "metadata": {
     "editable": true
    },
@@ -1193,8 +3194,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "id": "0603a76a",
+   "execution_count": 9,
+   "id": "9ac16dfb",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1273,7 +3274,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "17d62cce",
+   "id": "15fa07c3",
    "metadata": {
     "editable": true
    },
@@ -1294,7 +3295,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c27a9b11",
+   "id": "e8e7304b",
    "metadata": {
     "editable": true
    },
@@ -1307,8 +3308,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
-   "id": "a651821c",
+   "execution_count": 10,
+   "id": "da41ccbe",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1418,7 +3419,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "17d3f194",
+   "id": "6ed32b94",
    "metadata": {
     "editable": true
    },
@@ -1436,8 +3437,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "id": "e91aa14d",
+   "execution_count": 11,
+   "id": "a2566347",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1464,7 +3465,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "15fda52a",
+   "id": "779ebe17",
    "metadata": {
     "editable": true
    },
@@ -1477,8 +3478,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "03099d3d",
+   "execution_count": 12,
+   "id": "f55e622e",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1509,7 +3510,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "a7cf1e6c",
+   "id": "4a2ea09c",
    "metadata": {
     "editable": true
    },
@@ -1519,8 +3520,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "id": "192fd489",
+   "execution_count": 13,
+   "id": "d1dc32a5",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1564,7 +3565,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "f555ba04",
+   "id": "fb02adc3",
    "metadata": {
     "editable": true
    },
@@ -1586,8 +3587,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "93868564",
+   "execution_count": 14,
+   "id": "94531c6e",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1614,7 +3615,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "b938525d",
+   "id": "1373fc77",
    "metadata": {
     "editable": true
    },
@@ -1624,8 +3625,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
-   "id": "e65fc56d",
+   "execution_count": 15,
+   "id": "2693a59e",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1670,7 +3671,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "1705a4be",
+   "id": "a7e93079",
    "metadata": {
     "editable": true
    },
@@ -1697,7 +3698,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6ef76cb6",
+   "id": "5ed2f79a",
    "metadata": {
     "editable": true
    },
@@ -1735,7 +3736,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "ce3434c9",
+   "id": "cbf09e4f",
    "metadata": {
     "editable": true
    },
@@ -1747,7 +3748,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "af28a402",
+   "id": "fbb06edc",
    "metadata": {
     "editable": true
    },
@@ -1762,7 +3763,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "c33c7c5a",
+   "id": "9859c6a0",
    "metadata": {
     "editable": true
    },
@@ -1772,7 +3773,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "63299029",
+   "id": "d71c8649",
    "metadata": {
     "editable": true
    },
@@ -1784,8 +3785,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
-   "id": "c5677a08",
+   "execution_count": 16,
+   "id": "e8dc02bd",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1861,7 +3862,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "e4278da5",
+   "id": "87cfac77",
    "metadata": {
     "editable": true
    },
@@ -1871,7 +3872,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "d159f207",
+   "id": "fa50c112",
    "metadata": {
     "editable": true
    },
@@ -1881,8 +3882,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
-   "id": "7264c3ce",
+   "execution_count": 17,
+   "id": "6673546e",
    "metadata": {
     "collapsed": false,
     "editable": true
@@ -1947,921 +3948,6 @@
     "ax.set_xlabel(\"$\\lambda$\")\n",
     "plt.show()"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a799df9d",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Building neural networks in Tensorflow and Keras\n",
-    "\n",
-    "Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn\n",
-    "and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy\n",
-    "and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  \n",
-    "\n",
-    "In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite\n",
-    "clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or\n",
-    "NumPy arrays."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "989e19b4",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Tensorflow\n",
-    "\n",
-    "Tensorflow is an open source library machine learning library\n",
-    "developed by the Google Brain team for internal use. It was released\n",
-    "under the Apache 2.0 open source license in November 9, 2015.\n",
-    "\n",
-    "Tensorflow is a computational framework that allows you to construct\n",
-    "machine learning models at different levels of abstraction, from\n",
-    "high-level, object-oriented APIs like Keras, down to the C++ kernels\n",
-    "that Tensorflow is built upon. The higher levels of abstraction are\n",
-    "simpler to use, but less flexible, and our choice of implementation\n",
-    "should reflect the problems we are trying to solve.\n",
-    "\n",
-    "[Tensorflow uses](https://www.tensorflow.org/guide/graphs) so-called graphs to represent your computation\n",
-    "in terms of the dependencies between individual operations, such that you first build a Tensorflow *graph*\n",
-    "to represent your model, and then create a Tensorflow *session* to run the graph.\n",
-    "\n",
-    "In this guide we will analyze the same data as we did in our NumPy and\n",
-    "scikit-learn tutorial, gathered from the MNIST database of images. We\n",
-    "will give an introduction to the lower level Python Application\n",
-    "Program Interfaces (APIs), and see how we use them to build our graph.\n",
-    "Then we will build (effectively) the same graph in Keras, to see just\n",
-    "how simple solving a machine learning problem can be.\n",
-    "\n",
-    "To install tensorflow on Unix/Linux systems, use pip as"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "3310ee6d",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "pip3 install tensorflow"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cbfcd955",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "and/or if you use **anaconda**, just write (or install from the graphical user interface)\n",
-    "(current release of CPU-only TensorFlow)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "id": "0e7abca7",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "conda create -n tf tensorflow\n",
-    "conda activate tf"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "f002da9e",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "To install the current release of GPU TensorFlow"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "90156829",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "conda create -n tf-gpu tensorflow-gpu\n",
-    "conda activate tf-gpu"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a77fb74b",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Using Keras\n",
-    "\n",
-    "Keras is a high level [neural network](https://en.wikipedia.org/wiki/Application_programming_interface)\n",
-    "that supports Tensorflow, CTNK and Theano as backends.  \n",
-    "If you have Anaconda installed you may run the following command"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "fe9bcc62",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "conda install keras"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "13a135b9",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "You can look up the [instructions here](https://keras.io/) for more information.\n",
-    "\n",
-    "We will to a large extent use **keras** in this course."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "19dc5832",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Collect and pre-process data\n",
-    "\n",
-    "Let us look again at the MINST data set."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "50939b73",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "# import necessary packages\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt\n",
-    "import tensorflow as tf\n",
-    "from sklearn import datasets\n",
-    "\n",
-    "\n",
-    "# ensure the same random numbers appear every time\n",
-    "np.random.seed(0)\n",
-    "\n",
-    "# display images in notebook\n",
-    "%matplotlib inline\n",
-    "plt.rcParams['figure.figsize'] = (12,12)\n",
-    "\n",
-    "\n",
-    "# download MNIST dataset\n",
-    "digits = datasets.load_digits()\n",
-    "\n",
-    "# define inputs and labels\n",
-    "inputs = digits.images\n",
-    "labels = digits.target\n",
-    "\n",
-    "print(\"inputs = (n_inputs, pixel_width, pixel_height) = \" + str(inputs.shape))\n",
-    "print(\"labels = (n_inputs) = \" + str(labels.shape))\n",
-    "\n",
-    "\n",
-    "# flatten the image\n",
-    "# the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64\n",
-    "n_inputs = len(inputs)\n",
-    "inputs = inputs.reshape(n_inputs, -1)\n",
-    "print(\"X = (n_inputs, n_features) = \" + str(inputs.shape))\n",
-    "\n",
-    "\n",
-    "# choose some random images to display\n",
-    "indices = np.arange(n_inputs)\n",
-    "random_indices = np.random.choice(indices, size=5)\n",
-    "\n",
-    "for i, image in enumerate(digits.images[random_indices]):\n",
-    "    plt.subplot(1, 5, i+1)\n",
-    "    plt.axis('off')\n",
-    "    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')\n",
-    "    plt.title(\"Label: %d\" % digits.target[random_indices[i]])\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "id": "dc47cb3a",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "from tensorflow.keras.layers import Input\n",
-    "from tensorflow.keras.models import Sequential      #This allows appending layers to existing models\n",
-    "from tensorflow.keras.layers import Dense           #This allows defining the characteristics of a particular layer\n",
-    "from tensorflow.keras import optimizers             #This allows using whichever optimiser we want (sgd,adam,RMSprop)\n",
-    "from tensorflow.keras import regularizers           #This allows using whichever regularizer we want (l1,l2,l1_l2)\n",
-    "from tensorflow.keras.utils import to_categorical   #This allows using categorical cross entropy as the cost function\n",
-    "\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "\n",
-    "# one-hot representation of labels\n",
-    "labels = to_categorical(labels)\n",
-    "\n",
-    "# split into train and test data\n",
-    "train_size = 0.8\n",
-    "test_size = 1 - train_size\n",
-    "X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,\n",
-    "                                                    test_size=test_size)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "id": "bc5c2fe1",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "\n",
-    "epochs = 100\n",
-    "batch_size = 100\n",
-    "n_neurons_layer1 = 100\n",
-    "n_neurons_layer2 = 50\n",
-    "n_categories = 10\n",
-    "eta_vals = np.logspace(-5, 1, 7)\n",
-    "lmbd_vals = np.logspace(-5, 1, 7)\n",
-    "def create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):\n",
-    "    model = Sequential()\n",
-    "    model.add(Dense(n_neurons_layer1, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd)))\n",
-    "    model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd)))\n",
-    "    model.add(Dense(n_categories, activation='softmax'))\n",
-    "    \n",
-    "    sgd = optimizers.SGD(lr=eta)\n",
-    "    model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
-    "    \n",
-    "    return model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "id": "00800a93",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "DNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n",
-    "        \n",
-    "for i, eta in enumerate(eta_vals):\n",
-    "    for j, lmbd in enumerate(lmbd_vals):\n",
-    "        DNN = create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,\n",
-    "                                         eta=eta, lmbd=lmbd)\n",
-    "        DNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)\n",
-    "        scores = DNN.evaluate(X_test, Y_test)\n",
-    "        \n",
-    "        DNN_keras[i][j] = DNN\n",
-    "        \n",
-    "        print(\"Learning rate = \", eta)\n",
-    "        print(\"Lambda = \", lmbd)\n",
-    "        print(\"Test accuracy: %.3f\" % scores[1])\n",
-    "        print()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "8ace914b",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "# optional\n",
-    "# visual representation of grid search\n",
-    "# uses seaborn heatmap, could probably do this in matplotlib\n",
-    "import seaborn as sns\n",
-    "\n",
-    "sns.set()\n",
-    "\n",
-    "train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n",
-    "test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n",
-    "\n",
-    "for i in range(len(eta_vals)):\n",
-    "    for j in range(len(lmbd_vals)):\n",
-    "        DNN = DNN_keras[i][j]\n",
-    "\n",
-    "        train_accuracy[i][j] = DNN.evaluate(X_train, Y_train)[1]\n",
-    "        test_accuracy[i][j] = DNN.evaluate(X_test, Y_test)[1]\n",
-    "\n",
-    "        \n",
-    "fig, ax = plt.subplots(figsize = (10, 10))\n",
-    "sns.heatmap(train_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n",
-    "ax.set_title(\"Training Accuracy\")\n",
-    "ax.set_ylabel(\"$\\eta$\")\n",
-    "ax.set_xlabel(\"$\\lambda$\")\n",
-    "plt.show()\n",
-    "\n",
-    "fig, ax = plt.subplots(figsize = (10, 10))\n",
-    "sns.heatmap(test_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n",
-    "ax.set_title(\"Test Accuracy\")\n",
-    "ax.set_ylabel(\"$\\eta$\")\n",
-    "ax.set_xlabel(\"$\\lambda$\")\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "51a8d259",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## The Breast Cancer Data, now with Keras"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "id": "13a062b9",
-   "metadata": {
-    "collapsed": false,
-    "editable": true
-   },
-   "outputs": [],
-   "source": [
-    "\n",
-    "import tensorflow as tf\n",
-    "from tensorflow.keras.layers import Input\n",
-    "from tensorflow.keras.models import Sequential      #This allows appending layers to existing models\n",
-    "from tensorflow.keras.layers import Dense           #This allows defining the characteristics of a particular layer\n",
-    "from tensorflow.keras import optimizers             #This allows using whichever optimiser we want (sgd,adam,RMSprop)\n",
-    "from tensorflow.keras import regularizers           #This allows using whichever regularizer we want (l1,l2,l1_l2)\n",
-    "from tensorflow.keras.utils import to_categorical   #This allows using categorical cross entropy as the cost function\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt\n",
-    "import seaborn as sns\n",
-    "from sklearn.model_selection import train_test_split as splitter\n",
-    "from sklearn.datasets import load_breast_cancer\n",
-    "import pickle\n",
-    "import os \n",
-    "\n",
-    "\n",
-    "\"\"\"Load breast cancer dataset\"\"\"\n",
-    "\n",
-    "np.random.seed(0)        #create same seed for random number every time\n",
-    "\n",
-    "cancer=load_breast_cancer()      #Download breast cancer dataset\n",
-    "\n",
-    "inputs=cancer.data                     #Feature matrix of 569 rows (samples) and 30 columns (parameters)\n",
-    "outputs=cancer.target                  #Label array of 569 rows (0 for benign and 1 for malignant)\n",
-    "labels=cancer.feature_names[0:30]\n",
-    "\n",
-    "print('The content of the breast cancer dataset is:')      #Print information about the datasets\n",
-    "print(labels)\n",
-    "print('-------------------------')\n",
-    "print(\"inputs =  \" + str(inputs.shape))\n",
-    "print(\"outputs =  \" + str(outputs.shape))\n",
-    "print(\"labels =  \"+ str(labels.shape))\n",
-    "\n",
-    "x=inputs      #Reassign the Feature and Label matrices to other variables\n",
-    "y=outputs\n",
-    "\n",
-    "#%% \n",
-    "\n",
-    "# Visualisation of dataset (for correlation analysis)\n",
-    "\n",
-    "plt.figure()\n",
-    "plt.scatter(x[:,0],x[:,2],s=40,c=y,cmap=plt.cm.Spectral)\n",
-    "plt.xlabel('Mean radius',fontweight='bold')\n",
-    "plt.ylabel('Mean perimeter',fontweight='bold')\n",
-    "plt.show()\n",
-    "\n",
-    "plt.figure()\n",
-    "plt.scatter(x[:,5],x[:,6],s=40,c=y, cmap=plt.cm.Spectral)\n",
-    "plt.xlabel('Mean compactness',fontweight='bold')\n",
-    "plt.ylabel('Mean concavity',fontweight='bold')\n",
-    "plt.show()\n",
-    "\n",
-    "\n",
-    "plt.figure()\n",
-    "plt.scatter(x[:,0],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)\n",
-    "plt.xlabel('Mean radius',fontweight='bold')\n",
-    "plt.ylabel('Mean texture',fontweight='bold')\n",
-    "plt.show()\n",
-    "\n",
-    "plt.figure()\n",
-    "plt.scatter(x[:,2],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)\n",
-    "plt.xlabel('Mean perimeter',fontweight='bold')\n",
-    "plt.ylabel('Mean compactness',fontweight='bold')\n",
-    "plt.show()\n",
-    "\n",
-    "\n",
-    "# Generate training and testing datasets\n",
-    "\n",
-    "#Select features relevant to classification (texture,perimeter,compactness and symmetery) \n",
-    "#and add to input matrix\n",
-    "\n",
-    "temp1=np.reshape(x[:,1],(len(x[:,1]),1))\n",
-    "temp2=np.reshape(x[:,2],(len(x[:,2]),1))\n",
-    "X=np.hstack((temp1,temp2))      \n",
-    "temp=np.reshape(x[:,5],(len(x[:,5]),1))\n",
-    "X=np.hstack((X,temp))       \n",
-    "temp=np.reshape(x[:,8],(len(x[:,8]),1))\n",
-    "X=np.hstack((X,temp))       \n",
-    "\n",
-    "X_train,X_test,y_train,y_test=splitter(X,y,test_size=0.1)   #Split datasets into training and testing\n",
-    "\n",
-    "y_train=to_categorical(y_train)     #Convert labels to categorical when using categorical cross entropy\n",
-    "y_test=to_categorical(y_test)\n",
-    "\n",
-    "del temp1,temp2,temp\n",
-    "\n",
-    "# %%\n",
-    "\n",
-    "# Define tunable parameters\"\n",
-    "\n",
-    "eta=np.logspace(-3,-1,3)                    #Define vector of learning rates (parameter to SGD optimiser)\n",
-    "lamda=0.01                                  #Define hyperparameter\n",
-    "n_layers=2                                  #Define number of hidden layers in the model\n",
-    "n_neuron=np.logspace(0,3,4,dtype=int)       #Define number of neurons per layer\n",
-    "epochs=100                                   #Number of reiterations over the input data\n",
-    "batch_size=100                              #Number of samples per gradient update\n",
-    "\n",
-    "# %%\n",
-    "\n",
-    "\"\"\"Define function to return Deep Neural Network model\"\"\"\n",
-    "\n",
-    "def NN_model(inputsize,n_layers,n_neuron,eta,lamda):\n",
-    "    model=Sequential()      \n",
-    "    for i in range(n_layers):       #Run loop to add hidden layers to the model\n",
-    "        if (i==0):                  #First layer requires input dimensions\n",
-    "            model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda),input_dim=inputsize))\n",
-    "        else:                       #Subsequent layers are capable of automatic shape inferencing\n",
-    "            model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda)))\n",
-    "    model.add(Dense(2,activation='softmax'))  #2 outputs - ordered and disordered (softmax for prob)\n",
-    "    sgd=optimizers.SGD(lr=eta)\n",
-    "    model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])\n",
-    "    return model\n",
-    "\n",
-    "    \n",
-    "Train_accuracy=np.zeros((len(n_neuron),len(eta)))      #Define matrices to store accuracy scores as a function\n",
-    "Test_accuracy=np.zeros((len(n_neuron),len(eta)))       #of learning rate and number of hidden neurons for \n",
-    "\n",
-    "for i in range(len(n_neuron)):     #run loops over hidden neurons and learning rates to calculate \n",
-    "    for j in range(len(eta)):      #accuracy scores \n",
-    "        DNN_model=NN_model(X_train.shape[1],n_layers,n_neuron[i],eta[j],lamda)\n",
-    "        DNN_model.fit(X_train,y_train,epochs=epochs,batch_size=batch_size,verbose=1)\n",
-    "        Train_accuracy[i,j]=DNN_model.evaluate(X_train,y_train)[1]\n",
-    "        Test_accuracy[i,j]=DNN_model.evaluate(X_test,y_test)[1]\n",
-    "               \n",
-    "\n",
-    "def plot_data(x,y,data,title=None):\n",
-    "\n",
-    "    # plot results\n",
-    "    fontsize=16\n",
-    "\n",
-    "\n",
-    "    fig = plt.figure()\n",
-    "    ax = fig.add_subplot(111)\n",
-    "    cax = ax.matshow(data, interpolation='nearest', vmin=0, vmax=1)\n",
-    "    \n",
-    "    cbar=fig.colorbar(cax)\n",
-    "    cbar.ax.set_ylabel('accuracy (%)',rotation=90,fontsize=fontsize)\n",
-    "    cbar.set_ticks([0,.2,.4,0.6,0.8,1.0])\n",
-    "    cbar.set_ticklabels(['0%','20%','40%','60%','80%','100%'])\n",
-    "\n",
-    "    # put text on matrix elements\n",
-    "    for i, x_val in enumerate(np.arange(len(x))):\n",
-    "        for j, y_val in enumerate(np.arange(len(y))):\n",
-    "            c = \"${0:.1f}\\\\%$\".format( 100*data[j,i])  \n",
-    "            ax.text(x_val, y_val, c, va='center', ha='center')\n",
-    "\n",
-    "    # convert axis vaues to to string labels\n",
-    "    x=[str(i) for i in x]\n",
-    "    y=[str(i) for i in y]\n",
-    "\n",
-    "\n",
-    "    ax.set_xticklabels(['']+x)\n",
-    "    ax.set_yticklabels(['']+y)\n",
-    "\n",
-    "    ax.set_xlabel('$\\\\mathrm{learning\\\\ rate}$',fontsize=fontsize)\n",
-    "    ax.set_ylabel('$\\\\mathrm{hidden\\\\ neurons}$',fontsize=fontsize)\n",
-    "    if title is not None:\n",
-    "        ax.set_title(title)\n",
-    "\n",
-    "    plt.tight_layout()\n",
-    "\n",
-    "    plt.show()\n",
-    "    \n",
-    "plot_data(eta,n_neuron,Train_accuracy, 'training')\n",
-    "plot_data(eta,n_neuron,Test_accuracy, 'testing')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9b1579eb",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Fine-tuning neural network hyperparameters\n",
-    "\n",
-    "The flexibility of neural networks is also one of their main\n",
-    "drawbacks: there are many hyperparameters to tweak. Not only can you\n",
-    "use any imaginable network topology (how neurons/nodes are interconnected),\n",
-    "but even in a simple FFNN you can change the number of layers, the\n",
-    "number of neurons per layer, the type of activation function to use in\n",
-    "each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you\n",
-    "know what combination of hyperparameters is the best for your task?\n",
-    "\n",
-    "* You can use grid search with cross-validation to find the right hyperparameters.\n",
-    "\n",
-    "However,since there are many hyperparameters to tune, and since\n",
-    "training a neural network on a large dataset takes a lot of time, you\n",
-    "will only be able to explore a tiny part of the hyperparameter space.\n",
-    "\n",
-    "* You can use randomized search.\n",
-    "\n",
-    "* Or use tools like [Oscar](http://oscar.calldesk.ai/), which implements more complex algorithms to help you find a good set of hyperparameters quickly."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0d7e41b0",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Hidden layers\n",
-    "\n",
-    "For many problems you can start with just one or two hidden layers and it will work just fine.\n",
-    "For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a\n",
-    "few hundred neurons.\n",
-    "You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of\n",
-    "neurons, in roughly the same amount of training time. \n",
-    "\n",
-    "For more complex problems, you can gradually\n",
-    "ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such\n",
-    "as large image classification or speech recognition, typically require networks with dozens of layers\n",
-    "and they need a huge amount\n",
-    "of training data. However, you will rarely have to train such networks from scratch: it is much more\n",
-    "common to reuse parts of a pretrained state-of-the-art network that performs a similar task."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b000195c",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Which activation function should I use?\n",
-    "\n",
-    "The Back propagation algorithm we derived above works by going from\n",
-    "the output layer to the input layer, propagating the error gradient on\n",
-    "the way. Once the algorithm has computed the gradient of the cost\n",
-    "function with regards to each parameter in the network, it uses these\n",
-    "gradients to update each parameter with a Gradient Descent (GD) step.\n",
-    "\n",
-    "Unfortunately for us, the gradients often get smaller and smaller as the\n",
-    "algorithm progresses down to the first hidden layers. As a result, the\n",
-    "GD update leaves the lower layer connection weights\n",
-    "virtually unchanged, and training never converges to a good\n",
-    "solution. This is known in the literature as \n",
-    "**the vanishing gradients problem**. \n",
-    "\n",
-    "In other cases, the opposite can happen, namely the the gradients can grow bigger and\n",
-    "bigger. The result is that many of the layers get large updates of the \n",
-    "weights the\n",
-    "algorithm diverges. This is the **exploding gradients problem**, which is\n",
-    "mostly encountered in recurrent neural networks. More generally, deep\n",
-    "neural networks suffer from unstable gradients, different layers may\n",
-    "learn at widely different speeds"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2b02569e",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Is the Logistic activation function (Sigmoid)  our choice?\n",
-    "\n",
-    "Although this unfortunate behavior has been empirically observed for\n",
-    "quite a while (it was one of the reasons why deep neural networks were\n",
-    "mostly abandoned for a long time), it is only around 2010 that\n",
-    "significant progress was made in understanding it.\n",
-    "\n",
-    "A paper titled [Understanding the Difficulty of Training Deep\n",
-    "Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio](http://proceedings.mlr.press/v9/glorot10a.html) found that\n",
-    "the problems with the popular logistic\n",
-    "sigmoid activation function and the weight initialization technique\n",
-    "that was most popular at the time, namely random initialization using\n",
-    "a normal distribution with a mean of 0 and a standard deviation of\n",
-    "1. \n",
-    "\n",
-    "They showed that with this activation function and this\n",
-    "initialization scheme, the variance of the outputs of each layer is\n",
-    "much greater than the variance of its inputs. Going forward in the\n",
-    "network, the variance keeps increasing after each layer until the\n",
-    "activation function saturates at the top layers. This is actually made\n",
-    "worse by the fact that the logistic function has a mean of 0.5, not 0\n",
-    "(the hyperbolic tangent function has a mean of 0 and behaves slightly\n",
-    "better than the logistic function in deep networks)."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9789688f",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## The derivative of the Logistic funtion\n",
-    "\n",
-    "Looking at the logistic activation function, when inputs become large\n",
-    "(negative or positive), the function saturates at 0 or 1, with a\n",
-    "derivative extremely close to 0. Thus when backpropagation kicks in,\n",
-    "it has virtually no gradient to propagate back through the network,\n",
-    "and what little gradient exists keeps getting diluted as\n",
-    "backpropagation progresses down through the top layers, so there is\n",
-    "really nothing left for the lower layers.\n",
-    "\n",
-    "In their paper, Glorot and Bengio propose a way to significantly\n",
-    "alleviate this problem. We need the signal to flow properly in both\n",
-    "directions: in the forward direction when making predictions, and in\n",
-    "the reverse direction when backpropagating gradients. We don’t want\n",
-    "the signal to die out, nor do we want it to explode and saturate. For\n",
-    "the signal to flow properly, the authors argue that we need the\n",
-    "variance of the outputs of each layer to be equal to the variance of\n",
-    "its inputs, and we also need the gradients to have equal variance\n",
-    "before and after flowing through a layer in the reverse direction.\n",
-    "\n",
-    "One of the insights in the 2010 paper by Glorot and Bengio was that\n",
-    "the vanishing/exploding gradients problems were in part due to a poor\n",
-    "choice of activation function. Until then most people had assumed that\n",
-    "if Nature had chosen to use roughly sigmoid activation functions in\n",
-    "biological neurons, they must be an excellent choice. But it turns out\n",
-    "that other activation functions behave much better in deep neural\n",
-    "networks, in particular the ReLU activation function, mostly because\n",
-    "it does not saturate for positive values (and also because it is quite\n",
-    "fast to compute)."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b259486d",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## The RELU function family\n",
-    "\n",
-    "The ReLU activation function suffers from a problem known as the dying\n",
-    "ReLUs: during training, some neurons effectively die, meaning they\n",
-    "stop outputting anything other than 0.\n",
-    "\n",
-    "In some cases, you may find that half of your network’s neurons are\n",
-    "dead, especially if you used a large learning rate. During training,\n",
-    "if a neuron’s weights get updated such that the weighted sum of the\n",
-    "neuron’s inputs is negative, it will start outputting 0. When this\n",
-    "happen, the neuron is unlikely to come back to life since the gradient\n",
-    "of the ReLU function is 0 when its input is negative.\n",
-    "\n",
-    "To solve this problem, nowadays practitioners use a  variant of the ReLU\n",
-    "function, such as the leaky ReLU discussed above or the so-called\n",
-    "exponential linear unit (ELU) function"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "05bcd0f9",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "$$\n",
-    "ELU(z) = \\left\\{\\begin{array}{cc} \\alpha\\left( \\exp{(z)}-1\\right) & z < 0,\\\\  z & z \\ge 0.\\end{array}\\right.\n",
-    "$$"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e1a8592e",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Which activation function should we use?\n",
-    "\n",
-    "In general it seems that the ELU activation function is better than\n",
-    "the leaky ReLU function (and its variants), which is better than\n",
-    "ReLU. ReLU performs better than $\\tanh$ which in turn performs better\n",
-    "than the logistic function. \n",
-    "\n",
-    "If runtime\n",
-    "performance is an issue, then you may opt for the  leaky ReLU function  over the \n",
-    "ELU function If you don’t\n",
-    "want to tweak yet another hyperparameter, you may just use the default\n",
-    "$\\alpha$ of $0.01$ for the leaky ReLU, and $1$ for ELU. If you have\n",
-    "spare time and computing power, you can use cross-validation or\n",
-    "bootstrap to evaluate other activation functions."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c27319dc",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## More on activation functions, output layers\n",
-    "\n",
-    "In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).\n",
-    "\n",
-    "It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.\n",
-    "\n",
-    "**For the output layer:**\n",
-    "\n",
-    "* For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).\n",
-    "\n",
-    "* For regression tasks, you can simply use no activation function at all."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "13edfd00",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Batch Normalization\n",
-    "\n",
-    "Batch Normalization\n",
-    "aims to address the vanishing/exploding gradients problems, and more generally the problem that the\n",
-    "distribution of each layer’s inputs changes during training, as the parameters of the previous layers change.\n",
-    "\n",
-    "The technique consists of adding an operation in the model just before the activation function of each\n",
-    "layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new\n",
-    "parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model\n",
-    "learn the optimal scale and mean of the inputs for each layer.\n",
-    "In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs’ mean and\n",
-    "standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current\n",
-    "mini-batch, from this the name batch normalization."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2058a4fe",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Dropout\n",
-    "\n",
-    "It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but\n",
-    "excluding the output neurons) has a probability $p$ of being temporarily dropped out, meaning it will be\n",
-    "entirely ignored during this training step, but it may be active during the next step.\n",
-    "\n",
-    "The\n",
-    "hyperparameter $p$ is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.\n",
-    " It is viewed as one of the most popular regularization techniques."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "98d5373c",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Gradient Clipping\n",
-    "\n",
-    "A popular technique to lessen the exploding gradients problem is to simply clip the gradients during\n",
-    "backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural\n",
-    "networks).\n",
-    "\n",
-    "This technique is called Gradient Clipping.\n",
-    "\n",
-    "In general however, Batch\n",
-    "Normalization is preferred."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e762e70c",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## A very nice website on Neural Networks\n",
-    "\n",
-    "You may find this [website](https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false) very useful."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "92e6fbe6",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## A top-down perspective on Neural networks\n",
-    "\n",
-    "The first thing we would like to do is divide the data into two or three\n",
-    "parts. A training set, a validation or dev (development) set, and a\n",
-    "test set. The test set is the data on which we want to make\n",
-    "predictions. The dev set is a subset of the training data we use to\n",
-    "check how well we are doing out-of-sample, after training the model on\n",
-    "the training dataset. We use the validation error as a proxy for the\n",
-    "test error in order to make tweaks to our model. It is crucial that we\n",
-    "do not use any of the test data to train the algorithm. This is a\n",
-    "cardinal sin in ML. Then:\n",
-    "\n",
-    "* Estimate optimal error rate\n",
-    "\n",
-    "* Minimize underfitting (bias) on training data set.\n",
-    "\n",
-    "* Make sure you are not overfitting.\n",
-    "\n",
-    "If the validation and test sets are drawn from the same distributions,\n",
-    "then a good performance on the validation set should lead to similarly\n",
-    "good performance on the test set. \n",
-    "\n",
-    "However, sometimes\n",
-    "the training data and test data differ in subtle ways because, for\n",
-    "example, they are collected using slightly different methods, or\n",
-    "because it is cheaper to collect data in one way versus another. In\n",
-    "this case, there can be a mismatch between the training and test\n",
-    "data. This can lead to the neural network overfitting these small\n",
-    "differences between the test and training sets, and a poor performance\n",
-    "on the test set despite having a good performance on the validation\n",
-    "set. To rectify this, Andrew Ng suggests making two validation or dev\n",
-    "sets, one constructed from the training data and one constructed from\n",
-    "the test data. The difference between the performance of the algorithm\n",
-    "on these two validation sets quantifies the train-test mismatch. This\n",
-    "can serve as another important diagnostic when using DNNs for\n",
-    "supervised learning."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "58d322ba",
-   "metadata": {
-    "editable": true
-   },
-   "source": [
-    "## Limitations of supervised learning with deep networks\n",
-    "\n",
-    "Like all statistical methods, supervised learning using neural\n",
-    "networks has important limitations. This is especially important when\n",
-    "one seeks to apply these methods, especially to physics problems. Like\n",
-    "all tools, DNNs are not a universal solution. Often, the same or\n",
-    "better performance on a task can be achieved by using a few\n",
-    "hand-engineered features (or even a collection of random\n",
-    "features). \n",
-    "\n",
-    "Here we list some of the important limitations of supervised neural network based models. \n",
-    "\n",
-    "* **Need labeled data**. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).\n",
-    "\n",
-    "* **Supervised neural networks are extremely data intensive.** DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.\n",
-    "\n",
-    "* **Homogeneous data.** Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e. some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.\n",
-    "\n",
-    "* **Many problems are not about prediction.** In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a *wrong* model. The model might or might not be useful for understanding the underlying science.\n",
-    "\n",
-    "Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems."
-   ]
   }
  ],
  "metadata": {},
diff --git a/doc/src/week41/figures/nns.png b/doc/src/week41/figures/nns.png
new file mode 100644
index 0000000000000000000000000000000000000000..19e31ef05662705639bcf65a9c41f433a27399a8
GIT binary patch
literal 33457
zcmeEsWl&tfvoG!}4gnT-cXxN!1Wj0cad&sOV8IfcV2is2vS@JF;O-hE_{;yjx>c|0
zeR-emr(0d8W=>C^(^Ea&UDeaSIBiWOYz#6CI5;?L6=iu{I5-5@zvl-U;y=#u%IeX-
z0m)WILk146F$MGaBl5p@cn@7AS-83>;C~j;y<KH&;owj?{!K&SEQkba{xO%e)%6wL
z-rjO^b8T#FCMPGmy1H<2aW5_|{w;cYd&|qqudS`o($ezs^2Wu*)zsAJ>+4feQVtCb
z#l*yX`0yb+JDZ)IT~$?8TwEL<AOGs=s->mH*VlJzYfDj4QCL_Q007L*&6SmvF*7q8
z8X8hlQ`_0uAs`@p`}Pe314B|$(%jq}1qH>~+4<eOcVuK_XlQ7Hf`V9BSgx+F_4W0H
zgoOV7{`B<p!NI{uNJ!k=+@hkQKp-$BB}G$Hv#+nu)YSCXuU{Y#2n+@{Ha12@MQv<s
z=<4bk8ynxfR*=HMQNyXo%jo;UpY@=(-BhM=<f%@Moley2?^T;qc8+nj{d8VAD}2&C
zS-)QlczdH-qfkdcLQAR=N3HeJRE?!UYYnGH+jLhk+Lr#msDE7Ka%jmSgpYQc&1jR^
z5JMR#%!Atddc9{$cOCq&H?CE!WA~5gO9cqL-5bxYAK87~9nWs0r&3a*i9@vA>;Kgo
z_%J1t&$Nukhs*%@l7>hW>2X1QT7kPKB|NC7z1<(+8jAa@M^c66L$gjK5cN+c1#w!m
z7h1B7V-a+l)!k=lc{Hm#WfUQ=vmKg%xDoVl&4Ky$@T)j`^YIatvi9f8FEbWh<L!Pi
zSGd0~0bl<dL^^oCMDCD!OSR=)16ox1gZ%-ilPOhm?GNX+K{$Av-&c+&D-Lu%^^3Vn
zZ?!!>X3I<Es`lCtJ;;1|rKriWd%XctJN@QPvfTU|<$){&+$kzliGz26!ppYig*}Ua
z={6WaeINN4)0KelQq9m;peWGu`2C0Cubt)AecwLmNpJU00kxs8B5e=wOEoQN`{qkt
zFf&fSxQzm~REQSP)qOJ?j<+dkI&o4cr(DNhFc&^oVmg--Xl!=!h+(<m;zgdEyUqwW
z9fG$ehfjw2>d=_<yw|_1?KL}JQc%(R`XdZmSUNq|&lVH(_66%VyM;UR`}3a|B`@7B
z(iUU1Zk9Btr7wA4+83h@Zl`?fGX6^iQd{koKME}W`Bp>t@?KUkBep!Dc#`)s*XJzV
z1(hN5k&a9UIzS*IJzvC~mI_YTGD`dV#n8XA@bqPUj)rNhxvzo_wV4{?sLC_M!ONXE
zCs8P=;9dzyHVGTl2&&WI)8lav^^9M}dCC?XA^3d6fmq3*^?E_jVp!bF#j9X6c2W{L
zHB91n<g1brj69IOc=Pm8_)<-Jc`F8c4b=TrqH|W^fjXhJ{Z86@?yC{5mA-S-=*Q$V
zhVqRc@cU~_Q5*dLGHu@50JJ<eqqRr1_d~}&hpUrMYq1!yCq3Oe6FM1xE5ql>Sj9hX
ziNN((eNU!N)mUY(lxPTudc2DKYPJSj){*lz%3ewu$tcL2(GGl<>&N?(-L5W^{xjlz
zXHWW^vX+N{Da}l(nYHw@;-;dCy#6=!1EI3FANcv(3;=nL(C!lDy-?+>yw+%N_cGQh
zN5+C#+(tN*EmI)6a5hW0fdiHDEF=F3(uyY{bSM5KxEbA`2!BT4FDTc_>av1Swu+A`
z4rI<#B!x4`WF10l054)Ohn*f@GmBB@_m=d66s7DHj2O!De}qMdOy9!YJBhS!<?HIV
z?IU<lrJ7~&^Pbg>O1iu9m2aB@jgg&aW%I}(*3e+ywF*akh*_fM$DbaSBJE+PiT2C8
zK?|!7NT9UTpg{6+d<md2nlp75s41EIjmTY+mu6*Xm++Yk;EBC&;{5V`i;5<n!S2L_
zr9y`QVlfw5r7wl*j&L6YXz$QKik7k#VfxnE7kR_zwy6NjL3jS%0y5IWWFXf<hjBS{
z>7OE@?uX`UO<M*B!FkKxA%)DHkhog{+MRg29P{Fu{n87yeuO<^WX}t#v7u|uGsa~m
z313pXN)faReX<?fR|hI@O6ob%e-PZ0Mz}9ORnIMmLquP-bx4vQF9-Ts(rET9ur_J1
zsO2&MzVGx4Iub3;r0OyOP6xw~&drwdcqI@~@!Nk*W5h+938(h=V4CQa+iv!ws4@T2
zPLh|~?$1HQTw+b+Q~ggq+WDsf_JMw;i4;{5r5*G11CMD6ZAX|BN~+J)VbVYWNzmLU
zk?FskZ5n#tVR3r}Cj2vX!VMtwGS%n2{JM}&)?eD4`>$Lj?C(LrSx$d)TDWfnvQ)~>
zp<x1!MBen{xn5Z_bq}5`PIF=kd`C<i6eJ0311qD$T7-WoH>Y3^BeyTVl`7}V6giJQ
zCr#(y;__ZR^lwi$&;37q!O{L@2e@LPHv1646X8U~b?i65QJzhFIH_&Xei)p5|MKk1
zk>jwt*UyrzO}rcZWE#92fxFfg>EzNC5>H}lQX@^yT5p)uap-(d$y@ZbQObnj7vec<
zg!dXJ$<l{$15a0#e5^GxHOaj*mVPq<hUC5X#b#akSSt}$ZZkHp4osDjgM<E>0gd)v
zr+`-n&8E*9d0|pgfA7jKv(%VNVlr1!9xyDs)R))Eg<VqLY8F|=i!G#l@ajrCE-pvA
zWlR|HIRM>otLhMEUgzfHK}tU3_6+pt;oM&&-rVh@@r^0{`x;7Q&vnRwl9AnRNhydv
zN=yjnj13H}G_4uPmq}OcEJr`tO9Fv(S{{K?*SHxP9htlVYJk%|L-;)CIzqljZ@5>0
zR@MQ>%Do;z4_mtmfe4oKI~B&HT|Fy5{fD)FGs`JATWphhtRSV8{E1uq*_RbJOA^<6
zR1ENxn;rd9+fERzuPp%F8+lE{J#_WwjFUiCj8DqCj|;Wcv_@M-z&Gp}sW}7E!=<{;
ze_z{QFtmJ+U}?eGz)M-R(r3>Xq6uk(6B<dbhripC-Owjf3f4T-T=M5=w_t3nazOt*
zhy9#uwY!_xcu6>xjqv17LnNiq5@K_yxe#>pAqpe8jBBRRItuQLSH{Gh?xj<h8~%K-
z#qdPI5U=1F$*F++XnisaxBcWvcoIEyk}y30;+R6t6qN(K4DWKYqo~g*<{2$x3ba7~
zZPi+uiib)G_kD+O%yG?FXKVFdm%=0+(N1bF=urcR=Gxz5SkeOi$=^c(-R(Cr6L~xa
zuuBl#8UkHI$-nAsRI&Wry?cMNqLkO@$L86Svy9i?rOVe~kn>wIrQ^(>@ss(gbJbro
zfbr(kxUQ-_ZvyZzMM8lMDx!hPtKjG#0s!cfdN*_kn2OtuVT9?FOB4sP^h_rKZAP=x
zGclwjP@7vYavNl&8zrLNgEq}VAX-kv>}v(Nh#0Fz$%iWsqHnE+)w8)Gxpra{s7UP>
z&2$J@!JWCggw2d|8!UNS5^<KB(P7x$X&J^4FG*P{w};k4o!Ox6&=f?Gf?rosQ*kAC
z;Lg63IF@G+kl8=Dbm!tNkh3cmL~xU)f*D}meWO|s5B}ct?EQWSkm;Py-p6lCQFEK2
zihox5Fnc2J0cSg~cyd@t=IzdpD4;V_;KSNgK(^d^+Y_|qC1z`TFADBy-R(9kwuv-l
zV%Ql!JS$huk9_Sd6+(wS@%kE^=bVWZ3~L!2Iy}%-lWa56tn4fM8v^YrcE83ygy+}w
zBCBaY0gwZ|KK7{|<7ZOh-+0g|LGs<Clmyk^(73*W4DzIehBX^}m|54_Skrd&(T5#X
zsNk|&HR?o7(F)@V3aY&xbZVS~qNlSf%S86BsvnHoH1BLcl<n+JHq30lC1-4>McuVR
zxXhjU;ei3f1zvc`u-=+lSCe+*a?9JHnFa(05J1<piMGB&C;zY!Kzh*pZWC-9>T$I1
zyDg|%Oj8MB+g=Tbc~c_O)-v-QKO}8^$WQI*4Z0tNpG(2q<9zPOw<7M*rwItag&9<F
z5md+bc6laApnVx#J}$5MjSl03`552W9HoCpCDvH?T0K>U7z4N1X=1^0y*9>&u$kNe
zOeyR4f#vu5=&*DS+oztW+TmdLeDYO|N2`6#k8bzX5;CBTM6LR2mtl$+r+WI%S%I@i
zu#~*{BV%MP_s51);uxo0!oRPw(r4<pQpaqchqAreKh3$+=iz@&m6)v0{(*M;Cj)8V
z+-I6-I#(8Yt`;E){-|queOt66_aul9!6q)+rx6wu!_JzRT&w`bq^{-UBd*&;X*RtA
zPgPGa@F7}A1WB1n(8BWJfCs0OVJkFPZHdjngnP(_<%mJPl@|2QlJ25Nh{f1fOuTYj
z{Mg-zAIK@@_+@F|&?*7r?ROqE1ly>xP+m|mjpV{^o^}3d8Q|R)2^PwR5Zl_KS#$5`
zF&e+nmz--hSsyyx1=}`TH17O<vWmp)s92`~IetLYZ3CSL%ug=#cQLJ4y7GPh^G~Nc
zMgm@Lf5sE7K5FOs$2^<5=dWvT(=~@i5yd@eK^+hAXtX95s>&ZS@|;A1eaRUwX=%jz
zfwvfS-luT#XmXtBmJEhmKE0#F>>|bAexCN=moN5!wUe$m?`Y@XAeeGfKG>tj-D{ie
zqmidM#8IoIZkjq)PdvnLAOAA^05u9vd5tO|5%@Y)@Y%`}&k-p9y`0Qaq!UYhST#b2
z=>T`2y>pc4?7bqzDPy1POJmo>f$yyj7)@U6u;?P`CXcC?nbzX<WbYi-&$Hm?WFJkC
zy&~feQwU%WviVd=v>H^MKT*t{kwQ#<Sqh=RI3uRW;xi@+?bu%*0oHA7RPR=%mkZ8a
zy_USY_c>X+NYipi!HR)IS&C^Z90R&A6qi87*ms~V@0A%FXvAKpIU}g6ofqR<){Yhy
z<v_6UyGTxV7sJVb6bv_%zx~^Z-%$sXJxZO@iOhTzyWl|~-xO1u$v@X|ds()cm*n2}
z{cTZg$cbT%NeMu#BLV##f_`tmF(ur$i>^7kX+T6oD9&NV-nS-TCAbpuJ<}^d$ofkj
zWiINzGH*k6>kRJ{a^J?PBis6E_z>j-8y^Q8OrwM%tklcz8~+N@YIXqYkfGmAvOOgW
zxpb?|EuBf(h)G_8Cyw$r;TnNSFzCRM@R|8olK`E9PRs$#{qloD`RUU+`=upAJYQ)i
znRc9N3uC(NOc^RnWNK?BRYPneUoOIXcz2czM50Eqy>xjlU>}AjxrVup^NN;j-n9|a
zpgDEIHCCv<B!}~MExk-?x>lr7Z~`s=IAP<AS(CbU$^{`7IJ}D^6XEBLRPF6M8xUel
zsG-fbp?YT8#;=Xr%Z>7+A@A?KA%<)TLSWOD6p)%<2xp3&2I~TNIc*DN#%CP_D7t-`
z(T(=rMN6LTy9zE6WVM*E@UrNM-bAIe1Zc7RNovrQbeH`Ko8BwmY2LFb50<fMeG&mH
zoK=js9}&nfBQ`tCG2~ChIkmiv;u@}?l+TwYp{ZiFND4`p8#me8R0pTwIsG2N(pc7&
zo!xC|Zzjq21hDKGX5MOJqtYy>w_N@sp{DScZDMDH9oj{_&r*2@NhP&?8>sUlpPVV@
zc`Q1Unuc)~a$e9cSOv{|`942@hy3;<)f{SnoJS;MrNyfaCE@!@y|RZ}6W>a8nhu3X
zcq3#6n0|m#rqD+0A)AvyU&TlaUK`j9R%lwDMCnq1hR1>XNU>hspHLB}^@%1KsSfzg
zib3Cs0jTT_;hqeTt#ACtJPF~r*c|c3<;D2|o=}2%Bw&Be%{l6KrNv~!R*|}{B=GOR
zl8@HQ&~-4OJ@u0ks!j>@d6W2T3#Ktvzxdr<=#arQEv$?s?5e<P9lB^nUfZwVdmMyv
z$T3RH)LQf9=EkmoDX;R^lR2W~qfuRQW&R2H<JP~!w5QAIAyRTlN$pu+o|V3vs8hNe
zqiKPad|_|PKR)-Ia`wkT_uv4RG}xmn*+v(DN>w2kFz%#wDeu@tQ7Rai=4-Aq!kTG!
zjZNG7hA8=`NU*U2(((wjT&HwCjqret=Q))Q{!SL4BKqn_c43%AZQaOehnZs2YpJKW
zdUS7?{o8%207Ha-rJ5y5b=z*&SM_-Xe)bYoqQ-9qiA}beyb9wqt8p%!6!wQO;k4p;
ze*~&%`>84md+~%PpHR~8h~A($g}4rz$Rab|s?hVRU@A=0jGQedR%EP~rG=p9wD!D&
zi_Ggc>xR+(oY{%kD0LMzdx>>qDh&bu&t<%9#roLU)l3=d*H2N#pltW;z{Jz;=AS4M
zwO&Ip&{br4h7V+D9tLX5Db&Ya3yuV!uIMzZxf~e+|5YcY%aTFUA6Ojs@aOK|TfT>a
zds=J^#s&N{Nu|Zr#A2}qxiC+UQP+8^I{L9C@L?U{LujytH6B4~Y-G%kCsY%S5f#Oz
zrjpjb)T5S_*kpn;irUl0G#`V9RY8uYE3d>KcBX?2dJG@V#zlF5{;Z8W?rN4#q%5Ok
zg-o|4=z8PcX;O#NbRqAGoJWz10%vyYW93_Qo3Q)v3=?Pd5@iQs)!RsC)M%!Q-@6S&
zn_;y$lILoi7Z~0)qDy+KiOZ`=FNw}q^*BZt@=EqTl?g0<F-fGH#3K?Ed-oyY@pj9L
zGjm<+_o)o_XxbWvAJ5NRetZ~Wpi}NyZeF<M;tvvABo5TDc-?jRH!DD^ploZ)s5k~v
zrRY&uTt-Lsc;V>&cU8NO*Imb#59#w;Mh8M8eFNk-{qyKtU4>Rp;qt85_-TzcA-Jtl
z%6;)wi*{V>aT&=-1vVB%w2b($q2x?r2o>FPJibgi*h&RaOc^)7E*=q?uj=9YeuX*!
z3;cNgu?t^bT0_?^!_(>RDNxs_%-Z&hu%gGW!X&3<i%ZP;SG%aN;Ai|UIGxA+WSOtC
zo2*A-(l4SBe9jGyrMSl(d7dfEq5#;#CbvCp7kLt;b#6{eYTNa_3M)L^DulDE^XQ!=
z7f4=9irNA-*sQ--I(00?yh=VyoVd?FlcJL~@yo*(;%ib33XZ8KFLWNFQSk51S?-9u
znZri^{&4zy%HrYWNU)8%vF&vzT;}L4{;KCSJ%X|Qs-hc{h#T}ghP`hFUW`maGV=?{
zL&W2cr+0QG+P;{1L`WR6ibfE&-=A7?2$uENrQ<zayxpz*EDNpV6Yp5B7HqD?#xl7-
z6ka9h8CtGq2~?Nx%j*rBKEvsq;GYff1JDXhuhqA)Lif$sEcIE-#OE4?6l8%AdzGs8
z=!H*c@6PT#GH?f<V`t%}*H6hA<mb+VuwYTs3hq_S;D!$q*x<|<*<ob#5^@me^0x)_
zm?=Z(u)6SCjCNyIs4oxlxklyYF0qctW4u*cBLzy$R=-woIvgsO;Z|71mxnuIP4A3S
zb}Oo+8)&hywQg@Ay3Uhd>4H*E7toLsTTP51$e~OoiTX;W%DggzR)6iV`{5(?(~_V9
z!LQTtE4Y<I9K`%+C82COTI?Ls<d}y|HNZ?kjpRhy;f;*&!Fl=Pn*uZ%>Sb0MoH{u0
zJ?sZ>Nnzy<e5NFZOVL1LU43#ssL&+xsOh9F|0C^UbJ`3m=3n##gfL6(WK`N3JHHSF
zI%K`vOA}~;5uX?PFEXdnV)(B(%X^nI1X6p(y4syHT8u9VPc?RN-Q6v<kF!Fo`%ti}
zd;*|M;fsfpDxwK4IxSRa>s0J7;rgzM`7wKGjPB@6|7{{V0y8$J%@Ndj%rK}eRxFXa
zF)8w6ga2SK-4A{hQ2Io6;VqR`G9z`~`+zGkb~GJGK$25r>KUB(lBae~bIL11Uaqf?
zf&WfJc#enuNAfvMSP6Ohls5al0u(S4RT9^ZQPsktwWN9Hl@%{<@(TD+rhfdwvC8*O
z6jIxa2q}3;;Up|*!xnSSD}N4<e4rQPVtTL9vE`v_H=d<$5(h?Y;OW9y`NkR_kD`-q
zdy~T{g{sVcn5~O@)1P!PiLaD9j$?T<h=<!(uE=NzjaH|$NG%|o0;&*JgYe6Glv^<T
z0KSyxsl)17C3dF$Hl>%=oaz3;J7X(1VkwAhOEObz;2<zgbU3d`ps^+qMi6SIQ|~%%
z1=63oQoI1qDnA$O_S$KE&qOZiySc(q+wTLk%C&soiV^R1EH;D>#$-mzJks{QjWjX1
z!@)@G=dP_q03_pxKe=$-^*-}BoxJ~bzdsR*tau@`?Gbu!Td<A>_`c$@>ni9^eTr_g
zJE`PtTHBmdd_X7WUb?T-yN&jI+PFFr-Q>QvPpKW1>h#FtWYjp|E^0v@_w}p)`pL?V
zcMPoGw8-pFAwD*WzP&@ZkRlJpaGQCv6HJ@&)5=e*?0CuXIkX}on=9Zo(uakxp>TwZ
zbX-V&2;YwLO(~;c+J+=_J$70vfAZRp)aFy;_v?4B4<lVPpo4KE>?qa+=DzI6Q`{o+
zdrhCdT%}=4%#(|$+yOt>@Q=HoXc~~kK38iZjl!=5B|~*dmML>!zuN$26LA2M_OVFK
zC*sc|=@0r9)Q}_v^I2w{T2F{ktF!q_E}wv-Z|ba~JDhX+;;FRhVBHMkFpg~NQW;+?
z_yw#{mIuoF=4cu<X*??zAH|sK#Ct%mg(;!b8IX@@H2KV&i~Xi4G#3sXc{NW)4|H;0
zEwJUjDS5A$UPu92I5EGIY*$b9JQ-zIF!CaV^q3Gu{;L{PhBkJpMDq-wQ+wgmQ{U*D
zm>RD0mM&jCVw+VyOxIH-pDG;GePisnUg@#EoGe!JaLW6B%q;6A^pqTVikF~?=W4Qx
zu)4ANHd_C!880w`GFk%)-osqO*2`f735Gh2g-3$5y1IpZVI>*0l5IW{XtNjD!cnJc
zQ`v(zK2Qhm+Em!Xnq&es&C@XM9bw<ZxsQRSi}CZ{`PB&VGB9r7QX4nVb-uiy;mn+D
z{y|Ij(dY5g6T2xf8WZ<h(>~+0Qm%SY5;rmd<KNqVaN?x}$14pS<Oh7j7a$gH|0>sT
zzW~O9KB=d|dEY+(_MK#fR-p?oE_}$+l}p>aG1xXa)|<`VU?eQfS@w=~DtPrrnNgU8
zP5}?_GB^Ys#^g}Rmp0oyBsyzrnG_&<m%Hgn`@xdgd{@jJM6}7r3sI@OCV&vJ%_a?m
znMhNT;aS{I$2?|5o-PSYbf`&go({BNP2cTM*=D6m?5kkV0Q&U<-EpkIMY|MIv#r2j
zjlED}O~XFKobCp+Q^q##J(<x#lQDV_)jZaPw)zIi?@GLb66gYB1wEz7>BBx%+<N4x
zUh8ki*iTPFpO+`t*wFZxgr1`m1XI^MjINfR?;n%;%I&3sszmOxhPPTg*M3Vqcf2nB
zaYx#LtBoL?Q|a>VyAge0ww`GGg*!_WG?$L_w&3R&kVS45pgd7OXz;8cCCn>mX1gmb
z6Z0<c?ZcC7EEg_}t0gGXPVGW-JdAbd&P|vakbhcea2<<Csx>|+zl<(2ZPpm%sWYQ8
zhe^_SVZrhxl5ie!UqPMK5RVA6v$!;X4bzWYH4BL%g`H_5j&HC$1x{z7*zwf*{T3EB
z+|~wpdHPZ|eM%7FZ@2lPLp8AEqX^MV`1S|Sn;k->1w{X8D>dJ!STV13s6X;Y!0_wg
z7hvThd30j9U^%GiiSL-McvLd%=8L$7eorAH_jT5FpqlnKO{Niqi$7k^!iK#3^{@WU
zj2N1O@HV?A0>cZ!kO$AZe-1Nuc||`hB0vh-<+~q;CQZG9y<^oWngSeJ8Mn<R{t3_h
z5P()Wmv8VbHmbcm4gb)+G9D@SMiYS-iHp>8FLrdk^#EB<XM!~aI^X8h6N*uJrW_<0
zEC_V!_oR8=J$TAO(uhsgB5I6oegtepWP&bN@<%TUq(~qBen~V-F+KCBSpTAgD1Fbv
zs5O)8EGdEN&U-!C@a3b!u0Klo$NghM&o5ylanX3FP^izZ37sA0bRxL-ehA7R+4B`s
z4982AM&aGDd6ELJH7@V{5a`^^87)!CkCM#2J-5d{laM~#h~yr2+aKP2TCwmF+wipG
zf76xEKwR;c=b_us-56MsBhNUl;E~Lm%q+C=uv7yLL4N}4Z3$@bE!+NP(_%)fBb}Bd
z`E+_3t4a=q<56$8>|)YG%w;AyEGkmL4{0VuYAG(`@VwVfmvL~v>t|co5-lz64SJl0
zLcoI^X)YO}`|lKcL0?OU*zI%`1nq}#GSCwPJ9lo!{0)|i_L05M5cYeWnju-YTBE-P
z!-f?lU69RrFwy!Jnz@qN6ePDWd9OKgGIBQwJCs};t%{f%2A^#|*mDn*{<_H{eYgvy
z&Q(?{tbcMNuHdJ9zZy)WBH}XeZ#zj8^=(j6zf_15)kwF_;x{m>ozy+QH}GmN0|gA)
z{o{e-DyEn$Yso~kvH>VM!6toJO<p3uh98N4?C5F<N4mp2h!)&68gmdp%}I0dfx0D?
zuELr%5X9_hYiO=$ctg`%>UM6o{pAW5&yqKA(N9a`*qsZTn7%Yw?MW(1O2xQi(ccrS
z&^NfKz2LvdR|qj}!R;D?PSVp{(spj+^N9QJBtPZbFN?8>v9T7W*X>KwLvqu&t_h4a
zG74o-to{Jt)EV&#MvLw#6?+YmmuiHwv-0nW{+b>O!WxVO#!;06W7D{PRA8L2j@vYi
zY90q4o46ZeH$L-q)6s39Brly957)lS&rvz#Q)xF}0#eZk@Ld;_&{BN)e0(sGL^%>E
z#hg!H|1<Hyl!p7->Cm_5vkwU2$G$k5c$qsGJ>Rvbm114$g)K4gW8Ge<vh<EQ{|%kV
z^B5PMx|#nIF|b%?@$R3JnpewDZQ%T$SY=(Tc}xRjzbE`LbO9It37nM);zkapYAiOl
zBU)cA)eFsVvR9g#tQ#k`ba$~lM9}Wa<duK^7~xA+GPvSo(L6uKD!XQThM$9Ap701J
z{1q&xHnyiB?218WL0Hf4i`aeEq_fn*5cI_YeO_nc`7(`C81tysQ-@$3e+qq`in6MS
z6`B~xhho#jyTRu^!{#QjN`Gsm^|?=PUAT@Xsd2q(0Kb+8C|r}q6@5p@<ktTaeqxg1
z-1qDoLlhGLJp9jTdIqoj*&g*I;}8LWe(1gEPe)V<7G^g~Oh9)*noELeg}9{Xzc{>i
z|9O%!wX5GZB+{Q47?Yicp$=Tm`j>SIpjZA&)#VdXc&O=V7X$)#Ox3K3fwrFAukswJ
z)t!j{T<M}Tc}YxP$ofG=7>;gZl%ifrMXuDrc{J>wJ<{`8>YH5Q6npR8yCpGq@F{Hf
z;rOn5zU|!R=H}T2h@e{TswFRRxkSnLF9MYnduv7VuEpU0lNfH2mzXNOX}2OgLK6c=
zuO1_Wt`BM`|M&Edz*9`U8}D_H@x4vce_;XsGQK7f(bHMDcfTfbyfvOoBnDD-#aDS3
zr6R&4M9f15ZQc$IG!=VO)(4B4g1N65exCe--B5KbPC&DP^-}HxA*r1+$@^H9!?&Ux
zFAvCyy}Q0IG9O+ej3NN!)h7D9Ll=CeFwidMAEq7&5>!#}@f%#j)m$WVM4C(1ju%Y;
z(z(T~_2`PgkMkR63Do$`Uhmr<f@kkd%(LZqT&9I$FUl)(6VL%7TE8D3N7O_jg+u2?
zRveiT5(7_w^$92k!Rb>GCyZ&6A%2#RMZyh;sG*%E(!r92Chs1on@Nu}Cc9#e%~8iz
zOFpxV|N04!McG5<jjYwH0jz(@8$ARl5nm1FtsLxlR$ny&4~y-()P9ienuwUo<ukoJ
z#_<u0KwNsdCJ6I{8;DS;n}V)A#1K(ZlVlupw)sAd46~QQIT<u%{;JrQ_KJvZ=$je`
z-PU$wf|=iErD#pw=szF|PdC3?VEJO8uy6*sPDs`06aa+FmlYB;$)e7S5nT<4q%Mi@
zBOmsXBHpx|yWF~uw-mj%emPYo97v>3DNXn*lC=)*4?EUfgFo)<wYVl9@~|++bT0Z=
z7fWD59T+P=T6OOtU3oG9PKV)hHXolTI_9uGj@r@`SuezjWH}``ouuC)YX^xw<{geX
zYv?hNquVMz_EDe~N6q_?i488ZS1erxY#=+j2U9?8EC^~)?%-A{n$L1F-TjpS{AXF@
z3VDWDG*s3G3-=N?r-tgD+a9deuPL2J4o57v$yn<Aeb}h+r@AVcE^3}^r(DvlgYF(R
zyhSWQE3<meafktK#YDdYIBzKlBmJNU9wckF^&d(D2~fcRPj_Brl~c7C8tZYwATf7;
zO`!7s(EpbOw1K{rU7i`jmtn%Lj#Apzx&}bt->sASKAFpZXqd9T=lGao8JFO<N)2Ym
zlv?i)mg5gK_EtTt%|1tWtH8gH<Jm><%ND(XM*^}T|3T-lo;{qo{L!`KclmAW-NAob
zdn8}Gjt_JCbLRwcLGDE;rKvqqQo7q?vgl1^PU-zk?3(MQqCcq&VRX=01j78@x3)d{
z*`Gir6_QaPe+;wFm{MU;T}9Jk83JEK1kb+Nk3YV88XfUs#~;0`mXvzB-V+l~Oyw9f
z=&s7|ETogGt^pKPbsP27=aG@HL8`(Ih%sWKu@GT4AL|wGcBYuKH;NTkrKN6|uvuzn
zaKnr~m@BOrYE@&K`DDvZXA(quM$e-w)luiG7^bAY=%HOv4caHQ{YiC#8-dzpqv9|7
z0HS)G`R=xGpm5VGYU@H0nDwQJ6Z!k_Wl<sFW{Wpj;o5})IGhrlU%T{i(mVrxA0;4N
zd11?GE)xIxtRe=DG9(vdh8RCEDGKt-9)_lT?I<zaNm&4HwdauH%6vOS+AN~=j66-K
zElf<$gRfBZHWS3^rS5Q0`2@$z2&qS+Vmw0%@nB@X(4tHlea25y7WU3Wrp;KkKaTF<
zLn<51CDNi$Mn<5tHZx?d?)?S|RC&i6D!Ooz3!Ng<2(_4^Z(@VX<4IaJ+)xUH5yP}a
zd$lpqL5{@uiR>yGaXtJXBT>h$tF}1yGUmj|AnR^Q1O1JFbwsL;I50GoX^SZK;2GKf
zxRu6}AQbM%Wau)n+UuIA$DUeu{ImHaxy=s)w2r1ans2B1kNzDuUcX*F0eMhC4@m+D
zysbdy+y%q1D8yr(Va#aTsHKGH-2eG}Jllok3(Zr7@8flbi*GmOy_64wTZ|j1%_|KL
zSQu&BFEnKP!6HsoMbHriY1`el*{E_;b?7SRJ=8hu1?Z_JJKz`2N}cq#n6$VP&Vs&)
z=dScOPjgAKy0!l2Y^N$C4wY4>%1Z<xC+tsbdNw{mb09QDj$*2r1p$qSi|m?Bq8>$A
zN#bzc1#x4LZcnj#Tq*ln@SF3khBaVRM%H9Z?$J6OaW{MnW$;@!e$*zHnp~e9is^hX
z56R}!J>fqm;dnjVrds53yPwNp_XWm*c@(8D(5sc}X$vsAC9xqVjykI}O&4W0w{lb|
z300I|rqe$@xOd<&)eBJl4(&nSD&jj-#g3x>d_y*Q<@*_-0QusoI4gBRQ8yVpX2qN;
zDen5=unwVDwr3kUwS4kaK$DMCagx4onU<Mpk(LQryO3qzr$anM+ySM=fwes=q@qir
zpv0<ZN`l2$L`WnA3&#egW(<%Mu~Il_^bfh3?7#3P9jjKPVcED6sE%~{kkk<MU&Qk|
zE3*q2Whpwtsn~z7v)K<hEvM{azT0dkM+zE5j?)AVi`SdIMr8Qm!ukd6)#Ndi1a()*
zqJ3PS!lLc$mPd-Cd^rNPtBu}ES<jl;8Rdotq;H`iY80z0wGsvwr-&-d!%-o;Hevm(
zKOWLAUnfVl=zO?7`^80se(u8jiqo#+kS2t9E9;&VdAp~JQJr%Q)9d2y-^7+fMN2-@
z*N1m>AvC)FYRSL<jQTKUga{GI@<rAg>knCMi}-m=+kO=fB{o(o<CEy+%lrzu%i`9-
z0wOi4OUe46%X5R|B-39r5hr6Y1>mmJ)Sde=s1?38b<X)vXM_i?iwTc!FyeB<aa_xe
z!l70uJk!RmBft`gV}BAs8Lb~HGAU@DOdMQ!KL={ika!IhmcOci<n)FY@U6;#HI;9~
zpdvWfP99R~Bpm&Nl&rTK8l;5Fl?G)@h5Trm=sG#o2>HA9ns_igMKqW?E2!ttDChMD
zJ=SmdD-bI3rBtPWv2LIk+3^VQH!Tvsy>&4NsxkWhE`gd=2_ITb$AI#dg-s*_pm}18
zuZR?ae>C~*P6{FYCQU)WYg%Zg+(!WFBZuUw+eqO0B5>n=_Irc+dLlu1g2v<_>9A~6
z`9|(v1VmT}o<qe$sT?$L+$4uSNW-T@rG5jPvIObw*aOlOdo3?fQ{h3D;<lUvYta37
zvR?kfXGy6qQAE8I5R+x#Iws7tI}95P`b+39oDpg&^MfsiuoE#gNtU$XeLgg#qfSgX
zO=Ap=bH^m|ib61JtMdaDvKC}1B$b6)jRCGYb(1&t1tYfMBIT=gaV8y;LTf5)BDOzY
zrn*8uEP|6|1UNiu6j6xHhfdDzicPXfYmc}<SW9ziKr$i5y#k%`7tHg-0yhCPL2Mlr
zx4e<y`NQ@2gi&xE4UoEt=kKhIZ9BRv0t|$^{qnED`Gy}(dYoLA#4Yi3;5&>J&PX1l
z^+oZa2i+-h1)G{Nr*P62s<3{Ph7FgQ68_yf(p0*W#)QT(<mTJGk|uy6wCj5;7@jKN
zbF|gFJ<rDJKOVFu%;EBlU(Zxz^}^%!EEyO0*0v};xxecywpX}l;SO~v<I2@n##AKC
zFxnsV=^{1WZ-uD#l#`>_k;%L!;#V1M0e4L_zeRzMU5z<J%Vs`?Cig481zHn7y`r?!
zTG;dEYmCE<J`)u=#5j+$B5^h)h&~sjC4DUfGr!wTLf2YEde_|qIwk7Wty6Eh3fG@z
zyO!a}h{k|zQuvG6$SWDHT7;%fHZ!7OZUTR#rX#PQovh6`r8~t-S%L=4<8Zw5$nYmH
zep>X_aT00By{6;i|LDt|k{NNO>q3g3$=;#+Ve1z&j#%=qCltJS0Ij(a;~zI7RY)`c
zzIJQgn)0QhdNU9vXfRG;f$6_pn8S`<oi=$Xn@}$x+8$fZzcreQ%}oG;`r-&~w67lg
zmBDNX0|w(L3X%d(Y<0YlH4Yk6&ddp9{XLU_Fl_e5;@mIR(o`)dCrzTNp^weC8WUz3
ze{k!9)%7zXM}}=2f(IpitOawYB0sgKB|RYU<jQBx4)0Meq=h89b%e|sgAnmVdE~z3
zV*@}LCjEZ~fE1xX*QY`rVR*LPY|kLVSh(6H_2a4FA?@q*0}^;Lk|B7)KJt<rZ0Au_
zIFi_1DuIAc4KkjC+NX)j8;Dc-NHqD~-wr>aoF@dzxzKSkK};UW9QWbiAcwE$*Hyam
zboHk4>qP<>jt<=MoXy=Ayj!D0vRmjJH%y-S?W*Or7Lg9ayY0F2M*ZGfMEU3opZU_@
zo18r~`LP{y^cdy1>v<+^=_JvRiNFXV>3J@pyY;yDsO_?5{>qnJ8*Y|!U-L*2m5tm?
zofa<H7~f3Y^l~B)cjxK3l9IgD<*nk6qFz+F7JoK5J*jpdZ_WgrVQg*w7V`a65Slwb
z<y&NX^v3D_>nv|(x_mY>S+(7FvNW2|Nx-s4{~wY$0va0mm&?}1Mj`tf8=J5MBOp?k
z`sbe#VV;XkrojqeEwt1|D|xJzf6+>|bvX6r-B4I<>Wdn6&cogNs6oZSL`K8QFFzws
zMOIbakWpJ@TTRjIc|L=KOGD~AN)^Bwlq_czJE6@3b#=n?fEdwmZ5wRUf}k;KxBr(m
zo7|{1B7<9r-`H<8ps(y4J?({;sdP9$e7e?2VFdyjFOuCMYz@pydG}jfI*(J(=#}XJ
z9uu7>LrXulPechskVcELWx=Y{S0NFfF_wl5hI`mRtKhOjoCrQPY4;^3%@gi^p8(|H
zT-5bXq^v@K<6IOn^(7%ezFzCD8mcY4a8&fF%KO2gR|XK(;4wUGxpL&2n~%c@St`G~
zxp<^{7j9h=T*HzgI1{OxJ6pS*zsXGN3Waj8`3}S7@LGRr%J1zUCG^y;;k)OQ$qX3+
zjL5`XI{$H2lxp3>DjTqRxwh*JPQrbjK<8++I*;GZX0(Qm3<CC{n3cQD6=(sz>8B`J
z^7O3`nW#&$lQm80=F0E#f`T<*6Yy;k5pR-l5%p@_)qcDg+fjJY-Ogwb1olVm4Y@yj
zn3xGMftq;sB6^6|xF~7CCox3g%t&60EVKDK#y4e6UFFfu=OOz#ho;T0Wq6i48ER8R
zw>(iYC><FRSaPbyLWw~F9l_~p_(4&`T+y&3M%rpE;2_(m+3&8uS=&Dtha9W2`B6;7
z3C6Q>fj5ieVt0fSNm;uU2$Jo^(zp#5<IH$J;5U-Jz$bv@SgwPpUv0a>QeqDFh*-Lf
z1FXA(B31)3W@y<jNxHNku|mBM!1bHTd7_U<CtbcL(353q0I~Kfz;f%y*ktCrIhs4P
zZm<^I^NzF)B0u#LeL*KRz10gA85QW`A(}3!V1Izu%mC#91mfo3a77>FV%(NsaE|sB
z)wQeGJZ1p+v6ARG&8*rZ7EIIBV~N*a;$*J<?`|N9jvQECU*zfVEDoz{15=rEqT!9K
zOOuz_F|`YS9^KX1C<G#E*SSPuA6B5HuCQ$F@|#x?;BqK$-tcSHg*No`<pLButuX1j
z_sy`LlxZ4&$vv2aDd4LOk(SFxq=fQq4iK+KOdfxwV4(G$5iX^92PvU#oRpKNxXk$D
z)Oh4fz4HZXb3=qvN%upqR|9b3#cjSRrpN0%F6mJ@g?V9WIq+`l%Rd`H7;XY9l@sXz
z5ua|tnhD~&54)rFXiNZJR}8!i=o{NfHKC>ik%McHNtR!gNv0UV>+l<e5w}TS_P*0?
zi!0&y$%6XOjfEjJFRrW~IPY7^u$$|@lV_UPd&szHg64?%uwb~$gYlspi8=V=!M|9V
z!mMXYxpnF7;T<)9;V{Tp#ATHH@KXIhKN;?o)Ma3BY5^HKy(9U1$nXoUFoTAiLN(*y
zw1K(2N!EnO5NW>pGyxTb=be5bKCC$FB8$lvLuOE8kT^q&`QHJ2q`(=K4&A%0W|M1e
zeoMcxntjmBFNm8Fg%#$oJb6OPtu7D40km<TB0A{sRQ&1JOFdeX+=h-qjWH@An+Vfh
z`@KC?6M{k<*aZSHOVg2bX0pDxAZJV^v8BvX1Sz6!ZU6osD7ZHY|L)rlcJr8?_0fLS
zcf`fid^oKiwoTZoNF|96Zr_%(8Pl*e|1@G|;!e4H=1d%;qWh|XtO)OjEw`X3O($YV
zf}Jjl^*ddJO`rHtM>)6e{8G<Gn8d7qz8uS!V8ISu#uAUz2GdnF`V}#E$y_g}{%XY#
z7i!t)QT+v2;!W*>W5xo!{o*<PwY2gF!YE<`tm^3V&zKu=rl?T`tTrUA!Kwc&uzK(W
zD4C3n-o~~PG}~VN`KY*6A$yk}J`9J<XTv8sCnXuHpHk5czhwP0SoIO*YozzQj0J5-
zr6O2qG!Z6lxGw#3-`W29EcKbt#|4x4koh9irYB7$Y&)jlq~yZmEt;KVB8Y0Nkxaq=
zup5V)`l;~KuXP{V4>R#sSmZ+0p+X&gYz@+8L5@fFj5*Vzf$6r0R8q$SF0OBemILdL
zp6kk;$vbel<d)k3Lu_Gt*yLx@>5ZTFIT-H3*i{u60QCP&<O<ll0lD=o6w+mtc246w
z1jHx~<9W8vemyQ&sh$>!uvNl1HQf}1`8+R6eTV;P0I}+59rZmhtv-!=A%q!7KDFU9
zh<VuToWgA8eHyt$oDBWO|CX?jXE(WdpL*z%S4W(?H_cUIf^|o#dS-jk7m}RX+OS`U
z3aFzKuJYlZ?<6;N1=)p}Nup0qra6e0aS5qmis^(s!XA}Y%A>A=vx-Jx6fX|owkl3V
zp$(8;6i4^Riu;C?E=#d+OPtMGL6ohE@%4pZERhxRo21A5D0T@+(gLol2@SlpJ$eKF
zX?Dh%NcHU;U&z{lNyqKsHqcr&Fj&Y$?aa>O{TG_ff#Lg)pJV7hrmQ8UP72od<<T3`
zn@qVXUZVtxX%WK-N?8oj#(l8Fw6nT*@`eMZ%1-8HebO_lY41oOLTZh|`K%Cg=~O&<
zUpDOAZ^{Ncl(yV8vY_bs*h<uOz<WJ3g4$rB7YePO-M=)ECO7HV`JAc>mMT496yyJN
z&mo#Q;Pp|hQKL9%ZokB0C)WAyqs>ZS>Y<&u$%IbQ-!1w&)v|^T;HIsTuWkvcV5p$J
zzH~zvI|Ill-?LM={eFK<DAAw%LQTtnV!-G(!>#`wd8d(3uyhsO`WI@~`TF_dQlvAH
zRjgs;2D#+|Uy8B!0ZPIrs&HT89U_Ni0{S+I1O9X(3yx!=mO3?*c_RvxGPBv~kii~U
zL*<CB5Xo&lt|HcmFX<Q>g>f^N34I8vG@#K(iAS|-D5-Q1;G?3z#5^2rGMeFD1tmJ{
zz7Rlhfwm>?Pb|BypA2PX&)nEGO|SCreU+F}&P^}YegzH{_?2!PC#DYEEPgIP`CLEO
z;s$uO{Hot9F0c&u+mRZ2jWj`4xooMc!?Ixe*I&QmO2{ZlHm!R6og9c#BgkUXId{qS
zbwM-#Pa{0WnL{kxY-*77-R<lju78-7IiprC(?fx~6GJHpIJZSvB>c}w=%HXY$pz#X
zih5|vyKy`;z#qDh6suYAV$m5a;&*hBKWk-B?A)3mE)IcUF3EJ)fhoz+Jp1rh(|YF&
z-?~YGw82lZHo+>oKB2HL@0FNieqvyvPi0V~I6Co(%mwYw<>|fyLZcC7Z!Z_61vQaz
zl6Y<-djmca$!M`*9Td!ko~x|GG}(ncSi)&_X~jhpQAolignyW!I|$BEX0Pk>rAFV}
z&3Ls`smliZ`1Sy=enr8=XDz6xt@<fxM==Ub06sd5EoCEjLsOXGhP_snSYdLNc4f-C
zWt!8Wj<4mneFh&m<kzEf;D^7|Nw%67lQzhat!5cF;u(mREmlCmF;LHCfP1b`Wv=WB
zCD9(>(6V^Omb{2_vM+$Ywkr6$;kR9GBM->fL2<IC!<^)3%t^%ekx$=N?I3fL6Iyu3
zpq6j@$#^|<hSbV8;~l&*Qc)@T)Ja#YF+89gS9UW<eLsJU0DIDtYf^YQrT@&4`tZ(0
z7ZcP_rvMxXbyimxg8Qh<7g^p#3Uq~#Wkg2(ixR0ydz&^hw+IW4dh&S-TLvX*?=f%N
zS!n<F4tri<M!Yl*gVcOAVxb)xxl#y`-ZN7YV!vv+Bapg+kKD({NS<Sorp<MG$KP>B
z(BG4;|Go}yh3rEY@(8#r-IWTl%lGgSTez+(U0UVK!^?8+b_c6G-L=Tv&*>cz+K^`I
zqEqq{zcbR*psHe*Y3*4#4MBwG;-x;wQE@u=5=UYY1!(%fw`Gz4yO5%UB_m1E&jOeg
zxTYf_9fxq&(f1r&Z8RUjATdUBAIN+pj|Qvf5Fl5ih{-y`m7SDTX7KCZKhXZ_p8B`p
z1rAKmmTNMhm%StQ3BdVvqq8uQi04Or%8vctPvVg-Csg?!rVWWhDZk#|uRy0GoFz5|
z_9j3T(v|eJb7-(74awQHB|7qjcD@2ggc^O#+}a|yPxTQC<d(XNG-Yw+QR-EVJuUD7
zr}I4%<fp^k-ZdBDgLpQg>C&ScIkM30Mcw;E2|*+}&N6!<p5bai?KQcwf|CMN*qLuL
zK;<DnL^$d%n{%0o#qdp%{iMz0`=yZQ7gAC3ce^KVKsBrnNp9qGYx8|F&k(b{@P0<|
z1l{kCc8U`sB*blHBN1Scn`sQklfK1`?9Uw90U`4o6TfpB-A>+F36T5WSp?scTeFMD
zL)}CI4^L*BO1K29_UfzNwq9#O`eauZ?-@<(m`Va0KZKL_w|8mI)t#Y@c!^oZt_W^x
zH;s~|Eo`in$~-8v4E#;Yeff}W_HNc?9{kty&$Uko8T(j0jcE?s!z?f(5*6t#sMJ*A
zq_=ZWqiwL675&2YqG2kJG^q&A{=;sV3u@fM<LfF<Bn8tm98G1H9G2KTxVT)}C1lds
zVR7s4u@xKQjHB=wj8h>06v6NPEBsI!^7L|LyhA+t-KoxJ>{F3nX(}!BW$k#HR({?L
z1m-K^`{*vs&PordNu^tS6-7!6Zej1PJGj2HfVk0B1&gsgdobv7q|-C7Ib+-B!Imfy
ze9})L7;=BnjGs^so#DTU=<-e;fZI@q_SEOO$iL_0c-w!(?=*YT>D#SEAWIU-8tRek
zy5gDb>Fd+x_hVVxikh>o{2s7^#jscn_nY73)8pEi^k=&9ZNRW>R7yoUr>2n!fI(W!
zHu}<!S1YS^d&G0kUHvW_cV{|ReaRTcL{vue5oL=);4><Ow0O&wWs%D3b|Z0un~l}y
zxD>z=zCOOFvw8AUM`60RxW%-~`t-flDfK5T3o}l+GSdoSTv{p9@H>SmBtX4S-?!Rg
zPaf_&)Dl0kSrV)?>@_XsxQEGgPC2xTfJhx&48i>BuyRgFgwQKBg?IgnBjz*-Kvk7&
z<rfu2&zCW)X~tCLVQI$Sw<JAH?+dZ<pbbBvOHqsrQ4?0vy_PlnlNPQ<@qYVruP`&Y
zO-D~0v-B;8;Hu7(SNe1iXE#$)_}2v#o=b4yq-~g>VV#oaOkP;N967ZJK3!H%AN6P2
zL%9tAO0;2RI^_1MrX$P4&Z2lOvxnH~U`3rNeXNrO50ABD!7458hnLD01vf^#m-MHb
zSoLG^xj+ZAj^N@3-S`&i0{^v5*5aUn#8FHyK@ST0$D75*&3~Y~dF*MMF_w`ME`Bpv
zGOZOmT$MLkpm|+&ZRBOL0LEAFz3o@tXK5jBGIew(U&_OGpQn5Kkt*BzOVq=J7p*Ub
z1m0&mCXk^29y~+qI_Q~_-27L&yG^Z6^YJaaDA=r5g9LM(2!Yl&0|;<<D>D<-G-qeF
ziNi$ATtOmMI>lLMV@g;Fj6YI(He$mC$(J+|rm~PaGpw#UwO`gQ@G#z6p(l3p-?}R4
za(3I2B2#n(ey6tk{n31AXJGpV?u;a_twK<>MB?kjXWePT;Pw3{xkB!DfTk7ezQc2Z
z*_bp=eit+sIa~_eT0LUk#uu1zWCIKN`z#Df>z`1mYsax4Vf^}{g38#<)S(g(1i&=o
z_?}GpI0EQC9Q@TMe64sTolbwL`Zkg7sLai6i^7VKKRsvyHT}0_B3wJCCXVhC_aDLe
z@|nGjCL}L+J2tXr96lF0-b7ARZ{+6g_P)mxQCS9(^$f)Uoz4^zIB!vQO#NQ~(bgtX
zPdf`Jyo<xTPNJp1A5ELpDb8Ju;rCwYRcO0Z=SzaV`{mDRIJO6eZxuQX6E>%-Z8lh^
zyboJn!?mG5Q$f23)zMCP{^$blRFRuKHSCJ8qOIKj*=lk{J-fdHeZHi<m}1#-E%a_?
z-wAjZdMmA<P*Ix0rv-fPTHwAI(-)MZcgAB$I4HKr{4Qg!Yj~7e?ZY@u`#3ag77<MT
zn;V`|&`Yj0$#%dyou2f*p|xpBrBIAPuq7VdDK+SeUw+nbxrW`wW7=LCI=F@tCHZ=`
z&_T%jiLx^>+2^raQN~-@=;=`Y!dx6}aUfz9D+&pW+}cwVmU4nQ7&WPRilRU_@VIC+
zh_GyM>VcJH@8P)Vm+6;Ze%DAKM78MNSUb?}Kh2A1gj(8&rgx^6B?*K><9h{a$<(g;
z^l@nWn1XX<{ydre7Z!kSEmN;c7&eWve^=BQw)t%swTCy*v-v&hhor;aneB}&c2S=M
z5OMQx#ddPHLFJ9L0qS$(&aiM(oco9?_R+aJ+fnmDumoOpWLp6EQ^jZ5DL}}$tsplY
zqa02m_a1CIx}crE&2BXh6Q@MnbEKj8WM=KUCQ%r3Df;ep-)V2Wy2xtq?pK=U(w@Mv
zDXhjqB5_B6U|mOXKoF###CEFQ7`fFF+b;m6l*u3QrZKQIGpLd~z|eBk9MpfiXnQkW
zdi!6Uo#j&;Pqg=O3GNO-0s(?Mi+dnA0TOI++r{18-Q@?tg1ap4?hcDXfIx6v93Jjd
zb^n1o{c2{a&WrAvJ~H3_97^w?2ej3vtmdzQe<Ae$EtooNXVbh@Rr|Y!rfy;i&}HHf
zXr5Y_O%o7o&dW*u;z(m!jwccc<6nuqJ(X{+s5zNQG_6@Hry>2gVHHDYv_QK0l-cZR
zE}xSPr{OG|`-q}!Q+yP&h~~IDfCOl{z$Ql~Xk??7YP7T8l(Wo>;mEn-o~VO^xlIaB
ztcPxdeZi-i*&OYNg)i5bp`M$Cj8}iWUvbYfg`t0V0Q2w#V%T@9b4srDVXZ_Oz<vUL
zK3~t|Yi6Zk(X_w*z%R#xBUx-`A1dK-lk`Tth8SyALV4-Z3Z>Pypi#k<Q)h-xMa}q&
z8>`4O6g)?=v(FK!;%EE)=&SU^b~c;E*apYQUJ$e+cD2|7>mR+~zkY}+R`#Gx$db@S
z=q@RblrB#gJT%qWKN1fx2?v65TFY^1eiP-|IpzLfu`~iVKgVm(Q$5Hy!fZ2jwV7{A
zVC`!S<<M4(&Q$Z&T21{{Se*j-+H>~o%#)k6F;i^aH%l4C6dnmAKR*)H={=atthM+G
z4~psi&c|5KV;HVYPl|US!a2DHBb<o&8f8)|h1XJoS?R1OZmN&dNNe59X<sP4?5Hf}
z_ctRt04ro7_tTYx9?fOdHiY}8Oyq=$o@`WSex#62b;H9$EERMmO0XHGuD1In%Ot>A
zg(q9B!s6ODDU3k&qK<6!ay1p*$J?;88nO}%Phsn++;Yt9b4P8(a}86x#`bKqdL&Sn
zU@AT$k&hwAIIb}{BT;D@HuI(35hX?b)X0@CG2co4t35^p$p8JZ@DRY6gtKZpxpp`s
z&z;XnWP^P&?x4RLWAl{PxzsT^&;`!#%F;xl<nSD*S5OVT+SfeIrHlo#cVwz3B(DOH
zd0myS+m#B7oW(uzm8q*&u>~7!;K%Qa6bO3GELNTgi43i3xb5|J5x!^n(Ie%%3g=Us
zUWxhf^_gjRq2X={oG?SBmr;V`rNTaC+dAB)x-L10H;$@+Tqo31XRx4~NC;51hIBHF
z=NUuLCO@XHSf4r0G|uY9VAh9j^1}8qD8|<Oj}X^bW<y`z$^+R=gKOn<FYJRZ90`g8
zq3I!hvRdCtN>bWDtVnL8sWuFW?9AqaxcHG7fgCx6c5dSBX}bSphdj}*2`0a(&sO4V
zok&z|Z#0Qh@r-VAdZDd4cQ&&jLgE#XPsYKQkOKlU?=~QgEF^!LKpBC;Ahs&jTjM^c
z;h_T0=Z8>#J9Y86CG4C)gKRtvvB&-Ivr3?33|vK+qkt|K_l>o*gV8!XbmPMc2}Ab8
znvK)Cd^l#w?1GE9w4WJ1ijFhy7oMz3TehN%L?>Yu<+RP}qQA;;+2w?@n#$=OOQgmz
zaBF9@5e7Ml@N01&b5Vgc$-c>EdN3CpA#CYifWL4jO06vw>Ob7qRGP+pDq^%*B9!5B
zyw%4uo|o4f#(nPKbNfhX^;UTyqcfJP!y+-W0kOM;qKO4TOe2h`{&O9tyf;Eh%K<s8
zJEt*ZBoT{f>d#T*$^$`>=ha`9Te}`|Cc;qCvRKT)jh{_@Gq%uUa3TG^08q&Mw&lrO
zps!~&MYeo`C>K)kM#%2PZ+MES7Chl)UP0{|IZ_h!!|{)2aX6gAkDvOvdK{r~IaM%P
zXwB4{NGM+Y?+03mFtM^Am4!t(n9TaW<zzF{x3ho`ROW}fT8BNk#jM;z3=fG9Ern!H
zso3h|&KT8b`BZ!!<R1iAHh7bbNu2La)wBvxGuIH-ngmDwR&1TXqXt0F)hR?)A8Ne(
zQV48=m-piO4fIFD4O$=8tj$Hn5{_ApD2?Z57<FbZ7qFPixx$dO$EyOV5W%2|6RBF7
z%QzTR)<jcCH?}s?WH*nDiAit~O*mPpd+!0?E5lVac`VVlKJ|()9&d(GFDuk!LLq+k
ztYRA06Bh*O_oe`wFAvD`_PHbS^S`#oQ<foyOnBZp#L}!(KWVe)a`Z&Vj+I^mxJ+_x
zX;D_wJlB{9J8#zzp^h_Q^Q~e?jEHW}cP&h{?<kw%)!WnCCPp%BkHo;Z-QRnxrzFZ?
zbiUL56?@Y><>~?_bA&d}w^Aql72g0Vbg;$7?gnRjPCnwv=Q^_FL$lFa61E<mB9?^1
zu|@3$>lN2^a$DE(64tfka9K&ctmfb)e0kYB1UH7p)H}D1{CFJ`d*JZgIk1d+ATDQN
z3Ui3KMa68*2BIWBd%bvO%!0P&*VJO!9vcipelM|xQ4v{SvA_`BczG4jAI`cb^|k86
zYwPSwMQY8csIBl(l#U<TxTZ&i;hsxz<!3%)<4E+|H#CJXy-@|5=D)D<Cvk;C31m*u
zPpqtr{GJZlUI#0YW8~o|81WCtya~Rzc2Ds*8FmVYw_OTrH;lH2@Sn%GLm9)*l2ob5
z5y2Ww6do;8q-^IHCkj@{`RBu1!xUSyTElSJnF<!@QJhg$4C>2U?R2q=ASqZS>(IZ+
z?>80wjQLd)^gNjK-iQE{RVU#&@TObC%V%KW&1CWF4uLEnaLM0^{fM+;B?RPtI-GBH
z)r4U-oJmd04G+#}Of5&A%{!C6ImJ2oYIPhYX7j6*t1qypxN!78`Yv3f;d9c;b5RI%
zT7NEcDuL?L(UHP0eOrn&b-5B8G|9E3c||`|R#&9j{u#z*@lFE1$u+S*ILQ1gwQl7G
zJ#GCrIGB-B9D~0tECm&W{v`Pt$}G&wT}owR`liul{Fl%8<$f3r00*|{;a}91pD7i>
zM&So%Hks^-Bik+@5@Q}AQU<u9J3V4DawK8#Fg7z@d#fV}G>W2tfB<MS4Kj$M@#_Pp
zqcZ4sv6Ht4dNB<)NLavjN2?v%R$~QyK2&Mcp>*Yfqpm0w81TyDjxV9=UG9zAh~8uB
z!&wgRDGrH8<ew~-)c%pap1k>CsIb~Aeya-(dcVGkoGZ^*i4N7j*`4^@{<FwpO&-<E
z?hRk`pEnNWZC1=6P4Tbi?celcKG!Gm2-)O|LHuYKAYXhns)h{q)x8x))#c26Ze0or
ziQB6p-EeWVg_DWZs}CaC<gHybSZj+6VGG59e?r*9rdRi+oxaq9tq@xYg5kn^h8JSh
zeM$V?FtWdMlilwX$V<*TDDa=1WJw}6+)_ncCR#PK+znN$X)8kuhjv=_kIk?fi6Di6
z$}`&dpDV#5!hp%OpZn$={eCocmM5ZC;&^pwI_h^b0jt;@xaaL_^A}lCoWiE5_sQ!c
zOQF2EoPKCw37x$Iu*;s}v_Hj1qj&{2a&vUb*O1D#D5pcLZw&m;L;C<@<-bJWQ~IF<
z$PBePQX+WCmv9#mr4|;dz%xr;{yG2a(+5L%5d0}=#ikm;c*o06U_%*<vr)9xqDR|+
zxSVM?Nm^OneP)?)R$TE02LIy)OaY0BEl0zWu!hvY^9S=s?l7OKmu%eQ8e)MYp2%>E
z40Q{oCK5!oD*L^nIqjD1nM6}ya_$>7N8*#cs1@X~tGaJ6mUoWF)&qnF>%uQ1(nGXX
zve+2`Z--WQ*R@y)B+{SChjwb*5;^kgZ_qvX%|>t5#AbSOjRTQC>djPQ&(roJ%6}zr
z{5<V(MQpuo9brU74PF~>uu5^@?4j+ek0QT~a(}t`DdzsO>YG`1I=ge$04R@4_4(qf
zP|Sacxawu?4}*t#)KiO@>1$(9m<<y4@u6sDdqL~m?23QL0*sToig(_lt6tGk0lQN(
z10zlH>8hr??9GIkn*v-(Xco_X!xjwtegFMM9*j0I*7j4>Cm{gK=+6Ql@~BX4T1NF6
zE8EO8oV_JhUfGw7i}l)|nh}<Hn2`YzxRlPnr^_VF6A-Vj(ALoxTrEQS_IE#w>v{y~
za~yGY`Ydr>oiQT8tTQlnGSqRS7Kmvi3(s*H_u?J_9xj?BmiZ-&bF46eJYyjY8|l~(
zG}sX^4K?1<MFDFsCVD=RH8*-UL}8K{mv;i&Tr2T4j5BpPo)kg3({UDgahRIIS*wQm
zkvnZAp_t}=>wuPYz3;hLU?C5)r#qXI_H%1tDsYCV5p5z-sHff?;UPpUmzP}RVN22!
zNMS6`L`K&KH-qjPY}^j|;8@pWcG*BEop$bn+TYhQRzFD3$`}}X2rT2aH>K{ax{$S?
zh~dX?j1VPRmF8o++!`@m4rTzoa1Xzp?Yp~HM23;cYMHGr#hD@pH&%c!WgX?Z?8%8F
zT&h>8Rz(U-$ZPSkTPa7iV!ca*9=t9&i2htavrxf>NtNyDRsPP{y~N2Bi3vahapV8`
zwA$(D>2hM`>h_VUx?mq|7jIX-`G(k*6@JBzG%8%Ia$hP$2X!Ewr05N8xaBwEb9n8A
ziRsJ<%|I#?L|aa}qH?+vNG8z;YXA0m&DtRGB4}No5&W#6`tnKomw%o~b2~jEH7^2x
z5bAf@t<8Wj+{5F#-Tc$iJlG(mU<5BvStQhuOJ^vmY;Y5bklQ@n{!$ZGEw^>!C&!lp
zzn0AC6d8#D9l`4dY*O+5{aEi&eL|~TW&F+iOCwc}oW-})f|}xnRI2|rWYasv%)j+?
zB;Zy4aGK;1U;&)GOo`p@qc+PDIZG5_r{(Yb<Q5Jf8p$sqFOrvBLj<8ww<es4mHO3+
zVrg+W+|D@hedeS7lXr^3&z)pD8N-o;IdS9#Q@Q~L%Uy1V@pTTc2pZM!PNC5Nw(P{V
zmdAwIJ&$pZ;*6_Ph^eH;=L^}rqx{x+5yP0Ul^a#vq6~rbnq093*k70`rIQUzBZFM+
zyif;8t-J%V+Co`elgDT74;1vIi9-Wg$mKjd1z!4aY!$NyYix$&FikqzU#+^W>+hn~
ziF#aDjxxrrmix#i<kBnjE?*i|qyR`Dv+-dwEWJL-mM*de)btY-g?N&HVAvT`=H@r)
z|9RchmK{`67KB-%e7(>pAm=UP1`NSxjO)hg&-$~s5B|)P@*8pkJm}Yw)~fUm$7yt%
zpx<8DNi>Zw;q-u)Ewn~hCFntccVUr{X5=OPP5Fz!jh~!t{yIVQ0O^R?N)=A;v#7u$
z-H*G{Xgs=j*DQ^qr^DNDce8DZxVCGVl=tLfnfqS7yzoAa+M1)m>mLXFnrDdWiSeaX
z%62Y=_G!>eE(*VsG%CCi3L0^<4Z-n5UYHwk;Mp)|BU51PH|xA8RlETJK>ATFeX`U=
zG)bi|tDHzb_T%w}ZJh7)i0Gu*D$fgktuNxc7)c{W;`WxTR&)_lVh!V@{4;l5Fk2=8
z+!4oATo@7h1mN*wG&z2f<A)Rl(jW*4G)GdvnpXUhSSdK-frsU?*5Z~FEM#04+LmIR
zBgwb%T6klL4p;kqnWq??{443Inrq3hkz1GH`@;G5Qp-Hk>F>6gSXx@j_m36S@QLgS
zm>)KLEw&VI?Ccq;C-&PAS~(SeHjUVDaHCe+r|ZSeMz>Xr+mMoC#IIGcy*%?}qenWK
zDMk3BT2VTw``_2S6J7bsN;sUxmmbKRG~3{VS)l~mJ1J7!bYL<(-^NzRx>?NXDX{9(
z-)09l2p>o^E|T>ncGnNSl9{AZ?jpXHOYt9-Y^O19N*8X)KY}}HKs#F>A9ODSh6$Vx
zEHk$2j~-j|OQSFpq<=YG6p|D}guW2AUk-+?NhdYnE~V%j#eQrc=$3s@U%~@^l~KFN
z?>ki82s`~lZQ29TWul>)$%Q{rlBvOZBl?l?Kw*2xqj_0UQ5c*J5Bh@ftFP+7j5L`P
z<#^3R2XKRF$1OJVb0eeE7jsa_vBgSfo4hq9PZSmUNG%sdcZ?z**xsd3T5mfrN4n}9
z5Nj<FJ$6}m7oDPxTzSB%$&G07+Yz1D_e4;}Z@j&<)7NPjrM{L~o}yOtzq&lEP_eG6
z+NR%3I&7pB(M?Q#faw_l#W*;McS~5``zvV6>ihKez;L$ONxT2K=pwzCI>~NkueXr<
ze|xlA`#?Ie^oRU6=?qHk1{DLHNmZaw7LGtyC0qz3b-!c2$Xslx2K^MbmckkRz$%)Y
z)%4+zWL2*>OR<lIGrH`o>xagzMYovmcx6}<KoHY+DiztHAc6^Nl@>@q_f3cO+b0>b
zc)TFmmC=CCUQ}y^cT3hGCO%2`A4_EUC&bRIh?Iw8U}-Ts!GJivMQS^Ecj5!A+lv`S
zD6U_KZThPXQ<SInOs)y5(73f+2mcf|GM^2u(w?cw_zWpIl_N?S6%xNNMe97#G?K|&
z;+o(Ht^Ga9a#qQx;NnTbbG5P=urRRdrz<}#fFZ)1Pv6!2ALorVZgReoKXPM9K<tUD
zG?fMTkJi;1aNYv@tI}xW{MYwSm|}F~xBl!A)NEraQ*_O;{{-HaMS&+#g6+GRu(el>
zga>Df+{mt?1mxOFoS33^II|uX<rEEANH)`qVen^%lw9~)hiBC6aP1<P@WUu?t<Z|b
zCr1j~p%jW$nqXJY4f(teD8^Dt8g0Weo!`;Em2i2{94&Aol9N3`sz2sYA%jvXhWrch
zs^UvgbsCKi^@I!eM#;pSg@%Chj3PxkEa=eF(J5?g3p)5?HgUy<%G?Rs21c>T8zx^C
z6LT7Ax30N>Q)4OZw<&&O50fibos6eWc@}!8459SMZI~FUXaKBx2%aKxkdg7wg;J<B
z=$?zGy+12z$R^|sH5=Am>aokYbfJFJd~079V@Db%`zb0q<*oYF$dQYgSd+B?Pou{*
z&h7VF;Kh;B%bRTSBro$b#oF@5k33Cbq$Mrk`?pSWN>_qgm3-#}gSUU(A<e50asP+-
zN19`!!2|Shl4@Oc&2@W!*5xGd!_)HPlm8>5m^5a<_?A3|rw^jTtU+6q<`-v=j>^}`
z13B2aA8kir^UFj3uVW+L?jEBX<U$vb`caxk868jwPUpT+R~#k6WdG%7{-HR}%1Miq
z`$l+YGni2`B#N^-qPMW#R<-^^;XQF$)~JVwl+N4PucN1}f#@Y@hJRHzTa^E6vqxF%
z`6X-@2*}BMkH_Mc{J^$dZmH<Uq{^kN6iKX8fUrqPH+Uj<Q$$1;5uvyDEOY{raLGRO
z6x4*uGQwyD;m<^`Kx;@C;1#*+i=vTCx(sJ~9JD~-df<pfv;^60k|{TFOkOhl1BOLW
z>N!?yCGGR%m$Lv(%}7mYdM^^OP^6hK$oA`0-MvAq&PJ4t`(Mj0Tm(w5#nFs({xa4V
zwpCre;W-cG{@QNnl$-=gN4FG7f0n3UnP_9Iiwv^1>i1$=!jo@{O|TWx%dJiAkJwH@
z;Gg^>^xUKzhCM#V>1xw;B}!Y6B1n}}qbEjUx3bR~5Z+Jb&6#4#*H^tEoslxRwL0I@
zh0ud|X(J7?#SiBG6~JHVeuM=0yjs8SOL*<DA5ixR!Ut59&UPhr+H9j_d<u544D+lV
ztJa-<o`6uR|0FnC)$&egx${6z&woEuuKB7Bg#=KA@q>CFZw}bx+_`f8^7ljFPnfNe
zAAE5`>kymjs)E4uL`|nsTHbW{|J7BYySlK~L9(GzhNK)SQMwHm*+6fk6Wm5)EPj4o
z$Ck<V3@c$DJjn9&if#|KRJMiq?bn3xqeuOPa?a*{k~Mf69%ppr()dSFmz_dR*~RRQ
z`<(ULC_)$^qRgS9?;1Qvypbza5KW8@GkVbE(PC{!348yhZL)&~9)NV32<Th5GE|9i
zuhz&=DstjUb`ceZo+nMyjL0CwE)K*P`g$5|3<(Co=?8<EY7uABibjkmciHV<x*ekH
zWM_WT;G@1jojRSyX<vI9O||FLzvCF+8q!CQC+_4({4^hil}!JK_Nw*VBm~laZWIj0
zsePG&WTGKv#gYS}T+|x-bd1-~K`5uzHW6{9Y$QM!wj#xjdVQb%+*5N#uMLvGUpD^S
zH#{lt+UJ(hTeo+&nE%q243O&Vi3^9ZeO#7_%xvi2@A4o2IgJPkg7A?$Yh@uLbAC#e
zFr%<tH*0y-tHayZ(EG@^@IpgJcp<1%t1``e&NYI53=YUopYMSMg@(cd<)tZJ{>dOp
zR6enww04>aFJhdK&E%R1=)3UQD#CHcvnu!8`Neq6A++^wOiz@-1U6qVIWfj4IO&jH
zkfGQJYdEKrVpKnQR89pEfbByH1+DwENV!YyD5wEcIN+EcQYXC4(Jo2($WG2*&z}OC
zGWx+{qB4oY9<?$y98a{-?yVTo-U!fAhP0T8q&0#bm?VV_LATYhydVlp{D{Yj-a`7v
zEsXce)yb`?B=9MJGX51WH%4%0o1(=kwv-A1*!&B1AroWOap9(sF>DbnDnf2*7K^uQ
zk}B`7!A2y9PuBe5`fi;lI*6u`gP}1nfi)pksI~2Nvzv?+{48qSh61t7aAyoS4TXqM
zDc+U5BNvf%BhIY1e~k2TsW>piIRE8mT&VJCln1w03uwz{yoI&qgqPDlj91j?gvQzb
zsiT5_FGGYoSx>c1eN5k2Fb0f&4w&eHG1PX>P?yRL2Asda-)wG#VF_0m&+13&2bfX_
z-PZr3Zuc1CWr`6T<JfVEudP|#|L-rEgELlHDHWZNWHgrPo*Q9pNuF}c5NAX|+5QPR
zhomB9ZDJU|GFlc?rVFnpH;iAbrG5~l&P6fGHbuecC?#L9qQsNJn4L_uH2cS)YCi<!
zg`$E!VU+<<*=RdxN^}-BHdArh5#Q^^P#ZkMsNXEx55WzqM*>kGT)-z<O<4oVQUkvT
zfPpOilUtvGsuC2>y<bBF>JJm1wyb=VsA6@LxkfKVMN21t{%S>X@=$r@^sZyyu3E%1
z;WyT6Q_{wy){PKY*+YrnF{077D-1jh=bw!nM1}r~CsBTh;`ZAY?InlHI!bk|XC97>
zl0yI>fy0iE<|1b4zbO`xD+$v&sK=wTxD<5W_38hy666#IesMBGGcuOjnir=(QgH42
zcxZ!U=gD@3KBQg(uQdq=L0!dM<{fCmX;}EC_O<-SH2o$1_#QJz9&rr=9B6xjpRYv|
z0e`GfWAYC+;B0S;J_$a95;DakYUj$fVS0}U$#eeqIM9?b2@je^fMBeCSk_es&d38e
zF2-&5(60p|PRV-Qr1kzhYRszpADR65J|{Y2f9p4>ET6MnHJbu!U6Roj{Kha+K<<z4
zxi07;i#3(85Pq83LoH1aM$Y979lV#G+{HRY`*UVz0lx+h))0}I8roU-uj^>`bB-DH
zAVJ;l))3iX6LH;ACj6tC2d$^?n)1-EmxpUgP{RXdynhc}$DIUg4S`@uiVsa$c=qG1
zALNl(10~W~_z0l<jp)l7{_I1296{AY>P7COdm(Mjm>HVY7i4KC7N`v4uXx>ZG<%bK
z^qr)p-te`qR>v(}H=6A97(f2?!9YNYnI%Uo0|pV+wuB=GZi_#UrY~>drYrXC;Ti(l
zkw)lu;^kN(-}k17cqNOvegeXAo-hX&aRZ_Qs`9|wj~c1%v?0cN_9@6c)bICMX%v7j
z#9qdLjXQsffdAA5*RhO6JNS`DZ7Stjl7;&mz_|}vqDy)#?&UXeE96i^G!QnJU^#t!
z{<yzc(sjm9@n8NQ66GGQk|pCwVWx@<YnQb=?k^aEQ(_sdW>bm{1>FdZMJfVhE5U%(
z?)BdlHCYC$<x%nn($K%T#nFMv&_|MEhWOzrw7~H60hkmpdEs^1zqWF35!~2wkbsg%
za0K#3b>_A!_d&P+`21nAa1jf@!Uk=cpqJRch!g1H4>ZH7HNJfbrR?S;*4jTq3rvkO
z_U5YPSQLLQehprE(F+?3&LT+O3y$Z`uFGiW!U?~#A95}(N15qz*TnfwThdModb5q$
zElPGeq&2_ZQjV4qZ$%l&T<nnObE_kTmb$z=`BUHyj^!F6^2QI8LL1vNX-U}4*fYh^
z@4Am?-bD%wYJ|5;@meqWxeekFZHiUQc+6e`y%Q^D{tjfwu|3JYd`nn*YG**Dfb4xi
z<pE<54@5c`$RpDm>)A*~t<AuAQF*>lw1=4UY^(j{9_jlY;CBm(X%@%gz(#al>{667
zBJU`52WmTa1Pbd4b-dq?F^Ky`HE)HY^LmWyD8ct$F?`r^t$e6TD${(Oa<DX}FL^cI
z3$jMG+hzba;r^fU?+U6(-K47-Y63pEyxfiy-06Fajjje86EdQ`@v{qF3)%5r^r-RI
zwctPG9YU0=CQ9#MY?Y5gjR|qK1OCj7qhxx@%`K6Z1Q%0<n$ErzLK#?0>d{R;*qy@$
zC*f$RJ#yxM>O}kaG-xov@E{gJ>NVo(L_rR|cnlF54`V4`sbJth?FJ{+U;Nmu8Cg0B
zjNLyB&p4H@o3hqDa_WzE<*!MBw*MrG<mYs&P|uyk;V2&DO(<rEj?F6Vd}-cQAwk#0
z)hAI5eHpC-`d#_a_@Cbih_TS-p~Xu0`l;tJB>vCHHcW&%eNF9e;<&3QwUYeBTVK{z
zB98O)RL)7y4Uta&ZRldM)D?fYM)^A=(#Z>n6A0s*k1F{WtDhfz9SN{}bpJpJPcaZ)
zcRI|MM2!NbfM#ekb^uSbFLuZ;4H3W*E3Fln>_Xu4ylkavgw>;k@9j%}od;US7OH%a
z2%<2cz%ZIMUe)8Y<^RG_CEs~!gfx9_aLWo02v+0hu2YcvH;{LD`&HZDLGKRm|7rnd
z;>557{$6~vJBlwkOyLskL8-C>nTC)J@ZnP<yoWF9EjVk&`t&=qGjWuFAnsxF3?-=g
z=qyv<wR1R?=aFP%4?RQYO$$@!9U=u%d>vJwZ4m%&RSa0@(g)cXHI=t3@vCYctD6~e
zdX?2n`y90b!v>jByP-{s4slCG_6nuSZ=+CpALpClLQ4f;i}OQCjx84J4f#adJ4rh5
zKzhKMleM_Fq6q#fbey42?toScd0xW{iqP);AYO>ISQ}S*(@U}Hwhs<Z((I9&L_N0W
zt!7T8tV8u|zBea@j%W{apT#Z`y=ue<L;R&SRCAuNc+%I8{IfPge-_8f^4mTMP$|m6
zQJj1Pxr543{7h;L^50#)8gt#du9HFd!|MGK0>M$8NZO^Etp3*caG8-{!;oQPxv^ZK
zYN`+PNe~!J34GEi{Zlyv#ard-L0PoAfHluvWjVj#TD=F(JJp1yRzPJ@7;>io>1C`@
zn{;0DfFYFU4;4|86_Cf%z@LJnmQ3C~_}#Zk%Dh%Qiu=W6WX$6_z%|yj|Ck7AlS?^(
z9RA8^v3_vFd%m$y{YEeNNKu-#^-aZ>Oz77hEjhGNt(J|`fPWsdk)ZLtp3k2|<1gM&
zv0_9Ezg5Vz@S^6)J45&i6QaDM2H!?gOBM`P75;hN`|qFfm)bxaZ0xqo@KaFyr>?pl
zM*Vp&w0ixiBd}ZADk;DYNZ1N&?wxSJi;;7qX|}+;c04bQn9*e(^U8Nnrr8I9EbF{!
zhW)P+dl)tLwKaNRl^jkwRvejX@qgl;fI{+fpWpD`wDsdCRfz~uL5RyTZ~7p6>{A7x
z-p2Sh&^apjv)h{g38gOH>qn7-w;tBl+}C-xeYtqf4&es_!dm&3V#Ub;*w9QIBL6ob
zNW#!hWx(ed8E?sK#sy1jIe79@owLIFQ(c$Jt3B+iqKFfbdH!PC>;zIEK}ZfFTG**o
za4`I-$#2vjgtb&HjMcQd<xKP6)=^i1Pi^S8A9ui3D~}0IKi?}#=)z`BOYLYt0-w$}
z;s!=>7prD<Y79fjAlzE8?U-{%WPUTlyp#r*XE+A^7AuC%FzYFUpsDWD#@@Zn--tD(
znM;o1K&k;gJQ372@<xiLABRal<>3GUwk(b)5ZAKKNjd+QXSOY-)|<noDQCeL<^=4D
zWkmaxiq4zgWllQXzfjy149EXi0=)xZrW%GLCsf_@J#lKqyLDBP-pK~x%{gi$&%Ipr
z{d6Hd<c(Z(YrT(mXjw@7d$~F|vlXQAVB}L9S$7file+OLm8uSTcf#+0{&2IGF7@|S
z<j-fXkm?2%^0p4{H-3oSy=Rn#<h}qlRN(={%UTCkAXY2e;iU<)Rl?d`2jx&XQ}2Y;
z(;^|`0)FTQC>z~XN5};ZS6qUq0$Cp~tQM!^>jF~k!FJX?^4-m6RCV;zLGojy+FeIw
ztzi?r)4#Ko|9u-gv(QwEkF4G&siI-6S?N}l0BlJv8mZhAGxzu8M%%tq0Hb;#0WkV0
zZoYD7Qs}jVawY+UUWCNic!`ZTiS4LvHsmpaRZb$+7;oZN(9DKKQ?UBc59**ElxmWd
z12PDc?MS`>+q`aY&k*eM^|LKmRfbrc!l!(y1DhRV1j;eDUw7Mf{;n_aub&KEw4X!j
z|CIGS1jB=Bc(8Yb{TTZzS!+e9Q~A<NM)Q4*3hKfFunfcz@)<jynOS}ZxKW%wi?E;v
ztlCQDq{~vT>V8xcqxfRN)IH&-rzXSO)qN|k)vW_}^!LePc{?j(y9ahSf{1eZiQE<r
zDsd3C|4jv?$GJCS*kvFhWO{D-amrg*lN1AoJEgZXj4Svs10eSu=TOM-M)W=~*1RKn
zzo~M9bWQa-Mbm*gk8H=F`0nj7Q28C;Gff{}652kO#biEX=;MZ!v1yac{Q7gsPm@Sr
zt9eeg9ZwkNr)IMgy{Vf-gz0qiig_+iAPMxF>Sn+b?7ra-sV6327JhXR6uXE4a_L}o
zRK`LEWmju-ZdmeI4WF8E+Iaq~=K7#dlVEwW{=5CGfUP-7>dXoqK_FaYEoiNzppt3-
zR_H#Ulc>5Zj2O~N8*&?GB2F*yB&KP43lCUw5}5S#fRI7$l#>1|I+*cFwj1n`1hmy0
zYf>(dOcIduOyQf#?~kG_E|R{GG|H|zkR2F}GtlfUBc8B1D*yMcEi@x-C2Xuk*F--g
zsjts*_lZS?uI*%%qLA=9I}bxBhdNwePFjl*tOTw{iuF16Vmjwq-)Vxcu4In*m0A?P
zMs+|OR1ZD=05?v~K}M?QenUH;eXOm?ueei1uu|+xj1`;BuW8=Xw!lud>d2#a#VGmn
z$!8VrbhmhNVT{3SXD}e2eZtE`s75v%5vm;%|9FVR?;zT5DwN9?0;Vmee*B2POX@My
z_s?5SyIRP4d-c$Bw*g~tp>AXB`jvE)q3u#~>bCh^+mp7zqvdm$!+E@sORvtRvH#nA
z27TY{${hzpdikYpJcN=I^`9$zwgb2{y_fQDXGl$6EtmKwIS-VFw}+RS%>5>7lO;N#
zPj2?qC8F{5YA5h(Iev0-D#K}8GrJSV^P^<m!N#qe$C6r4(QlkR)n4Biv2-35d|eEb
z58z<AEhy-?6_jhKVR*#}m@c~+tC4B$@PJK>#lgr(PP5I1&z<{BSCjE?+RNQ-7P5HP
zk{ZN3P3$DbbCq`|{eE!q@pN@NArpg}h@FHKOse0a;sZN*s>?DNAT$~avrD;+$4c`D
z1{{aj?RJ6zovhW7CnrN#k*5w&HUO>M+eg!$PB$rotXn81CLVE%9R;&!AYGh?Y{mYh
zbSsANc5f>%ah?Z(Us?Y2cevFTT&SN@%yu5u30XPDzaX#N9-miIX!~wf=>haCg%PJJ
zsB45csiD`-<A&4v;(caW35|@G=J`iwTaH>zU8;pQyX6zsSm8|h&u2DJV3IA2ZrmVo
zG<rKrog?86o@sg@O^_{JwsHGG30WcuUbD^|g+53g7U|ly7ehr4^^AM;H(uhg0N&iu
zz^L6A;sNG3F@oX-7*NzdSw924GU_W8fB;m9M(GB<$i_fX%yVq3jV@3$v9{PZf5Bp*
z$Z*<3%(Qp^c3>gujq*2OH8&s4C8mp+Ek;-0(1o!!zR$InqjcUuyyCcfyWx6-^HJO2
zES$KOcB7e-SEiZPD_hS%0><)tK;Z)ilfCo|)Uo=XUe@gp52sum%^m#bOZaT8p<ztE
z0AJkdI@$uqrng%fcrv*e>goB+cjpX{eZ^Kbh^)aM%K4R7g6@-$n<G7=<oKk>;28X4
zdgVaw@73mPP>-9G9xDHiS3dpqHwZdN;R{0huBzP#evSM+*Z&xHL^wMn$15Iz-SX|l
zh`y?Q=f0N9xP9i~w79`}LNaT6Jk9S)Ahtllf)4L^<%MtMo86&F$RR0Iq6yc6-9Kg8
zTyI73E1fshaCacmNP4dawuxy9{O+qH`7hA1<k=w~nbL)k1z9tgFon<=4tDI|+uC!h
z$26vn+|yf{i2}m)%b+$<=bx1})c=SwTou1CmhxVh-W<S;;W<Uc42g76#Ooi@{@Gm<
zz{aF(*hd&sGE^{GFd*{52M&heNW~S7W|duy*ckY4K~N{_*2Dt%h23{SJ_4k<=<uB|
zH(gOo5!g+*%ur7+5;=2OQKJn{(E>?tZ#nAFB*o6j8e6eT@+*i8`N83ZVp0B#b45)N
zRWWp@V-3C0gL?w%yU&fPCPgg$B3#KBA>(|P%IVYaY^J7UbsyTSySa&yTR1i<(B8^s
z+=^{s#!RtK`>vx`9i4!&Pl6q>=B(qSYb0st>p7swN>-}~P)9w+L*%OV?zONsw8Yp*
z%5S41P6jpt{=<`Azg$q{>WAMj%8G9={l7HP&?3SvTTsk(<qP5;&K!pfP7DUT7_dRO
zc78O0<o4G`#zYYYOR+c$njaY24;9`w1kh8Og{W_90aJx4#k&0C@wKQKIyB@Mi@0a%
zK+s|$^_S)z>enMdW;RUd@GkkM!sHoL{~!7~&WYaYrtUmzpSd$`=^95K^j|rd>Fyes
z@(mJN&v9z{_#7#15oOM)pCk>*HRREt9KC#aIi{VYFZH&hH6yrpH8TY#=&p84&R=v~
z4z4n#xp1S^<$CD;b4Lfh9YpUnS<$a@1WpbZ&|?aJQHGBt>N&$+mgdzm|4sfjM5!Ue
zY#h_)A48Rz`@Zj5qsW<p4z*)3r4ym;kk{hMZvR;70zA~z5H0f~$pZF|6g{u!0-;r<
zS`_jHj;xw#rV!l&bHo+i{Has%^~&(ks}V)}Z?Ae5@^oxxHpk+9g@S!^#OWJXx)=JL
zQj>8Ld8Kc^U$2hS=VM!a$mwNHYT1uMyHXxH6&eh)0wX~VWMJ(2WAWD@{f~zJT&VOv
zg3GqVm%AS}<pt7c!m7(EEjy>JVm{vCWvl!s^nYILo{uAda&ku=S%<A~{r&jlU1=>m
z$cM3F{t`-X>d;E@usw<1g6ermi2;YH?uS<)V<7k`BQTN@Wd-Z$->-xB6V*w<Y2!T>
zxGOF>&Km=QbuPbPBNtpMw+WWGdkO7VQPk9MD_g4Uy)?^op`WoksOpWWXk))mnw4gG
z*!^KW90fK2h4YyGRE?MGBPSIeFv)t4NUYDp7tfI}doSvKzqZC+lGA!k2Z2DIfjCZa
zxl}9cOjaqHNpJ+NMXiYAcZP<DpXvI%h8ck-hyMy=#kpG9VuPI}GrWyCHgp+zqO_YC
zogXXlOitAV*<Tl>WIoeX{@%vHUpA?8u5@f=i&;!DStI^8#GQ%rti-rmgeHj%SJ!)Y
z^=YYZY>nhaQ8t9cNI30W7!6Ts=v5}p!yARD**O1^Ee`Z8Th8Ox`PUoN5kU37LeT`+
zA2epVHeo`c&Q}j1sl15K;_O)*@dUVui)+zqG;~{*gf&K@xWSip4y<%tyAx2-)5CoO
ztmp48(Jm+H#4DeC(U8u*M+*KbP1lq2qN3SG_Q6pu1w3eet445RTw}kmaN+$3m=ezh
z$bjXC4kno8Ds%e#fQ_mPEY0chH1c{Oz^W{DtOMzF%j0iZ2&mW+R;pjc@6C>d&k|sv
z?p80OJlF+57Z4REFw<gx-mt^(ODs(>?+GJ5gr-N^{AA5BF3a>iC*Z(0*_bm{kq}Co
zSH(j%Fe86qDjoaxwTFDJ_J-oL$t8|uCm6#o(Scq4UCx)6QQBi+MewLF_nvaH{oEiO
z`64xjq;Y;<ercZ+t5Fx?fFK1(A|CNN3Z}3xA{hD7`N0{k9Ab&67W0=WHPvC$yDN10
zmkx{R6WRIS3-3AJ9D$S~H6z6Inxg1$&RpCmMG(_d@lQk5aJ_C3Ux2@?8pS)jYh3sV
zx>fU!on@I0U6%p}a>>OinYMZMB`_D8uW?<6!xRe*;8g~RTV{l`gJy^DF1@d)jcBh1
zT+@$8xX$jGU9u1Z+`zeJAq^II#0T2kId3XlaG{SDD!VgeH0s`kr!pEckeMbbn!Qk@
zNFBDA5q?=mr+_#%sRMR3jajb$2HrA+K$d=}$VomZIk6c8#`1N5r(eN`k*s093B>l!
z_zcFZ-}V18r%eV15vK&T>ORTfQ(78z4yqN~kT!i{TcN8z_U19?g4p^!^SpgLKN>K0
zlcD|3bc$Ue?qEUgQ$0blkcf|^qrKK#@ydniMC4iai3|UvfBXcCuwAkjh5mCih@koo
zc$2`Sx98<oCV)F}?o<FUQ)wXRcx|PKL+lo1+t|BZ`1AoEb;%)sBh9~t!%5j!b9Er_
z|2pDF1N)!{Hs7h9_HLV^{0~(0J(i=`^1a^>hqIid?f;&1MB}78_&@P}-~D0F*IasR
zMN!f!G!bLe1S{*{;i)aqaeqkw9`6&!FRvB9{zPBxl^eCs_hktBT9y9&*pMD1yLM97
z2F4mVYCA14`QEhk^Zl=)9_12^^P3%}S7XbQZ=mt4+J-yS&UV+Z=VR-2O@qLnNspG|
zPmW@Cu1jhCT^@-$-U$7SJFVe9vf{w>h85@b7dM&3G$-qZs9d_j*j9rL{<)Qs%;Zo3
zQ1(<0RUs67onsgO-&3-E#J0s1eEyGx;tL)$&y!#QRAg`T<paZTfBj~XmB#4%*?u>9
z&HuQpyz$n~@}AoS(-#3%92>HY8d!Flr~CT1(OBMRC8C2#4nrQ<(I?~kbtHejyq+!>
z-1*ifo%d(RYsh~ws~tYTMsTB<HPbX2r0%k2`p<6));nfO!&ahXiO`nig5$v(PFfHJ
zMNBqW#18-N9dpB;Zh^`yMYOUiM{O~c-LZj8JxD8MHhpgS%kaPw@FR8=>)?0`sd?Uv
z*xMxm5Ghi@KPfiG$!bnR)U~lNXbzNbpC{o-xd?Ngq;@NfwzXZD%xtjqKI<oTEQkI5
z4?1>i>D*_Sxg+*%g}^0B&l}eJWF`D$bw0D@j;9(md+ynQgK{^z_~*h<FyDl_J%$ig
zAV|9wgdz%QpP)K1N8)6uB~d9&@Ez(&`V(g_UixOVznb+9tqT@cumBiVp$Ug9J|#@q
zc5>7^Dg^sh%`ssc-+d75VE89?AsDilbQ8m0*IeB+6qJ+j5^H5D9KTZ@d2-M;Pv&=d
zoDMu#4^{No)~}mwqNEO|qX@sbD*S}hW*vS1oeq}`LQwxE7Z4bC$X%?H_1)`b>*cgn
z5{7?t<o|b2J0v`t@uBBw|Dxq}KVWR3n<#Wf$9m#yXJ#pN|ILkP&L>oft8a>$UnTgx
zjhuVL$II_Wl{-y@^e$0}JJXN4fhW=-5p*wCNR6@B7CC4#dM}eKMXlt3aHBr~AeRrt
zd%CyGjAj5qv7X>CJ`{hQi(^tVrA*)asSO7WLnegY;wp1DM|+99uoESF|Jb;pNUWc;
z8g;TAMk28*3uVUyOAk)r&C<G6iBrOq8xD{qOQ5MJMKo7=n?QnZzWSbA!)bW!Q*3RH
zcmOgMHOade{ovL~)Z0?~?(V6IN1;T058Tgl1(9|8?i88NlkLho^w~+s#m(Z*sn0N}
z-V1^ySjCeaOXoV;oKfrhV`IG>^H-jV@bP-Oe`H~lUG#}u<u9JkT`n%bD@LQ;t`lmm
zgFSet=#DY%6F0dmdsTmRr61fP(KEmDe`*JIF?8v3Q*+An?Oxjox{#mXIsD{wDIsoV
z>R37w%?{Rf63yF!kj(A+Q4qPT9qogg`R!yR0e{qjYuWdh967gnsXgvxq|c6$Z&os2
zgfve*c%v5N`IWwIrfw)ju5#z7F|?6}_F=_i#L@{Q`0)H-V~!L4$(M}uQ!szZm%fwF
zuDF<`mCz#v@e@@J?F2{DV@t~tR`@HfDlfp5mn9c%cRF<?B6a4c6Kw5=@WeMa`^eUp
z>#--xQ7oezYq)n*r^hxIxolgqYUq@)tK7kS!(W+ruGVge&6Fm=e``bNnGwyidb<f&
zhK}mbq-j<A{rbyrliCH##iY?It<d!6FS)fw8d3_RpMd2jUL-O9pmyo9V6c}^_(pwc
z^8@(jcChk5eqae<6y|fg^VG^jvHG6Zs6hCljxbjaUhvdPhN26Xcp?9tg2d4}6OV(r
zFn0EmTtabKtAWc{LtPrbLiUP78eeK+wZtuFnMA$*Vv+5|h~Dw2Qvj_(Z*HPk_{MHq
zuK3;%8CIyKyc@0eupAr$(Z7J#pgS*mi}}wQT&~u?59-J}F<I+uV#658zSIOJ=r7vI
z2*dZyc~le}ytvz!Z`;_K6f+xfQ`>(2guenRuFr-Fm8vz>mmQB?o`D10Hq%4%^wbuv
zgm@i7z87-EG|>3(!+Z3R8K0!(d-h?)QUuA{yZ(b4PK}H>TV*k)ovucZrsU3S9k8da
z=Qz^qaL&85(aPdk4`?+UWu;(Zkh#4{R|?w<@a0ttZPLLS@{x+M_?9>#kJ$RzOxl~S
z?Z%v#9c<Q<FYD;a0scgxfr{X{U|k?kkYsjXajwUmypgi_o=WnYE~H>e{+7CE(ewF4
ziTgz%BYNs%e-&ItMu-GuXQD-Y8_|Z>gp}7Nt!BTjAbdiqL&~YPXr4O}$wwV|x2<45
z#xX9mA{@eC)y`~b1;YBW14ItjjZMBFP%62dx=&NC&?0_UJ@!p>(nWXcme4X7wmR)K
z7w<4lhV^OE=9Bye!bQtZQ7R3@d4XqJy-)1S5$R6*-MB{m5!P;ZpS>yap8#moL1x3e
za&BjbYhRHJSwZ~S$9t09`_kNm6K%Qc+C6^~CsADyeRtb?DTZa7YFz1mvXpA2{R^fA
zbZeQ<ati~LU_SS*2?#5pw4xmBqE8whI?XZT>(&wH;R8B4ZPqO%Z(_KG`jpSxGNoib
zQaDF@k&WmXuyl|XRYWE$xjNsRl2{2hU-Ih`ojWO{ITKC3{#TeVS*ez6WXd&y82m@_
zMLX40;QnYqCqv+w_=Y(B_`9xY%i}faRB7g@T|<X}2LgT`FEsnP?dC*s64LH6A}X&O
z7|>3+6}`MI-g^=E1C1B~V!QlJ+j-VnNxUSts%#ELCGYI#bP(kpSn%Kq7zes1ae&71
z3&cplkjqL)S_OiO$S7H7RkU&D6~Nfs6q6$ypMD3NAS_xPw)<TeTk<rH&{n4=73)0L
zTSUR9F5zT31g!H66jvhOXm9~jXgh)r?WNY;q_5RuNa@T}?guS{PHq!3U~?T{)If`*
zBFRmKzz$^sGVk#=*X<u2l&A6d(I@TMM#9Le9)|J|J})i5W?4@PE&3+Bk25Mch>m(k
zbgObwzuxD#L1{@S_NfGin%pjiL6tT$Tq5E!h*vxiWUKl+^POJxl?M`lHCP6)UXTa~
z1g&vb;=^^wx%U~(Fers4^9e=`1qooUfIL_f$Fh2FcnNcjZct_ue47}5lmFxPna-7_
zs^MIz2XW2O;wkgvm%Xlgy>ot$M*K8vm5s8-)cp%(X-+gn)JCnhb^iI0`1z4qmy%XE
zt-Nd(R3r~`OzJyDeal*>Cbo9LY95@?G~)daC}ZnOVgq2@6Lnb|DK#g`59ILm1Ccv^
z7Eb;nYNxDRq3-<7ggW$1TQso6ZRL%h5Ugh)CLy@+X_1nF@7nlnOg?(*xSthyp)6@$
z<O|nXw5oWRFz8vX!O<eg^J7;;0IzlFqp%35wZuyyD?_VoD|GrEmvj~ZbLIOe)<%xV
zp=~v;qy>Fw&sXol+gI{`^y3$`muFOs$K=GzmR!CgF@=~b!9<-2O)=7tVyo>nC){VN
zUy8!pO-JS#WExoUE0aWuKawpEoi4e@p+Mo9a}$)n*-?~}BzRw;m}X12L~JQl>JXf@
zBz?P}rn+$UdpRdn*EzE}l!J%VYY5A67@;X||Lzqbf`TXSq&zI4`-|l>hIPexVma-T
z_3deT$-obPo=)`-Dx$))j7aa;6^(?y7W<s+A6BBoWy*_Tam*yN_$w25n@Iel4xa@o
zsYw?vuq(UAA3rP;=$e{sq2myE0^D_Xf1fvU<YA4c(UJpRB=I-+a#Cu)Sk1`EIo05H
zrAyc^*R$uC87*H;QPF<$kywZQ@g6e+=v&y(b8=(;GIPBCc4l-uRojDE#QMf=I8H}E
z7xx$VA!`BgLXgMouLN%)b$6D*UU|<|EO)fxP^^Ee{yV2~iMId5sEjUuVK$Tln+ly^
z_6M7aH2CT7s5hQa8+B!JZ9l9&e8tjPF8h&0Y@VF#)X9cX?IMPsG)d(6VA~|9-IyKI
zl(-SuMVztuK?}gEHD&(INJ&KN$^wFGZBC=K_jHpa&%BndqmhUmYvmHCRjE>H)#BQ(
z_1-}?lg@b|yceNcD!D*}=Xav&yS#UJP{gE>J68HXsi*FLLBMV*@nugji4`A%l=PjM
z1sd63)^ZPms;S86|K@F%YR|FMtG5Fk#}E6q54;BRArtKB=sS!F9|sGHgIFmR_04dk
z1!ru?aFvg4nZ61OS;Vu)9RggDIdR4JFSIN+BaA<ZeZV4;PUj2+Dd?nrV*2VUEF21Z
zF|2pJIH*&4#+?@AS}z^GwUdB)2#Ac-^nSX7qx!%|gbLxwf7+Mg9FN4C6hHS$&Du&a
zI#3ha*bkE|>9(xOf^gDjsp79xcjavEWSa_OAAJn;{TF(YCNGvn5Ey))E5R?8)?UfV
zbVN=cZ|dUM5a+0x*6yZXa2Lsk86r3-y-Z#ne6rp2^<I*xY(ytQfVPOxUh&&h?Q>D}
z{Fq-ZBg5}drN87XKRQiWg1!12v~=6TKHlZXK%UsdGUV=y#lmo8<6Hsnt;U0A82E<>
z#v+o?>HlLoA9IQtB<$7veOn}XVY;0vKKCr3o;C9fnogjz&$e+<YnTaVI?Nu(BYeIR
znD!vJa#iY?o+-%GxGub7pIW7Bo?@-4>KfLAzoqgtjWH@s?71hBbLgLOr2(wUsBRZ9
zGlG6ic9|+}iekIHwP6h*tFO;Ag^e{c;z+AocC<6cgcJ1IsYi7RnhnU8QG7BlJ$An;
zY$W*ZsbKXtm6w`+Lxub@DSozk@cu#ACvi9ZoC=}I-r*Vl`zZa1c7zSoOkh_^jo0RP
zk9ZrQS0E-h*Wg8smD23pj9YMDiwVoHT($cP=kAJ8Y+~x2G5_f1e>WsgJ{(@}UDIk7
zF#9EfmYT2wciRnwNAAD1o2Xd`I%c%n?=!q)=!+KV@C%E&WIr)e`57~>{@e}~5=^p2
z@f==@Co$8ldRbiC4`>(V63~)u^}v?Y+xUt&qy13kV=|1(A~_Jdxhx~&fBhMp;+x9C
zY%mO|yr4ck$WvAJ6yB`Di28nOnNC*~+wd!TLlF)Rp8Kneq!z&=!Oi2Bbn#Vr&-u0J
zx=y0w-{GMGLnkDur-NL8@D`-@o%8x%q%qfvu@&Oc-zf)9$QW`MZ1%Ie4OF)LB6zIF
z+~>k}Oj*GHSp@2d7>f_G)hBxH!O--(6uQ$V9Z3)9yO0l>eiX3PYkKMuyAgGbP8QFw
z*LnyUP-D-GohjEGi1C22$=iLAwC!8I36B|&+(bBeN3iVT0UU8t*TSPiupXK#HM_xX
zl5<1eQ%HrwonPKHo)baMn(~nG(V~k&k<8Z&R$JM2JsNj|W|Ep#K4vd{bB(Xr&Om9a
zd1EAbL!F(X?jM<7oi}1qEk{Zt{#(v0YC@khQY|*bH|Cl1+*x6>vG7O#%{-|jSeuw>
yTL`kK1I}{`P&rMM%SHVsnDGC{g{RPkKfEH@zbOn^{aCp7>#MAaOtlm+=>Gxl$y!JN

literal 0
HcmV?d00001

diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt
index e4fe80e29..c56dd3232 100644
--- a/doc/src/week41/week41.do.txt
+++ b/doc/src/week41/week41.do.txt
@@ -1,4 +1,4 @@
-TITLE: Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks
+TITLE: Week 41 Neural networks and constructing a neural network code
 AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University
 DATE: Week 41
 
@@ -6,30 +6,1080 @@ DATE: Week 41
 !split
 ===== Plan for week 41 =====
 
-* Building our own Feed-forward Neural Network and discussion of project 2.
-* Playing around with our own Feed-forward Neural Network and introduction to TensorFlow. Solving differential equations with neural networks. 
-"
-Reading suggestions: These notes, "Aurelien Geron's chapters 10-11":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf".
-For a more in depth discussion on  neural networks we recommend Goodfellow et al chapters 6 and 7. Bishop's chapter 5 on Neural Networks is an additional good read.
 
-!split
-===== Videos on Neural Networks =====
-
-* "Video on Neural Networks":"https://www.youtube.com/watch?v=CqOfi41LfDw"
-
-* "Video on the back propagation algorithm":"https://www.youtube.com/watch?v=Ilg3gGewQ5U"
+!bblock  Material for the active learning sessions on Tuesday and Wednesday
+  * Exercise on writing your own stochastic gradient and gradient descent codes. This exercise continues next week with studies of automatic differentiation
+  * One lecture at the beginning of each session on the material from weeks 39 and 40 and how to write your own gradient descent code
+  * Discussion of project 2
+  * Your task before the sessions: revisit the material from weeks 39 and 40 and in particular the material from week 40 on stochastic gradient descent
+!eblock  
 
+!bblock Material for the lecture on Thursday October 12, 2023
+  * Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model.
+  * Building our own Feed-forward Neural Network
+  * Readings and Videos:
+    * These lecture notes
+    * For neural networks we recommend Goodfellow et al chapter 6.
+    * "Neural Networks demystified":"https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs"
+    * "Building Neural Networks from scratch":"https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex"
+    * "Video on Neural Networks":"https://www.youtube.com/watch?v=CqOfi41LfDw"
+    * "Video on the back propagation algorithm":"https://www.youtube.com/watch?v=Ilg3gGewQ5U"
 I also  recommend Michael Nielsen's intuitive approach to the neural networks and the universal approximation theorem, see the slides at URL:"http://neuralnetworksanddeeplearning.com/chap4.html".
+!eblock
 
 
 
 !split
-===== Review of the back propagation algorithm =====
+===== Lecture Thursday October 12 =====
+
+!split
+===== Introduction to Neural networks =====
+
+Artificial neural networks are computational systems that can learn to
+perform tasks by considering examples, generally without being
+programmed with any task-specific rules. It is supposed to mimic a
+biological system, wherein neurons interact by sending signals in the
+form of mathematical functions between layers. All layers can contain
+an arbitrary number of neurons, and each connection is represented by
+a weight variable.
+
+
+!split
+===== Artificial neurons  =====
+
+The field of artificial neural networks has a long history of
+development, and is closely connected with the advancement of computer
+science and computers in general. A model of artificial neurons was
+first developed by McCulloch and Pitts in 1943 to study signal
+processing in the brain and has later been refined by others. The
+general idea is to mimic neural networks in the human brain, which is
+composed of billions of neurons that communicate with each other by
+sending electrical signals.  Each neuron accumulates its incoming
+signals, which must exceed an activation threshold to yield an
+output. If the threshold is not overcome, the neuron remains inactive,
+i.e. has zero output.
+
+This behaviour has inspired a simple mathematical model for an artificial neuron.
+
+!bt
+\begin{equation}
+ y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
+ label{artificialNeuron}
+\end{equation}
+!et
+Here, the output $y$ of the neuron is the value of its activation function, which have as input
+a weighted sum of signals $x_i, \dots ,x_n$ received by $n$ other neurons.
+
+Conceptually, it is helpful to divide neural networks into four
+categories:
+o general purpose neural networks for supervised learning,
+o neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),
+o neural networks for sequential data such as Recurrent Neural Networks (RNNs), and
+o neural networks for unsupervised learning such as Deep Boltzmann Machines.
+
+
+In natural science, DNNs and CNNs have already found numerous
+applications. In statistical physics, they have been applied to detect
+phase transitions in 2D Ising and Potts models, lattice gauge
+theories, and different phases of polymers, or solving the
+Navier-Stokes equation in weather forecasting.  Deep learning has also
+found interesting applications in quantum physics. Various quantum
+phase transitions can be detected and studied using DNNs and CNNs,
+topological phases, and even non-equilibrium many-body
+localization. Representing quantum states as DNNs quantum state
+tomography are among some of the impressive achievements to reveal the
+potential of DNNs to facilitate the study of quantum systems.
+
+In quantum information theory, it has been shown that one can perform
+gate decompositions with the help of neural. 
+
+The applications are not limited to the natural sciences. There is a
+plethora of applications in essentially all disciplines, from the
+humanities to life science and medicine.
+
+!split
+===== Neural network types =====
+
+An artificial neural network (ANN), is a computational model that
+consists of layers of connected neurons, or nodes or units.  We will
+refer to these interchangeably as units or nodes, and sometimes as
+neurons.
+
+It is supposed to mimic a biological nervous system by letting each
+neuron interact with other neurons by sending signals in the form of
+mathematical functions between layers.  A wide variety of different
+ANNs have been developed, but most of them consist of an input layer,
+an output layer and eventual layers in-between, called *hidden
+layers*. All layers can contain an arbitrary number of nodes, and each
+connection between two nodes is associated with a weight variable.
+
+Neural networks (also called neural nets) are neural-inspired
+nonlinear models for supervised learning.  As we will see, neural nets
+can be viewed as natural, more powerful extensions of supervised
+learning methods such as linear and logistic regression and soft-max
+methods we discussed earlier.
+
+
+!split
+===== Feed-forward neural networks =====
+
+The feed-forward neural network (FFNN) was the first and simplest type
+of ANNs that were devised. In this network, the information moves in
+only one direction: forward through the layers.
+
+Nodes are represented by circles, while the arrows display the
+connections between the nodes, including the direction of information
+flow. Additionally, each arrow corresponds to a weight variable
+(figure to come).  We observe that each node in a layer is connected
+to *all* nodes in the subsequent layer, making this a so-called
+*fully-connected* FFNN.
+
+
+
+!split
+===== Convolutional Neural Network =====
+
+A different variant of FFNNs are *convolutional neural networks*
+(CNNs), which have a connectivity pattern inspired by the animal
+visual cortex. Individual neurons in the visual cortex only respond to
+stimuli from small sub-regions of the visual field, called a receptive
+field. This makes the neurons well-suited to exploit the strong
+spatially local correlation present in natural images. The response of
+each neuron can be approximated mathematically as a convolution
+operation.  (figure to come)
+
+Convolutional neural networks emulate the behaviour of neurons in the
+visual cortex by enforcing a *local* connectivity pattern between
+nodes of adjacent layers: Each node in a convolutional layer is
+connected only to a subset of the nodes in the previous layer, in
+contrast to the fully-connected FFNN.  Often, CNNs consist of several
+convolutional layers that learn local features of the input, with a
+fully-connected layer at the end, which gathers all the local data and
+produces the outputs. They have wide applications in image and video
+recognition.
+
+!split
+===== Recurrent neural networks =====
+
+So far we have only mentioned ANNs where information flows in one
+direction: forward. *Recurrent neural networks* on the other hand,
+have connections between nodes that form directed *cycles*. This
+creates a form of internal memory which are able to capture
+information on what has been calculated before; the output is
+dependent on the previous computations. Recurrent NNs make use of
+sequential information by performing the same task for every element
+in a sequence, where each element depends on previous elements. An
+example of such information is sentences, making recurrent NNs
+especially well-suited for handwriting and speech recognition.
+
+!split
+===== Other types of networks =====
+
+There are many other kinds of ANNs that have been developed. One type
+that is specifically designed for interpolation in multidimensional
+space is the radial basis function (RBF) network. RBFs are typically
+made up of three layers: an input layer, a hidden layer with
+non-linear radial symmetric activation functions and a linear output
+layer (''linear'' here means that each node in the output layer has a
+linear activation function). The layers are normally fully-connected
+and there are no cycles, thus RBFs can be viewed as a type of
+fully-connected FFNN. They are however usually treated as a separate
+type of NN due the unusual activation functions.
+
+!split
+===== Multilayer perceptrons  =====
+
+One uses often so-called fully-connected feed-forward neural networks
+with three or more layers (an input layer, one or more hidden layers
+and an output layer) consisting of neurons that have non-linear
+activation functions.
+
+Such networks are often called *multilayer perceptrons* (MLPs).
+
+!split
+===== Why multilayer perceptrons?  =====
+
+According to the *Universal approximation theorem*, a feed-forward
+neural network with just a single hidden layer containing a finite
+number of neurons can approximate a continuous multidimensional
+function to arbitrary accuracy, assuming the activation function for
+the hidden layer is a _non-constant, bounded and
+monotonically-increasing continuous function_.
+
+Note that the requirements on the activation function only applies to
+the hidden layer, the output nodes are always assumed to be linear, so
+as to not restrict the range of output values.
+
+
+!split
+=====  Illustration of a single perceptron model and a multi-perceptron model =====
+
+FIGURE: [figures/nns.png, width=600 frac=0.8]  In a) we show a single perceptron model while in b) we dispay a network with two  hidden layers, an input layer and an output layer.
+
+
+!split
+===== Examples of XOR, OR and AND gates =====
+
+
+
+Let us first try to fit various gates using standard linear
+regression. The gates we are thinking of are the classical XOR, OR and
+AND gates, well-known elements in computer science. The tables here
+show how we can set up the inputs $x_1$ and $x_2$ in order to yield a
+specific target $y_i$.
+
+
+
+
+!bc pycod
+"""
+Simple code that tests XOR, OR and AND gates with linear regression
+"""
+
+import numpy as np
+# Design matrix
+X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
+print(f"The X.TX  matrix:{X.T @ X}")
+Xinv = np.linalg.pinv(X.T @ X)
+print(f"The invers of X.TX  matrix:{Xinv}")
+
+# The XOR gate 
+yXOR = np.array( [ 0, 1 ,1, 0])
+ThetaXOR  = Xinv @ X.T @ yXOR
+print(f"The values of theta for the XOR gate:{ThetaXOR}")
+print(f"The linear regression prediction  for the XOR gate:{X @ ThetaXOR}")
+
+
+# The OR gate 
+yOR = np.array( [ 0, 1 ,1, 1])
+ThetaOR  = Xinv @ X.T @ yOR
+print(f"The values of theta for the OR gate:{ThetaOR}")
+print(f"The linear regression prediction  for the OR gate:{X @ ThetaOR}")
+
+
+# The OR gate 
+yAND = np.array( [ 0, 0 ,0, 1])
+ThetaAND  = Xinv @ X.T @ yAND
+print(f"The values of theta for the AND gate:{ThetaAND}")
+print(f"The linear regression prediction  for the AND gate:{X @ ThetaAND}")
+!ec
+
+What is happening here?
+
+!split
+===== Does Logistic Regression do a better Job? =====
+
+!bc pycod
+"""
+Simple code that tests XOR and OR gates with linear regression
+and logistic regression
+"""
+
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LogisticRegression
+import numpy as np
+
+# Design matrix
+X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
+print(f"The X.TX  matrix:{X.T @ X}")
+Xinv = np.linalg.pinv(X.T @ X)
+print(f"The invers of X.TX  matrix:{Xinv}")
+
+# The XOR gate 
+yXOR = np.array( [ 0, 1 ,1, 0])
+ThetaXOR  = Xinv @ X.T @ yXOR
+print(f"The values of theta for the XOR gate:{ThetaXOR}")
+print(f"The linear regression prediction  for the XOR gate:{X @ ThetaXOR}")
+
+
+# The OR gate 
+yOR = np.array( [ 0, 1 ,1, 1])
+ThetaOR  = Xinv @ X.T @ yOR
+print(f"The values of theta for the OR gate:{ThetaOR}")
+print(f"The linear regression prediction  for the OR gate:{X @ ThetaOR}")
+
+
+# The OR gate 
+yAND = np.array( [ 0, 0 ,0, 1])
+ThetaAND  = Xinv @ X.T @ yAND
+print(f"The values of theta for the AND gate:{ThetaAND}")
+print(f"The linear regression prediction  for the AND gate:{X @ ThetaAND}")
+
+# Now we change to logistic regression
+
+
+# Logistic Regression
+logreg = LogisticRegression()
+logreg.fit(X, yOR)
+print("Test set accuracy with Logistic Regression for OR gate: {:.2f}".format(logreg.score(X,yOR)))
+
+logreg.fit(X, yXOR)
+print("Test set accuracy with Logistic Regression for XOR gate: {:.2f}".format(logreg.score(X,yXOR)))
+
+
+logreg.fit(X, yAND)
+print("Test set accuracy with Logistic Regression for AND gate: {:.2f}".format(logreg.score(X,yAND)))
+!ec
+
+Not exactly impressive, but somewhat better.
+
+!split
+===== Adding Neural Networks =====
+
+!bc pycod
+
+# and now neural networks with Scikit-Learn and the XOR
+
+from sklearn.neural_network import MLPClassifier
+from sklearn.datasets import make_classification
+X, yXOR = make_classification(n_samples=100, random_state=1)
+FFNN = MLPClassifier(random_state=1, max_iter=300).fit(X, yXOR)
+FFNN.predict_proba(X)
+print(f"Test set accuracy with Feed Forward Neural Network  for XOR gate:{FFNN.score(X, yXOR)}")
+
+!ec
+
+
+
+!split
+===== Mathematical model  =====
+
+The output $y$ is produced via the activation function $f$
+!bt
+\[
+ y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
+\]
+!et
+This function receives $x_i$ as inputs.
+Here the activation $z=(\sum_{i=1}^n w_ix_i+b_i)$. 
+In an FFNN of such neurons, the *inputs* $x_i$ are the *outputs* of
+the neurons in the preceding layer. Furthermore, an MLP is
+fully-connected, which means that each neuron receives a weighted sum
+of the outputs of *all* neurons in the previous layer.
+
+!split
+===== Mathematical model  =====
+
+First, for each node $i$ in the first hidden layer, we calculate a weighted sum $z_i^1$ of the input coordinates $x_j$,
+
+!bt
+\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
+\end{equation}
+!et
+
+Here $b_i$ is the so-called bias which is normally needed in
+case of zero activation weights or inputs. How to fix the biases and
+the weights will be discussed below.  The value of $z_i^1$ is the
+argument to the activation function $f_i$ of each node $i$, The
+variable $M$ stands for all possible inputs to a given node $i$ in the
+first layer.  We define  the output $y_i^1$ of all neurons in layer 1 as
+
+!bt
+\begin{equation}
+ y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
+ label{outputLayer1}
+\end{equation}
+!et
+
+where we assume that all nodes in the same layer have identical
+activation functions, hence the notation $f$. In general, we could assume in the more general case that different layers have different activation functions.
+In this case we would identify these functions with a superscript $l$ for the $l$-th layer,
+
+!bt
+\begin{equation}
+ y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
+ label{generalLayer}
+\end{equation}
+!et
+
+where $N_l$ is the number of nodes in layer $l$. When the output of
+all the nodes in the first hidden layer are computed, the values of
+the subsequent layer can be calculated and so forth until the output
+is obtained.
+
+
+
+!split
+===== Mathematical model  =====
+
+The output of neuron $i$ in layer 2 is thus,
+
+!bt
+\begin{align}
+ y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) \\
+ &= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
+ label{outputLayer2}
+\end{align}
+!et
+where we have substituted $y_k^1$ with the inputs $x_k$. Finally, the ANN output reads
+
+!bt
+\begin{align}
+ y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) \\
+ &= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+  + b_1^3\right]
+\end{align}
+!et
+
+!split
+===== Mathematical model  =====
+
+We can generalize this expression to an MLP with $l$ hidden
+layers. The complete functional form is,
+
+!bt
+\begin{align}
+&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
+ label{completeNN}
+\end{align}
+!et
+
+which illustrates a basic property of MLPs: The only independent
+variables are the input values $x_n$.
+
+!split
+===== Mathematical model  =====
+
+This confirms that an MLP, despite its quite convoluted mathematical
+form, is nothing more than an analytic function, specifically a
+mapping of real-valued vectors $\hat{x} \in \mathbb{R}^n \rightarrow
+\hat{y} \in \mathbb{R}^m$.
+
+Furthermore, the flexibility and universality of an MLP can be
+illustrated by realizing that the expression is essentially a nested
+sum of scaled activation functions of the form
+
+!bt
+\begin{equation}
+ f(x) = c_1 f(c_2 x + c_3) + c_4
+\end{equation}
+!et
+
+where the parameters $c_i$ are weights and biases. By adjusting these
+parameters, the activation functions can be shifted up and down or
+left and right, change slope or be rescaled which is the key to the
+flexibility of a neural network.
+
+!split
+=== Matrix-vector notation ===
+
+We can introduce a more convenient notation for the activations in an A NN. 
+
+Additionally, we can represent the biases and activations
+as layer-wise column vectors $\hat{b}_l$ and $\hat{y}_l$, so that the $i$-th element of each vector 
+is the bias $b_i^l$ and activation $y_i^l$ of node $i$ in layer $l$ respectively. 
+
+We have that $\mathrm{W}_l$ is an $N_{l-1} \times N_l$ matrix, while $\hat{b}_l$ and $\hat{y}_l$ are $N_l \times 1$ column vectors. 
+With this notation, the sum becomes a matrix-vector multiplication, and we can write
+the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
+!bt
+\begin{equation}
+ \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
+ f_2\left(\left[\begin{array}{ccc}
+    w^2_{11} &w^2_{12} &w^2_{13} \\
+    w^2_{21} &w^2_{22} &w^2_{23} \\
+    w^2_{31} &w^2_{32} &w^2_{33} \\
+    \end{array} \right] \cdot
+    \left[\begin{array}{c}
+           y^1_1 \\
+           y^1_2 \\
+           y^1_3 \\
+          \end{array}\right] + 
+    \left[\begin{array}{c}
+           b^2_1 \\
+           b^2_2 \\
+           b^2_3 \\
+          \end{array}\right]\right).
+\end{equation}
+!et
+
+!split
+=== Matrix-vector notation  and activation ===
+
+The activation of node $i$ in layer 2 is
+
+!bt
+\begin{equation}
+ y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
+ f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
+\end{equation}
+!et 
+
+This is not just a convenient and compact notation, but also a useful
+and intuitive way to think about MLPs: The output is calculated by a
+series of matrix-vector multiplications and vector additions that are
+used as input to the activation functions. For each operation
+$\mathrm{W}_l \hat{y}_{l-1}$ we move forward one layer.
+
+
+!split
+=== Activation functions  ===
+
+
+A property that characterizes a neural network, other than its
+connectivity, is the choice of activation function(s).  As described
+in, the following restrictions are imposed on an activation function
+for a FFNN to fulfill the universal approximation theorem
+
+  * Non-constant
+
+  * Bounded
+
+  * Monotonically-increasing
+
+  * Continuous
+
+!split
+=== Activation functions, Logistic and Hyperbolic ones  ===
+
+The second requirement excludes all linear functions. Furthermore, in
+a MLP with only linear activation functions, each layer simply
+performs a linear transformation of its inputs.
+
+Regardless of the number of layers, the output of the NN will be
+nothing but a linear function of the inputs. Thus we need to introduce
+some kind of non-linearity to the NN to be able to fit non-linear
+functions Typical examples are the logistic *Sigmoid*
+
+!bt
+\[
+ f(x) = \frac{1}{1 + e^{-x}},
+\]
+!et
+and the *hyperbolic tangent* function
+!bt
+\[
+ f(x) = \tanh(x)
+\]
+!et
+
+!split
+=== Relevance === 
+
+The *sigmoid* function are more biologically plausible because the
+output of inactive neurons are zero. Such activation function are
+called *one-sided*. However, it has been shown that the hyperbolic
+tangent performs better than the sigmoid for training MLPs.  has
+become the most popular for *deep neural networks*
+
+!bc pycod
+"""The sigmoid function (or the logistic curve) is a 
+function that takes any real number, z, and outputs a number (0,1).
+It is useful in neural networks for assigning weights on a relative scale.
+The value z is the weighted sum of parameters involved in the learning algorithm."""
+
+import numpy
+import matplotlib.pyplot as plt
+import math as mt
+
+z = numpy.arange(-5, 5, .1)
+sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))
+sigma = sigma_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, sigma)
+ax.set_ylim([-0.1, 1.1])
+ax.set_xlim([-5,5])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('sigmoid function')
+
+plt.show()
+
+"""Step Function"""
+z = numpy.arange(-5, 5, .02)
+step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)
+step = step_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, step)
+ax.set_ylim([-0.5, 1.5])
+ax.set_xlim([-5,5])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('step function')
+
+plt.show()
+
+"""Sine Function"""
+z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)
+t = numpy.sin(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, t)
+ax.set_ylim([-1.0, 1.0])
+ax.set_xlim([-2*mt.pi,2*mt.pi])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('sine function')
+
+plt.show()
+
+"""Plots a graph of the squashing function used by a rectified linear
+unit"""
+z = numpy.arange(-2, 2, .1)
+zero = numpy.zeros(len(z))
+y = numpy.max([zero, z], axis=0)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, y)
+ax.set_ylim([-2.0, 2.0])
+ax.set_xlim([-2.0, 2.0])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('Rectified linear unit')
+
+plt.show()
+!ec
+
+
+
+!split
+===== The multilayer  perceptron (MLP) =====
+
+The multilayer perceptron is a very popular, and easy to implement approach, to deep learning. It consists of
+o A neural network with one or more layers of nodes between the input and the output nodes.
+o The multilayer network structure, or architecture, or topology, consists of an input layer, one or more hidden layers, and one output layer.
+o The input nodes pass values to the first hidden layer, its nodes pass the information on to the second and so on till we reach the output layer.
+
+As a convention it is normal to call  a  network with one layer of input units, one layer of hidden
+units and one layer of output units as  a two-layer network. A network with two layers of hidden units is called a three-layer network etc etc.
+
+For an MLP network there is no direct connection between the output nodes/neurons/units and  the input nodes/neurons/units.
+Hereafter we will call the various entities of a layer for nodes.
+There are also no connections within a single layer.
+
+The number of input nodes does not need to equal the number of output
+nodes. This applies also to the hidden layers. Each layer may have its
+own number of nodes and activation functions.
+
+The hidden layers have their name from the fact that they are not
+linked to observables and as we will see below when we define the
+so-called activation $\hat{z}$, we can think of this as a basis
+expansion of the original inputs $\hat{x}$. The difference however
+between neural networks and say linear regression is that now these
+basis functions (which will correspond to the weights in the network)
+are learned from data. This results in  an important difference between
+neural networks and deep learning approaches on one side and methods
+like logistic regression or linear regression and their modifications on the other side.
+
+
+!split
+=====  From one to many layers, the universal approximation theorem =====
+
+
+A neural network with only one layer, what we called the simple
+perceptron, is best suited if we have a standard binary model with
+clear (linear) boundaries between the outcomes. As such it could
+equally well be replaced by standard linear regression or logistic
+regression. Networks with one or more hidden layers approximate
+systems with more complex boundaries.
+
+As stated earlier, 
+an important theorem in studies of neural networks, restated without
+proof here, is the "universal approximation
+theorem":"http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873&rep=rep1&type=pdf".
+
+It states that a feed-forward network with a single hidden layer
+containing a finite number of neurons can approximate continuous
+functions on compact subsets of real functions. The theorem thus
+states that simple neural networks can represent a wide variety of
+interesting functions when given appropriate parameters. It is the
+multilayer feedforward architecture itself which gives neural networks
+the potential of being universal approximators.
+
+
+!split
+=====  Deriving the back propagation code for a multilayer perceptron model =====
+
+
+
+As we have seen now in a feed forward network, we can express the final output of our network in terms of basic matrix-vector multiplications.
+The unknowwn quantities are our weights $w_{ij}$ and we need to find an algorithm for changing them so that our errors are as small as possible.
+This leads us to the famous "back propagation algorithm":"https://www.nature.com/articles/323533a0".
+
+The questions we want to ask are how do changes in the biases and the
+weights in our network change the cost function and how can we use the
+final output to modify the weights?
+
+To derive these equations let us start with a plain regression problem
+and define our cost function as
+
+!bt
+\[
+{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2, 
+\]
+!et
+
+where the $t_i$s are our $n$ targets (the values we want to
+reproduce), while the outputs of the network after having propagated
+all inputs $\hat{x}$ are given by $y_i$.  Below we will demonstrate
+how the basic equations arising from the back propagation algorithm
+can be modified in order to study classification problems with $K$
+classes.
+
+!split
+===== Definitions =====
+
+With our definition of the targets $\hat{t}$, the outputs of the
+network $\hat{y}$ and the inputs $\hat{x}$ we
+define now the activation $z_j^l$ of node/neuron/unit $j$ of the
+$l$-th layer as a function of the bias, the weights which add up from
+the previous layer $l-1$ and the forward passes/outputs
+$\hat{a}^{l-1}$ from the previous layer as
+
+
+!bt
+\[
+z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,
+\]
+!et
+
+where $b_k^l$ are the biases from layer $l$.  Here $M_{l-1}$
+represents the total number of nodes/neurons/units of layer $l-1$. The
+figure here illustrates this equation.  We can rewrite this in a more
+compact form as the matrix-vector products we discussed earlier,
+
+!bt
+\[
+\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
+\]
+!et
+
+With the activation values $\hat{z}^l$ we can in turn define the
+output of layer $l$ as $\hat{a}^l = f(\hat{z}^l)$ where $f$ is our
+activation function. In the examples here we will use the sigmoid
+function discussed in our logistic regression lectures. We will also use the same activation function $f$ for all layers
+and their nodes.  It means we have
+
+!bt
+\[
+a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
+\]
+!et
+
+
+!split
+===== Derivatives and the chain rule =====
+
+From the definition of the activation $z_j^l$ we have
+!bt
+\[
+\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1},
+\]
+!et
+and
+!bt
+\[
+\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. 
+\]
+!et
+
+With our definition of the activation function we have that (note that this function depends only on $z_j^l$)
+!bt
+\[
+\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)). 
+\]
+!et
+
+
+!split
+===== Derivative of the cost function =====
+
+With these definitions we can now compute the derivative of the cost function in terms of the weights.
+
+Let us specialize to the output layer $l=L$. Our cost function is
+!bt
+\[
+{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2, 
+\]
+!et
+The derivative of this function with respect to the weights is
+
+!bt
+\[
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}}, 
+\]
+!et
+The last partial derivative can easily be computed and reads (by applying the chain rule)
+!bt
+\[
+\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},  
+\]
+!et
+
+
+
+!split
+===== Bringing it together, first back propagation equation =====
+
+We have thus
+!bt
+\[
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1}, 
+\]
+!et
+
+Defining
+!bt
+\[
+\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\]
+!et
+and using the Hadamard product of two vectors we can write this as
+!bt
+\[
+\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}^L)}.
+\]
+!et
+
+This is an important expression. The second term on the right handside
+measures how fast the cost function is changing as a function of the $j$th
+output activation.  If, for example, the cost function doesn't depend
+much on a particular output node $j$, then $\delta_j^L$ will be small,
+which is what we would expect. The first term on the right, measures
+how fast the activation function $f$ is changing at a given activation
+value $z_j^L$.
+
+Notice that everything in the above equations is easily computed.  In
+particular, we compute $z_j^L$ while computing the behaviour of the
+network, and it is only a small additional overhead to compute
+$f'(z^L_j)$.  The exact form of the derivative with respect to the
+output depends on the form of the cost function.
+However, provided the cost function is known there should be little
+trouble in calculating
+
+!bt
+\[
+\frac{\partial {\cal C}}{\partial (a_j^L)}
+\]
+!et
+
+With the definition of $\delta_j^L$ we have a more compact definition of the derivative of the cost function in terms of the weights, namely
+!bt
+\[
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
+\]
+!et
+
+!split
+===== Derivatives in terms of $z_j^L$ =====
+
+It is also easy to see that our previous equation can be written as
+
+!bt
+\[
+\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
+\]
+!et
+which can also be interpreted as the partial derivative of the cost function with respect to the biases $b_j^L$, namely
+!bt
+\[
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
+\]
+!et
+That is, the error $\delta_j^L$ is exactly equal to the rate of change of the cost function as a function of the bias. 
+!split
+===== Bringing it together =====
+
+We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are
+
+!bblock The starting equations
+
+!bt
+\begin{equation}
+\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
+\end{equation}
+!et
+and
+!bt
+\begin{equation}
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
+\end{equation}
+!et
+and
+
+!bt
+\begin{equation}
+\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
+\end{equation}
+!et
+!eblock
+
+
+An interesting consequence of the above equations is that when the
+activation $a_k^{L-1}$ is small, the gradient term, that is the
+derivative of the cost function with respect to the weights, will also
+tend to be small. We say then that the weight learns slowly, meaning
+that it changes slowly when we minimize the weights via say gradient
+descent. In this case we say the system learns slowly.
+
+Another interesting feature is that is when the activation function,
+represented by the sigmoid function here, is rather flat when we move towards
+its end values $0$ and $1$ (see the above Python codes). In these
+cases, the derivatives of the activation function will also be close
+to zero, meaning again that the gradients will be small and the
+network learns slowly again.
+
+
+
+We need a fourth equation and we are set. We are going to propagate
+backwards in order to the determine the weights and biases. In order
+to do so we need to represent the error in the layer before the final
+one $L-1$ in terms of the errors in the final output layer.
+
+!split
+===== Final back propagating equation =====
+
+We have that (replacing $L$ with a general layer $l$)
+!bt
+\[
+\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
+\]
+!et
+We want to express this in terms of the equations for layer $l+1$. Using the chain rule and summing over all $k$ entries we have
+
+!bt
+\[
+\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
+\]
+!et
+and recalling that
+!bt
+\[
+z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},
+\]
+!et
+with $M_l$ being the number of nodes in layer $l$, we obtain
+!bt
+\[
+\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
+\]
+!et
+This is our final equation.
+
+We are now ready to set up the algorithm for back propagation and learning the weights and biases.
+
+!split
+===== Setting up the Back propagation algorithm =====
+
+
+
+The four equations  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.
+
+!bblock
+First, we set up the input data $\hat{x}$ and the activations
+$\hat{z}_1$ of the input layer and compute the activation function and
+the pertinent outputs $\hat{a}^1$.
+!eblock
+
+!bblock
+Secondly, we perform then the feed forward till we reach the output
+layer and compute all $\hat{z}_l$ of the input layer and compute the
+activation function and the pertinent outputs $\hat{a}^l$ for
+$l=2,3,\dots,L$.
+!eblock
+
+!bblock
+Thereafter we compute the ouput error $\hat{\delta}^L$ by computing all
+!bt
+\[
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+\]
+!et
+!eblock
+
+!bblock
+Then we compute the back propagate error for each $l=L-1,L-2,\dots,2$ as
+!bt
+\[
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+\]
+!et
+!eblock
+
+!bblock
+Finally, we update the weights and the biases using gradient descent for each $l=L-1,L-2,\dots,2$ and update the weights and biases according to the rules
+!bt
+\[
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+\]
+!et
+
+!bt
+\[
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+\]
+!et
+!eblock
+
+The parameter $\eta$ is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+
+
+!split
+===== Setting up the Back propagation algorithm =====
+
+
+
+The four equations  above  provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.
+
+!bblock
+First, we set up the input data $\bm{x}$ and the activations
+$\bm{z}_1$ of the input layer and compute the activation function and
+the pertinent outputs $\bm{a}^1$.
+!eblock
+
+!bblock
+Secondly, we perform then the feed forward till we reach the output
+layer and compute all $\bm{z}_l$ of the input layer and compute the
+activation function and the pertinent outputs $\bm{a}^l$ for
+$l=2,3,\dots,L$.
+!eblock
+
+!bblock
+Thereafter we compute the ouput error $\bm{\delta}^L$ by computing all
+!bt
+\[
+\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
+\]
+!et
+!eblock
+
+!bblock
+Then we compute the back propagate error for each $l=L-1,L-2,\dots,2$ as
+!bt
+\[
+\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
+\]
+!et
+!eblock
+
+!bblock
+Finally, we update the weights and the biases using gradient descent for each $l=L-1,L-2,\dots,2$ and update the weights and biases according to the rules
+!bt
+\[
+w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
+\]
+!et
+
+!bt
+\[
+b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l,
+\]
+!et
+!eblock
+
+The parameter $\eta$ is the learning parameter discussed in connection with the gradient descent methods.
+Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.
+
+
+
 
-During the last lecture we discussed in detail the back propagation
-algorithm. This algorithm is based on a repeated application of the
-chain rule. Let us bring back the basic equation and at the same time
-link this with the basic mathematics of automatic differentiation.
 
 
 
@@ -39,7 +1089,7 @@ link this with the basic mathematics of automatic differentiation.
 
 
 
-The four equations  derived last week provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.
+The four equations  derived discussed above provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.
 
 !bblock
 First, we set up the input data $\bm{x}$ and the activations
@@ -1277,689 +2327,10 @@ plt.show()
 !ec
 
 
-!split
-===== Building neural networks in Tensorflow and Keras =====
 
-Now we want  to build on the experience gained from our neural network implementation in NumPy and scikit-learn
-and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy
-and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer.  
-  
-In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite
-clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or
-NumPy arrays.
 
-!split
-===== Tensorflow =====
-  
-Tensorflow is an open source library machine learning library
-developed by the Google Brain team for internal use. It was released
-under the Apache 2.0 open source license in November 9, 2015.
-  
-Tensorflow is a computational framework that allows you to construct
-machine learning models at different levels of abstraction, from
-high-level, object-oriented APIs like Keras, down to the C++ kernels
-that Tensorflow is built upon. The higher levels of abstraction are
-simpler to use, but less flexible, and our choice of implementation
-should reflect the problems we are trying to solve.
-  
-"Tensorflow uses":"https://www.tensorflow.org/guide/graphs" so-called graphs to represent your computation
-in terms of the dependencies between individual operations, such that you first build a Tensorflow *graph*
-to represent your model, and then create a Tensorflow *session* to run the graph.
-  
-In this guide we will analyze the same data as we did in our NumPy and
-scikit-learn tutorial, gathered from the MNIST database of images. We
-will give an introduction to the lower level Python Application
-Program Interfaces (APIs), and see how we use them to build our graph.
-Then we will build (effectively) the same graph in Keras, to see just
-how simple solving a machine learning problem can be.
-  
-To install tensorflow on Unix/Linux systems, use pip as
-!bc pycod
-pip3 install tensorflow
-!ec
-and/or if you use _anaconda_, just write (or install from the graphical user interface)
-(current release of CPU-only TensorFlow)
-!bc pycod 
-conda create -n tf tensorflow
-conda activate tf
-!ec
-To install the current release of GPU TensorFlow
-!bc pycod
-conda create -n tf-gpu tensorflow-gpu
-conda activate tf-gpu
-!ec
 
-!split
-===== Using Keras =====
-  
-Keras is a high level "neural network":"https://en.wikipedia.org/wiki/Application_programming_interface"
-that supports Tensorflow, CTNK and Theano as backends.  
-If you have Anaconda installed you may run the following command
-!bc pycod
-conda install keras
-!ec
-You can look up the "instructions here":"https://keras.io/" for more information.
 
-We will to a large extent use _keras_ in this course. 
-
-!split
-===== Collect and pre-process data =====
-
-Let us look again at the MINST data set.
-
-!bc pycod
-# import necessary packages
-import numpy as np
-import matplotlib.pyplot as plt
-import tensorflow as tf
-from sklearn import datasets
-
-
-# ensure the same random numbers appear every time
-np.random.seed(0)
-
-# display images in notebook
-%matplotlib inline
-plt.rcParams['figure.figsize'] = (12,12)
-
-
-# download MNIST dataset
-digits = datasets.load_digits()
-
-# define inputs and labels
-inputs = digits.images
-labels = digits.target
-
-print("inputs = (n_inputs, pixel_width, pixel_height) = " + str(inputs.shape))
-print("labels = (n_inputs) = " + str(labels.shape))
-
-
-# flatten the image
-# the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64
-n_inputs = len(inputs)
-inputs = inputs.reshape(n_inputs, -1)
-print("X = (n_inputs, n_features) = " + str(inputs.shape))
-
-
-# choose some random images to display
-indices = np.arange(n_inputs)
-random_indices = np.random.choice(indices, size=5)
-
-for i, image in enumerate(digits.images[random_indices]):
-    plt.subplot(1, 5, i+1)
-    plt.axis('off')
-    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
-    plt.title("Label: %d" % digits.target[random_indices[i]])
-plt.show()
-!ec
-
-!bc pycod
-from tensorflow.keras.layers import Input
-from tensorflow.keras.models import Sequential      #This allows appending layers to existing models
-from tensorflow.keras.layers import Dense           #This allows defining the characteristics of a particular layer
-from tensorflow.keras import optimizers             #This allows using whichever optimiser we want (sgd,adam,RMSprop)
-from tensorflow.keras import regularizers           #This allows using whichever regularizer we want (l1,l2,l1_l2)
-from tensorflow.keras.utils import to_categorical   #This allows using categorical cross entropy as the cost function
-
-from sklearn.model_selection import train_test_split
-
-# one-hot representation of labels
-labels = to_categorical(labels)
-
-# split into train and test data
-train_size = 0.8
-test_size = 1 - train_size
-X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
-                                                    test_size=test_size)
-!ec
-
-
-
-!bc pycod
-
-epochs = 100
-batch_size = 100
-n_neurons_layer1 = 100
-n_neurons_layer2 = 50
-n_categories = 10
-eta_vals = np.logspace(-5, 1, 7)
-lmbd_vals = np.logspace(-5, 1, 7)
-def create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories, eta, lmbd):
-    model = Sequential()
-    model.add(Dense(n_neurons_layer1, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd)))
-    model.add(Dense(n_categories, activation='softmax'))
-    
-    sgd = optimizers.SGD(lr=eta)
-    model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
-    
-    return model
-!ec
-
-!bc pycod
-DNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
-        
-for i, eta in enumerate(eta_vals):
-    for j, lmbd in enumerate(lmbd_vals):
-        DNN = create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories,
-                                         eta=eta, lmbd=lmbd)
-        DNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)
-        scores = DNN.evaluate(X_test, Y_test)
-        
-        DNN_keras[i][j] = DNN
-        
-        print("Learning rate = ", eta)
-        print("Lambda = ", lmbd)
-        print("Test accuracy: %.3f" % scores[1])
-        print()
-!ec
-
-
-
-!bc pycod
-# optional
-# visual representation of grid search
-# uses seaborn heatmap, could probably do this in matplotlib
-import seaborn as sns
-
-sns.set()
-
-train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-
-for i in range(len(eta_vals)):
-    for j in range(len(lmbd_vals)):
-        DNN = DNN_keras[i][j]
-
-        train_accuracy[i][j] = DNN.evaluate(X_train, Y_train)[1]
-        test_accuracy[i][j] = DNN.evaluate(X_test, Y_test)[1]
-
-        
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Training Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
-
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Test Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
-!ec
-
-
-
-!split
-=====  The Breast Cancer Data, now with Keras =====
-
-!bc pycod
-
-import tensorflow as tf
-from tensorflow.keras.layers import Input
-from tensorflow.keras.models import Sequential      #This allows appending layers to existing models
-from tensorflow.keras.layers import Dense           #This allows defining the characteristics of a particular layer
-from tensorflow.keras import optimizers             #This allows using whichever optimiser we want (sgd,adam,RMSprop)
-from tensorflow.keras import regularizers           #This allows using whichever regularizer we want (l1,l2,l1_l2)
-from tensorflow.keras.utils import to_categorical   #This allows using categorical cross entropy as the cost function
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-from sklearn.model_selection import train_test_split as splitter
-from sklearn.datasets import load_breast_cancer
-import pickle
-import os 
-
-
-"""Load breast cancer dataset"""
-
-np.random.seed(0)        #create same seed for random number every time
-
-cancer=load_breast_cancer()      #Download breast cancer dataset
-
-inputs=cancer.data                     #Feature matrix of 569 rows (samples) and 30 columns (parameters)
-outputs=cancer.target                  #Label array of 569 rows (0 for benign and 1 for malignant)
-labels=cancer.feature_names[0:30]
-
-print('The content of the breast cancer dataset is:')      #Print information about the datasets
-print(labels)
-print('-------------------------')
-print("inputs =  " + str(inputs.shape))
-print("outputs =  " + str(outputs.shape))
-print("labels =  "+ str(labels.shape))
-
-x=inputs      #Reassign the Feature and Label matrices to other variables
-y=outputs
-
-#%% 
-
-# Visualisation of dataset (for correlation analysis)
-
-plt.figure()
-plt.scatter(x[:,0],x[:,2],s=40,c=y,cmap=plt.cm.Spectral)
-plt.xlabel('Mean radius',fontweight='bold')
-plt.ylabel('Mean perimeter',fontweight='bold')
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,5],x[:,6],s=40,c=y, cmap=plt.cm.Spectral)
-plt.xlabel('Mean compactness',fontweight='bold')
-plt.ylabel('Mean concavity',fontweight='bold')
-plt.show()
-
-
-plt.figure()
-plt.scatter(x[:,0],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)
-plt.xlabel('Mean radius',fontweight='bold')
-plt.ylabel('Mean texture',fontweight='bold')
-plt.show()
-
-plt.figure()
-plt.scatter(x[:,2],x[:,1],s=40,c=y,cmap=plt.cm.Spectral)
-plt.xlabel('Mean perimeter',fontweight='bold')
-plt.ylabel('Mean compactness',fontweight='bold')
-plt.show()
-
-
-# Generate training and testing datasets
-
-#Select features relevant to classification (texture,perimeter,compactness and symmetery) 
-#and add to input matrix
-
-temp1=np.reshape(x[:,1],(len(x[:,1]),1))
-temp2=np.reshape(x[:,2],(len(x[:,2]),1))
-X=np.hstack((temp1,temp2))      
-temp=np.reshape(x[:,5],(len(x[:,5]),1))
-X=np.hstack((X,temp))       
-temp=np.reshape(x[:,8],(len(x[:,8]),1))
-X=np.hstack((X,temp))       
-
-X_train,X_test,y_train,y_test=splitter(X,y,test_size=0.1)   #Split datasets into training and testing
-
-y_train=to_categorical(y_train)     #Convert labels to categorical when using categorical cross entropy
-y_test=to_categorical(y_test)
-
-del temp1,temp2,temp
-
-# %%
-
-# Define tunable parameters"
-
-eta=np.logspace(-3,-1,3)                    #Define vector of learning rates (parameter to SGD optimiser)
-lamda=0.01                                  #Define hyperparameter
-n_layers=2                                  #Define number of hidden layers in the model
-n_neuron=np.logspace(0,3,4,dtype=int)       #Define number of neurons per layer
-epochs=100                                   #Number of reiterations over the input data
-batch_size=100                              #Number of samples per gradient update
-
-# %%
-
-"""Define function to return Deep Neural Network model"""
-
-def NN_model(inputsize,n_layers,n_neuron,eta,lamda):
-    model=Sequential()      
-    for i in range(n_layers):       #Run loop to add hidden layers to the model
-        if (i==0):                  #First layer requires input dimensions
-            model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda),input_dim=inputsize))
-        else:                       #Subsequent layers are capable of automatic shape inferencing
-            model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda)))
-    model.add(Dense(2,activation='softmax'))  #2 outputs - ordered and disordered (softmax for prob)
-    sgd=optimizers.SGD(lr=eta)
-    model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])
-    return model
-
-    
-Train_accuracy=np.zeros((len(n_neuron),len(eta)))      #Define matrices to store accuracy scores as a function
-Test_accuracy=np.zeros((len(n_neuron),len(eta)))       #of learning rate and number of hidden neurons for 
-
-for i in range(len(n_neuron)):     #run loops over hidden neurons and learning rates to calculate 
-    for j in range(len(eta)):      #accuracy scores 
-        DNN_model=NN_model(X_train.shape[1],n_layers,n_neuron[i],eta[j],lamda)
-        DNN_model.fit(X_train,y_train,epochs=epochs,batch_size=batch_size,verbose=1)
-        Train_accuracy[i,j]=DNN_model.evaluate(X_train,y_train)[1]
-        Test_accuracy[i,j]=DNN_model.evaluate(X_test,y_test)[1]
-               
-
-def plot_data(x,y,data,title=None):
-
-    # plot results
-    fontsize=16
-
-
-    fig = plt.figure()
-    ax = fig.add_subplot(111)
-    cax = ax.matshow(data, interpolation='nearest', vmin=0, vmax=1)
-    
-    cbar=fig.colorbar(cax)
-    cbar.ax.set_ylabel('accuracy (%)',rotation=90,fontsize=fontsize)
-    cbar.set_ticks([0,.2,.4,0.6,0.8,1.0])
-    cbar.set_ticklabels(['0%','20%','40%','60%','80%','100%'])
-
-    # put text on matrix elements
-    for i, x_val in enumerate(np.arange(len(x))):
-        for j, y_val in enumerate(np.arange(len(y))):
-            c = "${0:.1f}\\%$".format( 100*data[j,i])  
-            ax.text(x_val, y_val, c, va='center', ha='center')
-
-    # convert axis vaues to to string labels
-    x=[str(i) for i in x]
-    y=[str(i) for i in y]
-
-
-    ax.set_xticklabels(['']+x)
-    ax.set_yticklabels(['']+y)
-
-    ax.set_xlabel('$\\mathrm{learning\\ rate}$',fontsize=fontsize)
-    ax.set_ylabel('$\\mathrm{hidden\\ neurons}$',fontsize=fontsize)
-    if title is not None:
-        ax.set_title(title)
-
-    plt.tight_layout()
-
-    plt.show()
-    
-plot_data(eta,n_neuron,Train_accuracy, 'training')
-plot_data(eta,n_neuron,Test_accuracy, 'testing')
-
-!ec
-
-
-
-
-
-
-!split
-===== Fine-tuning neural network hyperparameters =====
-
-The flexibility of neural networks is also one of their main
-drawbacks: there are many hyperparameters to tweak. Not only can you
-use any imaginable network topology (how neurons/nodes are interconnected),
-but even in a simple FFNN you can change the number of layers, the
-number of neurons per layer, the type of activation function to use in
-each layer, the weight initialization logic, the stochastic gradient optmized and much more. How do you
-know what combination of hyperparameters is the best for your task?
-
-* You can use grid search with cross-validation to find the right hyperparameters.
-
-However,since there are many hyperparameters to tune, and since
-training a neural network on a large dataset takes a lot of time, you
-will only be able to explore a tiny part of the hyperparameter space.
-
-
-* You can use randomized search.
-* Or use tools like "Oscar":"http://oscar.calldesk.ai/", which implements more complex algorithms to help you find a good set of hyperparameters quickly.  
-
-!split
-===== Hidden layers =====
-
-
-
-For many problems you can start with just one or two hidden layers and it will work just fine.
-For the MNIST data set you ca easily get a high accuracy using just one hidden layer with a
-few hundred neurons.
-You can reach for this data set above 98% accuracy using two hidden layers with the same total amount of
-neurons, in roughly the same amount of training time. 
-
-For more complex problems, you can gradually
-ramp up the number of hidden layers, until you start overfitting the training set. Very complex tasks, such
-as large image classification or speech recognition, typically require networks with dozens of layers
-and they need a huge amount
-of training data. However, you will rarely have to train such networks from scratch: it is much more
-common to reuse parts of a pretrained state-of-the-art network that performs a similar task.
-
-
-
-
-!split 
-===== Which activation function should I use? =====
-
-The Back propagation algorithm we derived above works by going from
-the output layer to the input layer, propagating the error gradient on
-the way. Once the algorithm has computed the gradient of the cost
-function with regards to each parameter in the network, it uses these
-gradients to update each parameter with a Gradient Descent (GD) step.
-
-
-Unfortunately for us, the gradients often get smaller and smaller as the
-algorithm progresses down to the first hidden layers. As a result, the
-GD update leaves the lower layer connection weights
-virtually unchanged, and training never converges to a good
-solution. This is known in the literature as 
-_the vanishing gradients problem_. 
-
-In other cases, the opposite can happen, namely the the gradients can grow bigger and
-bigger. The result is that many of the layers get large updates of the 
-weights the
-algorithm diverges. This is the _exploding gradients problem_, which is
-mostly encountered in recurrent neural networks. More generally, deep
-neural networks suffer from unstable gradients, different layers may
-learn at widely different speeds
-
-
-
-!split 
-===== Is the Logistic activation function (Sigmoid)  our choice? =====
-
-Although this unfortunate behavior has been empirically observed for
-quite a while (it was one of the reasons why deep neural networks were
-mostly abandoned for a long time), it is only around 2010 that
-significant progress was made in understanding it.
-
-A paper titled "Understanding the Difficulty of Training Deep
-Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio":"http://proceedings.mlr.press/v9/glorot10a.html" found that
-the problems with the popular logistic
-sigmoid activation function and the weight initialization technique
-that was most popular at the time, namely random initialization using
-a normal distribution with a mean of 0 and a standard deviation of
-1. 
-
-They showed that with this activation function and this
-initialization scheme, the variance of the outputs of each layer is
-much greater than the variance of its inputs. Going forward in the
-network, the variance keeps increasing after each layer until the
-activation function saturates at the top layers. This is actually made
-worse by the fact that the logistic function has a mean of 0.5, not 0
-(the hyperbolic tangent function has a mean of 0 and behaves slightly
-better than the logistic function in deep networks).
-
-
-!split
-===== The derivative of the Logistic funtion =====
-
-Looking at the logistic activation function, when inputs become large
-(negative or positive), the function saturates at 0 or 1, with a
-derivative extremely close to 0. Thus when backpropagation kicks in,
-it has virtually no gradient to propagate back through the network,
-and what little gradient exists keeps getting diluted as
-backpropagation progresses down through the top layers, so there is
-really nothing left for the lower layers.
-
-In their paper, Glorot and Bengio propose a way to significantly
-alleviate this problem. We need the signal to flow properly in both
-directions: in the forward direction when making predictions, and in
-the reverse direction when backpropagating gradients. We don’t want
-the signal to die out, nor do we want it to explode and saturate. For
-the signal to flow properly, the authors argue that we need the
-variance of the outputs of each layer to be equal to the variance of
-its inputs, and we also need the gradients to have equal variance
-before and after flowing through a layer in the reverse direction.
-
-
-
-One of the insights in the 2010 paper by Glorot and Bengio was that
-the vanishing/exploding gradients problems were in part due to a poor
-choice of activation function. Until then most people had assumed that
-if Nature had chosen to use roughly sigmoid activation functions in
-biological neurons, they must be an excellent choice. But it turns out
-that other activation functions behave much better in deep neural
-networks, in particular the ReLU activation function, mostly because
-it does not saturate for positive values (and also because it is quite
-fast to compute).
-
-
-!split
-===== The RELU function family =====
-
-The ReLU activation function suffers from a problem known as the dying
-ReLUs: during training, some neurons effectively die, meaning they
-stop outputting anything other than 0.
-
-In some cases, you may find that half of your network’s neurons are
-dead, especially if you used a large learning rate. During training,
-if a neuron’s weights get updated such that the weighted sum of the
-neuron’s inputs is negative, it will start outputting 0. When this
-happen, the neuron is unlikely to come back to life since the gradient
-of the ReLU function is 0 when its input is negative.
-
-To solve this problem, nowadays practitioners use a  variant of the ReLU
-function, such as the leaky ReLU discussed above or the so-called
-exponential linear unit (ELU) function
-
-
-!bt
-\[
-ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\  z & z \ge 0.\end{array}\right. 
-\]
-!et
-
-!split
-===== Which activation function should we use? =====
-
-In general it seems that the ELU activation function is better than
-the leaky ReLU function (and its variants), which is better than
-ReLU. ReLU performs better than $\tanh$ which in turn performs better
-than the logistic function. 
-
-If runtime
-performance is an issue, then you may opt for the  leaky ReLU function  over the 
-ELU function If you don’t
-want to tweak yet another hyperparameter, you may just use the default
-$\alpha$ of $0.01$ for the leaky ReLU, and $1$ for ELU. If you have
-spare time and computing power, you can use cross-validation or
-bootstrap to evaluate other activation functions.
-
-
-!split
-=====  More on activation functions, output layers =====
-
-In most cases you can use the ReLU activation function in the hidden layers (or one of its variants).
-
-It is a bit faster to compute than other activation functions, and the gradient descent optimization does in general not get stuck.
-
-_For the output layer:_
-
-* For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).
-* For regression tasks, you can simply use no activation function at all.
-
-
-
-
-!split
-===== Batch Normalization =====
-
-Batch Normalization
-aims to address the vanishing/exploding gradients problems, and more generally the problem that the
-distribution of each layer’s inputs changes during training, as the parameters of the previous layers change.
-
-The technique consists of adding an operation in the model just before the activation function of each
-layer, simply zero-centering and normalizing the inputs, then scaling and shifting the result using two new
-parameters per layer (one for scaling, the other for shifting). In other words, this operation lets the model
-learn the optimal scale and mean of the inputs for each layer.
-In order to zero-center and normalize the inputs, the algorithm needs to estimate the inputs’ mean and
-standard deviation. It does so by evaluating the mean and standard deviation of the inputs over the current
-mini-batch, from this the name batch normalization.
-
-!split
-===== Dropout =====
-
-It is a fairly simple algorithm: at every training step, every neuron (including the input neurons but
-excluding the output neurons) has a probability $p$ of being temporarily dropped out, meaning it will be
-entirely ignored during this training step, but it may be active during the next step.
-
-The
-hyperparameter $p$ is called the dropout rate, and it is typically set to 50%. After training, the neurons are not dropped anymore.
- It is viewed as one of the most popular regularization techniques.
-
-!split
-===== Gradient Clipping =====
-
-A popular technique to lessen the exploding gradients problem is to simply clip the gradients during
-backpropagation so that they never exceed some threshold (this is mostly useful for recurrent neural
-networks).
-
-This technique is called Gradient Clipping.
-
-In general however, Batch
-Normalization is preferred.
-
-!split
-=====  A very nice website on Neural Networks =====
-
-You may find this "website":"https://playground.tensorflow.org/#activation=tanh&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,2&seed=0.29243&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false" very useful. 
-
-!split 
-===== A top-down perspective on Neural networks =====
-
-
-The first thing we would like to do is divide the data into two or three
-parts. A training set, a validation or dev (development) set, and a
-test set. The test set is the data on which we want to make
-predictions. The dev set is a subset of the training data we use to
-check how well we are doing out-of-sample, after training the model on
-the training dataset. We use the validation error as a proxy for the
-test error in order to make tweaks to our model. It is crucial that we
-do not use any of the test data to train the algorithm. This is a
-cardinal sin in ML. Then:
-
- 
-* Estimate optimal error rate
-
-* Minimize underfitting (bias) on training data set.
-
-* Make sure you are not overfitting.
-
-If the validation and test sets are drawn from the same distributions,
-then a good performance on the validation set should lead to similarly
-good performance on the test set. 
-
-However, sometimes
-the training data and test data differ in subtle ways because, for
-example, they are collected using slightly different methods, or
-because it is cheaper to collect data in one way versus another. In
-this case, there can be a mismatch between the training and test
-data. This can lead to the neural network overfitting these small
-differences between the test and training sets, and a poor performance
-on the test set despite having a good performance on the validation
-set. To rectify this, Andrew Ng suggests making two validation or dev
-sets, one constructed from the training data and one constructed from
-the test data. The difference between the performance of the algorithm
-on these two validation sets quantifies the train-test mismatch. This
-can serve as another important diagnostic when using DNNs for
-supervised learning.
-
-!split
-===== Limitations of supervised learning with deep networks =====
-
-Like all statistical methods, supervised learning using neural
-networks has important limitations. This is especially important when
-one seeks to apply these methods, especially to physics problems. Like
-all tools, DNNs are not a universal solution. Often, the same or
-better performance on a task can be achieved by using a few
-hand-engineered features (or even a collection of random
-features). 
-
-Here we list some of the important limitations of supervised neural network based models. 
-
-
-
-* _Need labeled data_. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).
-* _Supervised neural networks are extremely data intensive._ DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.
-* _Homogeneous data._ Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e.~some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.
-* _Many problems are not about prediction._ In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a *wrong* model. The model might or might not be useful for understanding the underlying science.
-
-Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.