Morten Hjorth-Jensen
696 lines
48 KiB
HTML
696 lines
48 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks">
|
|
<title>Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week41.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week41-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 41', 2, None, 'plan-for-week-41'),
|
|
('Videos on Neural Networks',
|
|
2,
|
|
None,
|
|
'videos-on-neural-networks'),
|
|
('Review of the back propagation algorithm',
|
|
2,
|
|
None,
|
|
'review-of-the-back-propagation-algorithm'),
|
|
('Setting up the Back propagation algorithm',
|
|
2,
|
|
None,
|
|
'setting-up-the-back-propagation-algorithm'),
|
|
('Setting up a Multi-layer perceptron model for classification',
|
|
2,
|
|
None,
|
|
'setting-up-a-multi-layer-perceptron-model-for-classification'),
|
|
('Defining the cost function',
|
|
2,
|
|
None,
|
|
'defining-the-cost-function'),
|
|
('Example: binary classification problem',
|
|
2,
|
|
None,
|
|
'example-binary-classification-problem'),
|
|
('The Softmax function', 2, None, 'the-softmax-function'),
|
|
('Developing a code for doing neural networks with back '
|
|
'propagation',
|
|
2,
|
|
None,
|
|
'developing-a-code-for-doing-neural-networks-with-back-propagation'),
|
|
('Collect and pre-process data',
|
|
2,
|
|
None,
|
|
'collect-and-pre-process-data'),
|
|
('Train and test datasets', 2, None, 'train-and-test-datasets'),
|
|
('Define model and architecture',
|
|
2,
|
|
None,
|
|
'define-model-and-architecture'),
|
|
('Layers', 2, None, 'layers'),
|
|
('Weights and biases', 2, None, 'weights-and-biases'),
|
|
('Feed-forward pass', 2, None, 'feed-forward-pass'),
|
|
('Matrix multiplications', 2, None, 'matrix-multiplications'),
|
|
('Choose cost function and optimizer',
|
|
2,
|
|
None,
|
|
'choose-cost-function-and-optimizer'),
|
|
('Optimizing the cost function',
|
|
2,
|
|
None,
|
|
'optimizing-the-cost-function'),
|
|
('Regularization', 2, None, 'regularization'),
|
|
('Matrix multiplication', 2, None, 'matrix-multiplication'),
|
|
('Improving performance', 2, None, 'improving-performance'),
|
|
('Full object-oriented implementation',
|
|
2,
|
|
None,
|
|
'full-object-oriented-implementation'),
|
|
('Evaluate model performance on test data',
|
|
2,
|
|
None,
|
|
'evaluate-model-performance-on-test-data'),
|
|
('Adjust hyperparameters', 2, None, 'adjust-hyperparameters'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('scikit-learn implementation',
|
|
2,
|
|
None,
|
|
'scikit-learn-implementation'),
|
|
('Visualization', 2, None, 'visualization'),
|
|
('Testing our code for the XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'testing-our-code-for-the-xor-or-and-and-gates'),
|
|
('The AND and XOR Gates', 2, None, 'the-and-and-xor-gates'),
|
|
('Representing the Data Sets',
|
|
2,
|
|
None,
|
|
'representing-the-data-sets'),
|
|
('Setting up the Neural Network',
|
|
2,
|
|
None,
|
|
'setting-up-the-neural-network'),
|
|
('The Code using Scikit-Learn',
|
|
2,
|
|
None,
|
|
'the-code-using-scikit-learn'),
|
|
('Building neural networks in Tensorflow and Keras',
|
|
2,
|
|
None,
|
|
'building-neural-networks-in-tensorflow-and-keras'),
|
|
('Tensorflow', 2, None, 'tensorflow'),
|
|
('Using Keras', 2, None, 'using-keras'),
|
|
('Collect and pre-process data',
|
|
2,
|
|
None,
|
|
'collect-and-pre-process-data'),
|
|
('The Breast Cancer Data, now with Keras',
|
|
2,
|
|
None,
|
|
'the-breast-cancer-data-now-with-keras'),
|
|
('The Mathematics of Neural Networks',
|
|
2,
|
|
None,
|
|
'the-mathematics-of-neural-networks'),
|
|
('Fine-tuning neural network hyperparameters',
|
|
2,
|
|
None,
|
|
'fine-tuning-neural-network-hyperparameters'),
|
|
('Hidden layers', 2, None, 'hidden-layers'),
|
|
('Which activation function should I use?',
|
|
2,
|
|
None,
|
|
'which-activation-function-should-i-use'),
|
|
('Is the Logistic activation function (Sigmoid) our choice?',
|
|
2,
|
|
None,
|
|
'is-the-logistic-activation-function-sigmoid-our-choice'),
|
|
('The derivative of the Logistic funtion',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-logistic-funtion'),
|
|
('The RELU function family', 2, None, 'the-relu-function-family'),
|
|
('Which activation function should we use?',
|
|
2,
|
|
None,
|
|
'which-activation-function-should-we-use'),
|
|
('More on activation functions, output layers',
|
|
2,
|
|
None,
|
|
'more-on-activation-functions-output-layers'),
|
|
('Batch Normalization', 2, None, 'batch-normalization'),
|
|
('Dropout', 2, None, 'dropout'),
|
|
('Gradient Clipping', 2, None, 'gradient-clipping'),
|
|
('A very nice website on Neural Networks',
|
|
2,
|
|
None,
|
|
'a-very-nice-website-on-neural-networks'),
|
|
('A top-down perspective on Neural networks',
|
|
2,
|
|
None,
|
|
'a-top-down-perspective-on-neural-networks'),
|
|
('Limitations of supervised learning with deep networks',
|
|
2,
|
|
None,
|
|
'limitations-of-supervised-learning-with-deep-networks'),
|
|
('Overarching Views, a personal note',
|
|
2,
|
|
None,
|
|
'overarching-views-a-personal-note'),
|
|
('Using Automatic differentiation',
|
|
2,
|
|
None,
|
|
'using-automatic-differentiation'),
|
|
('Solving ODEs with Deep Learning',
|
|
2,
|
|
None,
|
|
'solving-odes-with-deep-learning'),
|
|
('Ordinary Differential Equations',
|
|
2,
|
|
None,
|
|
'ordinary-differential-equations'),
|
|
('The trial solution', 2, None, 'the-trial-solution'),
|
|
('Minimization process', 2, None, 'minimization-process'),
|
|
('Minimizing the cost function using gradient descent and '
|
|
'automatic differentiation',
|
|
2,
|
|
None,
|
|
'minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation'),
|
|
('Example: Exponential decay',
|
|
2,
|
|
None,
|
|
'example-exponential-decay'),
|
|
('The function to solve for',
|
|
2,
|
|
None,
|
|
'the-function-to-solve-for'),
|
|
('The trial solution', 2, None, 'the-trial-solution'),
|
|
('Setup of Network', 2, None, 'setup-of-network'),
|
|
('Reformulating the problem',
|
|
2,
|
|
None,
|
|
'reformulating-the-problem'),
|
|
('More technicalities', 2, None, 'more-technicalities'),
|
|
('More details', 2, None, 'more-details'),
|
|
('A possible implementation of a neural network',
|
|
2,
|
|
None,
|
|
'a-possible-implementation-of-a-neural-network'),
|
|
('Technicalities', 2, None, 'technicalities'),
|
|
('Final technicalities I', 2, None, 'final-technicalities-i'),
|
|
('Final technicalities II', 2, None, 'final-technicalities-ii'),
|
|
('Final technicalities III', 2, None, 'final-technicalities-iii'),
|
|
('Final technicalities IV', 2, None, 'final-technicalities-iv'),
|
|
('Back propagation', 2, None, 'back-propagation'),
|
|
('Gradient descent', 2, None, 'gradient-descent'),
|
|
('The code for solving the ODE',
|
|
2,
|
|
None,
|
|
'the-code-for-solving-the-ode'),
|
|
('The network with one input layer, specified number of hidden '
|
|
'layers, and one output layer',
|
|
2,
|
|
None,
|
|
'the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer'),
|
|
('Example: Population growth',
|
|
2,
|
|
None,
|
|
'example-population-growth'),
|
|
('Setting up the problem', 2, None, 'setting-up-the-problem'),
|
|
('The trial solution', 2, None, 'the-trial-solution'),
|
|
('The program using Autograd',
|
|
2,
|
|
None,
|
|
'the-program-using-autograd'),
|
|
('Using forward Euler to solve the ODE',
|
|
2,
|
|
None,
|
|
'using-forward-euler-to-solve-the-ode'),
|
|
('Example: Solving the one dimensional Poisson equation',
|
|
2,
|
|
None,
|
|
'example-solving-the-one-dimensional-poisson-equation'),
|
|
('The specific equation to solve for',
|
|
2,
|
|
None,
|
|
'the-specific-equation-to-solve-for'),
|
|
('Solving the equation using Autograd',
|
|
2,
|
|
None,
|
|
'solving-the-equation-using-autograd'),
|
|
('Comparing with a numerical scheme',
|
|
2,
|
|
None,
|
|
'comparing-with-a-numerical-scheme'),
|
|
('Setting up the code', 2, None, 'setting-up-the-code'),
|
|
('Partial Differential Equations',
|
|
2,
|
|
None,
|
|
'partial-differential-equations'),
|
|
('Type of problem', 2, None, 'type-of-problem'),
|
|
('Network requirements', 2, None, 'network-requirements'),
|
|
('More details', 2, None, 'more-details'),
|
|
('Example: The diffusion equation',
|
|
2,
|
|
None,
|
|
'example-the-diffusion-equation'),
|
|
('Defining the problem', 2, None, 'defining-the-problem'),
|
|
('Setting up the network using Autograd',
|
|
2,
|
|
None,
|
|
'setting-up-the-network-using-autograd'),
|
|
('Setting up the network using Autograd; The trial solution',
|
|
2,
|
|
None,
|
|
'setting-up-the-network-using-autograd-the-trial-solution'),
|
|
('Why the jacobian?', 2, None, 'why-the-jacobian'),
|
|
('Setting up the network using Autograd; The full program',
|
|
2,
|
|
None,
|
|
'setting-up-the-network-using-autograd-the-full-program'),
|
|
('Example: Solving the wave equation with Neural Networks',
|
|
2,
|
|
None,
|
|
'example-solving-the-wave-equation-with-neural-networks'),
|
|
('The problem to solve for', 2, None, 'the-problem-to-solve-for'),
|
|
('The trial solution', 2, None, 'the-trial-solution'),
|
|
('The analytical solution', 2, None, 'the-analytical-solution'),
|
|
('Solving the wave equation - the full program using Autograd',
|
|
2,
|
|
None,
|
|
'solving-the-wave-equation-the-full-program-using-autograd'),
|
|
('Resources on differential equations and deep learning',
|
|
2,
|
|
None,
|
|
'resources-on-differential-equations-and-deep-learning')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week41-bs.html">Week 41 Constructing a Neural Network code, Tensor flow and start Convolutional Neural Networks</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week41-bs001.html#plan-for-week-41" style="font-size: 80%;">Plan for week 41</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs002.html#videos-on-neural-networks" style="font-size: 80%;">Videos on Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs003.html#review-of-the-back-propagation-algorithm" style="font-size: 80%;">Review of the back propagation algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs004.html#setting-up-the-back-propagation-algorithm" style="font-size: 80%;">Setting up the Back propagation algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs005.html#setting-up-a-multi-layer-perceptron-model-for-classification" style="font-size: 80%;">Setting up a Multi-layer perceptron model for classification</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs006.html#defining-the-cost-function" style="font-size: 80%;">Defining the cost function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs007.html#example-binary-classification-problem" style="font-size: 80%;">Example: binary classification problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs008.html#the-softmax-function" style="font-size: 80%;">The Softmax function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs009.html#developing-a-code-for-doing-neural-networks-with-back-propagation" style="font-size: 80%;">Developing a code for doing neural networks with back propagation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs011.html#train-and-test-datasets" style="font-size: 80%;">Train and test datasets</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs012.html#define-model-and-architecture" style="font-size: 80%;">Define model and architecture</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs013.html#layers" style="font-size: 80%;">Layers</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs014.html#weights-and-biases" style="font-size: 80%;">Weights and biases</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs015.html#feed-forward-pass" style="font-size: 80%;">Feed-forward pass</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs016.html#matrix-multiplications" style="font-size: 80%;">Matrix multiplications</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs017.html#choose-cost-function-and-optimizer" style="font-size: 80%;">Choose cost function and optimizer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs018.html#optimizing-the-cost-function" style="font-size: 80%;">Optimizing the cost function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs019.html#regularization" style="font-size: 80%;">Regularization</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs020.html#matrix-multiplication" style="font-size: 80%;">Matrix multiplication</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs021.html#improving-performance" style="font-size: 80%;">Improving performance</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs022.html#full-object-oriented-implementation" style="font-size: 80%;">Full object-oriented implementation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs023.html#evaluate-model-performance-on-test-data" style="font-size: 80%;">Evaluate model performance on test data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs024.html#adjust-hyperparameters" style="font-size: 80%;">Adjust hyperparameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs026.html#scikit-learn-implementation" style="font-size: 80%;">scikit-learn implementation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs027.html#visualization" style="font-size: 80%;">Visualization</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs028.html#testing-our-code-for-the-xor-or-and-and-gates" style="font-size: 80%;">Testing our code for the XOR, OR and AND gates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs029.html#the-and-and-xor-gates" style="font-size: 80%;">The AND and XOR Gates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs030.html#representing-the-data-sets" style="font-size: 80%;">Representing the Data Sets</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs031.html#setting-up-the-neural-network" style="font-size: 80%;">Setting up the Neural Network</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs032.html#the-code-using-scikit-learn" style="font-size: 80%;">The Code using Scikit-Learn</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs033.html#building-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">Building neural networks in Tensorflow and Keras</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs034.html#tensorflow" style="font-size: 80%;">Tensorflow</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs035.html#using-keras" style="font-size: 80%;">Using Keras</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs036.html#collect-and-pre-process-data" style="font-size: 80%;">Collect and pre-process data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs037.html#the-breast-cancer-data-now-with-keras" style="font-size: 80%;">The Breast Cancer Data, now with Keras</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs038.html#the-mathematics-of-neural-networks" style="font-size: 80%;">The Mathematics of Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs039.html#fine-tuning-neural-network-hyperparameters" style="font-size: 80%;">Fine-tuning neural network hyperparameters</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs040.html#hidden-layers" style="font-size: 80%;">Hidden layers</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs041.html#which-activation-function-should-i-use" style="font-size: 80%;">Which activation function should I use?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs042.html#is-the-logistic-activation-function-sigmoid-our-choice" style="font-size: 80%;">Is the Logistic activation function (Sigmoid) our choice?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs043.html#the-derivative-of-the-logistic-funtion" style="font-size: 80%;">The derivative of the Logistic funtion</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs044.html#the-relu-function-family" style="font-size: 80%;">The RELU function family</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs045.html#which-activation-function-should-we-use" style="font-size: 80%;">Which activation function should we use?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs046.html#more-on-activation-functions-output-layers" style="font-size: 80%;">More on activation functions, output layers</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs047.html#batch-normalization" style="font-size: 80%;">Batch Normalization</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs048.html#dropout" style="font-size: 80%;">Dropout</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs049.html#gradient-clipping" style="font-size: 80%;">Gradient Clipping</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs050.html#a-very-nice-website-on-neural-networks" style="font-size: 80%;">A very nice website on Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs051.html#a-top-down-perspective-on-neural-networks" style="font-size: 80%;">A top-down perspective on Neural networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs052.html#limitations-of-supervised-learning-with-deep-networks" style="font-size: 80%;">Limitations of supervised learning with deep networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs053.html#overarching-views-a-personal-note" style="font-size: 80%;">Overarching Views, a personal note</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs054.html#using-automatic-differentiation" style="font-size: 80%;">Using Automatic differentiation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs055.html#solving-odes-with-deep-learning" style="font-size: 80%;">Solving ODEs with Deep Learning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs056.html#ordinary-differential-equations" style="font-size: 80%;">Ordinary Differential Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs058.html#minimization-process" style="font-size: 80%;">Minimization process</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs059.html#minimizing-the-cost-function-using-gradient-descent-and-automatic-differentiation" style="font-size: 80%;">Minimizing the cost function using gradient descent and automatic differentiation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs060.html#example-exponential-decay" style="font-size: 80%;">Example: Exponential decay</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs061.html#the-function-to-solve-for" style="font-size: 80%;">The function to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs063.html#setup-of-network" style="font-size: 80%;">Setup of Network</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs064.html#reformulating-the-problem" style="font-size: 80%;">Reformulating the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs065.html#more-technicalities" style="font-size: 80%;">More technicalities</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs067.html#a-possible-implementation-of-a-neural-network" style="font-size: 80%;">A possible implementation of a neural network</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs068.html#technicalities" style="font-size: 80%;">Technicalities</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs069.html#final-technicalities-i" style="font-size: 80%;">Final technicalities I</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs070.html#final-technicalities-ii" style="font-size: 80%;">Final technicalities II</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs071.html#final-technicalities-iii" style="font-size: 80%;">Final technicalities III</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs072.html#final-technicalities-iv" style="font-size: 80%;">Final technicalities IV</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs073.html#back-propagation" style="font-size: 80%;">Back propagation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs074.html#gradient-descent" style="font-size: 80%;">Gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs075.html#the-code-for-solving-the-ode" style="font-size: 80%;">The code for solving the ODE</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs076.html#the-network-with-one-input-layer-specified-number-of-hidden-layers-and-one-output-layer" style="font-size: 80%;">The network with one input layer, specified number of hidden layers, and one output layer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs077.html#example-population-growth" style="font-size: 80%;">Example: Population growth</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs078.html#setting-up-the-problem" style="font-size: 80%;">Setting up the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="#the-program-using-autograd" style="font-size: 80%;">The program using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs081.html#using-forward-euler-to-solve-the-ode" style="font-size: 80%;">Using forward Euler to solve the ODE</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs082.html#example-solving-the-one-dimensional-poisson-equation" style="font-size: 80%;">Example: Solving the one dimensional Poisson equation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs083.html#the-specific-equation-to-solve-for" style="font-size: 80%;">The specific equation to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs084.html#solving-the-equation-using-autograd" style="font-size: 80%;">Solving the equation using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs085.html#comparing-with-a-numerical-scheme" style="font-size: 80%;">Comparing with a numerical scheme</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs086.html#setting-up-the-code" style="font-size: 80%;">Setting up the code</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs087.html#partial-differential-equations" style="font-size: 80%;">Partial Differential Equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs088.html#type-of-problem" style="font-size: 80%;">Type of problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs089.html#network-requirements" style="font-size: 80%;">Network requirements</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs090.html#more-details" style="font-size: 80%;">More details</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs091.html#example-the-diffusion-equation" style="font-size: 80%;">Example: The diffusion equation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs092.html#defining-the-problem" style="font-size: 80%;">Defining the problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs093.html#setting-up-the-network-using-autograd" style="font-size: 80%;">Setting up the network using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs094.html#setting-up-the-network-using-autograd-the-trial-solution" style="font-size: 80%;">Setting up the network using Autograd; The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs095.html#why-the-jacobian" style="font-size: 80%;">Why the jacobian?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs096.html#setting-up-the-network-using-autograd-the-full-program" style="font-size: 80%;">Setting up the network using Autograd; The full program</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs097.html#example-solving-the-wave-equation-with-neural-networks" style="font-size: 80%;">Example: Solving the wave equation with Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs098.html#the-problem-to-solve-for" style="font-size: 80%;">The problem to solve for</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs099.html#the-trial-solution" style="font-size: 80%;">The trial solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs100.html#the-analytical-solution" style="font-size: 80%;">The analytical solution</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs101.html#solving-the-wave-equation-the-full-program-using-autograd" style="font-size: 80%;">Solving the wave equation - the full program using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week41-bs102.html#resources-on-differential-equations-and-deep-learning" style="font-size: 80%;">Resources on differential equations and deep learning</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0080"></a>
|
|
<!-- !split -->
|
|
<h2 id="the-program-using-autograd" class="anchor">The program using Autograd </h2>
|
|
|
|
<p>The network will be the similar as for the exponential decay example, but with some small modifications for our problem.</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad, elementwise_grad
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy.random</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">npr</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> pyplot <span style="color: #008000; font-weight: bold">as</span> plt
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(z):
|
|
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1/</span>(<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>z))
|
|
|
|
<span style="color: #408080; font-style: italic"># Function to get the parameters.</span>
|
|
<span style="color: #408080; font-style: italic"># Done such that one can easily change the paramaters after one's liking.</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_parameters</span>():
|
|
alpha <span style="color: #666666">=</span> <span style="color: #666666">2</span>
|
|
A <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
g0 <span style="color: #666666">=</span> <span style="color: #666666">1.2</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> alpha, A, g0
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">deep_neural_network</span>(P, x):
|
|
<span style="color: #408080; font-style: italic"># N_hidden is the number of hidden layers</span>
|
|
N_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(P) <span style="color: #666666">-</span> <span style="color: #666666">1</span> <span style="color: #408080; font-style: italic"># -1 since params consist of parameters to all the hidden layers AND the output layer</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Assumes input x being an one-dimensional array</span>
|
|
num_values <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(x)
|
|
x <span style="color: #666666">=</span> x<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, num_values)
|
|
|
|
<span style="color: #408080; font-style: italic"># Assume that the input layer does nothing to the input x</span>
|
|
x_input <span style="color: #666666">=</span> x
|
|
|
|
<span style="color: #408080; font-style: italic"># Due to multiple hidden layers, define a variable referencing to the</span>
|
|
<span style="color: #408080; font-style: italic"># output of the previous layer:</span>
|
|
x_prev <span style="color: #666666">=</span> x_input
|
|
|
|
<span style="color: #408080; font-style: italic">## Hidden layers:</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(N_hidden):
|
|
<span style="color: #408080; font-style: italic"># From the list of parameters P; find the correct weigths and bias for this layer</span>
|
|
w_hidden <span style="color: #666666">=</span> P[l]
|
|
|
|
<span style="color: #408080; font-style: italic"># Add a row of ones to include bias</span>
|
|
x_prev <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_prev ), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
|
|
|
z_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_hidden, x_prev)
|
|
x_hidden <span style="color: #666666">=</span> sigmoid(z_hidden)
|
|
|
|
<span style="color: #408080; font-style: italic"># Update x_prev such that next layer can use the output from this layer</span>
|
|
x_prev <span style="color: #666666">=</span> x_hidden
|
|
|
|
<span style="color: #408080; font-style: italic">## Output layer:</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Get the weights and bias for this layer</span>
|
|
w_output <span style="color: #666666">=</span> P[<span style="color: #666666">-1</span>]
|
|
|
|
<span style="color: #408080; font-style: italic"># Include bias:</span>
|
|
x_prev <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((np<span style="color: #666666">.</span>ones((<span style="color: #666666">1</span>,num_values)), x_prev), axis <span style="color: #666666">=</span> <span style="color: #666666">0</span>)
|
|
|
|
z_output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>matmul(w_output, x_prev)
|
|
x_output <span style="color: #666666">=</span> z_output
|
|
|
|
<span style="color: #008000; font-weight: bold">return</span> x_output
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">cost_function_deep</span>(P, x):
|
|
|
|
<span style="color: #408080; font-style: italic"># Evaluate the trial function with the current parameters P</span>
|
|
g_t <span style="color: #666666">=</span> g_trial_deep(x,P)
|
|
|
|
<span style="color: #408080; font-style: italic"># Find the derivative w.r.t x of the trial function</span>
|
|
d_g_t <span style="color: #666666">=</span> elementwise_grad(g_trial_deep,<span style="color: #666666">0</span>)(x,P)
|
|
|
|
<span style="color: #408080; font-style: italic"># The right side of the ODE</span>
|
|
func <span style="color: #666666">=</span> f(x, g_t)
|
|
|
|
err_sqr <span style="color: #666666">=</span> (d_g_t <span style="color: #666666">-</span> func)<span style="color: #666666">**2</span>
|
|
cost_sum <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(err_sqr)
|
|
|
|
<span style="color: #008000; font-weight: bold">return</span> cost_sum <span style="color: #666666">/</span> np<span style="color: #666666">.</span>size(err_sqr)
|
|
|
|
<span style="color: #408080; font-style: italic"># The right side of the ODE:</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">f</span>(x, g_trial):
|
|
alpha,A, g0 <span style="color: #666666">=</span> get_parameters()
|
|
<span style="color: #008000; font-weight: bold">return</span> alpha<span style="color: #666666">*</span>g_trial<span style="color: #666666">*</span>(A <span style="color: #666666">-</span> g_trial)
|
|
|
|
<span style="color: #408080; font-style: italic"># The trial solution using the deep neural network:</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g_trial_deep</span>(x, params):
|
|
alpha,A, g0 <span style="color: #666666">=</span> get_parameters()
|
|
<span style="color: #008000; font-weight: bold">return</span> g0 <span style="color: #666666">+</span> x<span style="color: #666666">*</span>deep_neural_network(params,x)
|
|
|
|
<span style="color: #408080; font-style: italic"># The analytical solution:</span>
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">g_analytic</span>(t):
|
|
alpha,A, g0 <span style="color: #666666">=</span> get_parameters()
|
|
<span style="color: #008000; font-weight: bold">return</span> A<span style="color: #666666">*</span>g0<span style="color: #666666">/</span>(g0 <span style="color: #666666">+</span> (A <span style="color: #666666">-</span> g0)<span style="color: #666666">*</span>np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>alpha<span style="color: #666666">*</span>A<span style="color: #666666">*</span>t))
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">solve_ode_deep_neural_network</span>(x, num_neurons, num_iter, lmb):
|
|
<span style="color: #408080; font-style: italic"># num_hidden_neurons is now a list of number of neurons within each hidden layer</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Find the number of hidden layers:</span>
|
|
N_hidden <span style="color: #666666">=</span> np<span style="color: #666666">.</span>size(num_neurons)
|
|
|
|
<span style="color: #408080; font-style: italic">## Set up initial weigths and biases</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Initialize the list of parameters:</span>
|
|
P <span style="color: #666666">=</span> [<span style="color: #008000; font-weight: bold">None</span>]<span style="color: #666666">*</span>(N_hidden <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #408080; font-style: italic"># + 1 to include the output layer</span>
|
|
|
|
P[<span style="color: #666666">0</span>] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(num_neurons[<span style="color: #666666">0</span>], <span style="color: #666666">2</span> )
|
|
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,N_hidden):
|
|
P[l] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(num_neurons[l], num_neurons[l<span style="color: #666666">-1</span>] <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #408080; font-style: italic"># +1 to include bias</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># For the output layer</span>
|
|
P[<span style="color: #666666">-1</span>] <span style="color: #666666">=</span> npr<span style="color: #666666">.</span>randn(<span style="color: #666666">1</span>, num_neurons[<span style="color: #666666">-1</span>] <span style="color: #666666">+</span> <span style="color: #666666">1</span> ) <span style="color: #408080; font-style: italic"># +1 since bias is included</span>
|
|
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Initial cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function_deep(P, x))
|
|
|
|
<span style="color: #408080; font-style: italic">## Start finding the optimal weigths using gradient descent</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Find the Python function that represents the gradient of the cost function</span>
|
|
<span style="color: #408080; font-style: italic"># w.r.t the 0-th input argument -- that is the weights and biases in the hidden and output layer</span>
|
|
cost_function_deep_grad <span style="color: #666666">=</span> grad(cost_function_deep,<span style="color: #666666">0</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># Let the update be done num_iter times</span>
|
|
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(num_iter):
|
|
<span style="color: #408080; font-style: italic"># Evaluate the gradient at the current weights and biases in P.</span>
|
|
<span style="color: #408080; font-style: italic"># The cost_grad consist now of N_hidden + 1 arrays; the gradient w.r.t the weights and biases</span>
|
|
<span style="color: #408080; font-style: italic"># in the hidden layers and output layers evaluated at x.</span>
|
|
cost_deep_grad <span style="color: #666666">=</span> cost_function_deep_grad(P, x)
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> l <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(N_hidden<span style="color: #666666">+1</span>):
|
|
P[l] <span style="color: #666666">=</span> P[l] <span style="color: #666666">-</span> lmb <span style="color: #666666">*</span> cost_deep_grad[l]
|
|
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Final cost: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">'</span><span style="color: #666666">%</span>cost_function_deep(P, x))
|
|
|
|
<span style="color: #008000; font-weight: bold">return</span> P
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">'__main__'</span>:
|
|
npr<span style="color: #666666">.</span>seed(<span style="color: #666666">4155</span>)
|
|
|
|
<span style="color: #408080; font-style: italic">## Decide the vales of arguments to the function to solve</span>
|
|
Nt <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
|
T <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
t <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>,T, Nt)
|
|
|
|
<span style="color: #408080; font-style: italic">## Set up the initial parameters</span>
|
|
num_hidden_neurons <span style="color: #666666">=</span> [<span style="color: #666666">100</span>, <span style="color: #666666">50</span>, <span style="color: #666666">25</span>]
|
|
num_iter <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
|
lmb <span style="color: #666666">=</span> <span style="color: #666666">1e-3</span>
|
|
|
|
P <span style="color: #666666">=</span> solve_ode_deep_neural_network(t, num_hidden_neurons, num_iter, lmb)
|
|
|
|
g_dnn_ag <span style="color: #666666">=</span> g_trial_deep(t,P)
|
|
g_analytical <span style="color: #666666">=</span> g_analytic(t)
|
|
|
|
<span style="color: #408080; font-style: italic"># Find the maximum absolute difference between the solutons:</span>
|
|
diff_ag <span style="color: #666666">=</span> np<span style="color: #666666">.</span>max(np<span style="color: #666666">.</span>abs(g_dnn_ag <span style="color: #666666">-</span> g_analytical))
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"The max absolute difference between the solutions is: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">"</span><span style="color: #666666">%</span>diff_ag)
|
|
|
|
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>,<span style="color: #666666">10</span>))
|
|
|
|
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">'Performance of neural network solving an ODE compared to the analytical solution'</span>)
|
|
plt<span style="color: #666666">.</span>plot(t, g_analytical)
|
|
plt<span style="color: #666666">.</span>plot(t, g_dnn_ag[<span style="color: #666666">0</span>,:])
|
|
plt<span style="color: #666666">.</span>legend([<span style="color: #BA2121">'analytical'</span>,<span style="color: #BA2121">'nn'</span>])
|
|
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'t'</span>)
|
|
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'g(t)'</span>)
|
|
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week41-bs079.html">«</a></li>
|
|
<li><a href="._week41-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs072.html">73</a></li>
|
|
<li><a href="._week41-bs073.html">74</a></li>
|
|
<li><a href="._week41-bs074.html">75</a></li>
|
|
<li><a href="._week41-bs075.html">76</a></li>
|
|
<li><a href="._week41-bs076.html">77</a></li>
|
|
<li><a href="._week41-bs077.html">78</a></li>
|
|
<li><a href="._week41-bs078.html">79</a></li>
|
|
<li><a href="._week41-bs079.html">80</a></li>
|
|
<li class="active"><a href="._week41-bs080.html">81</a></li>
|
|
<li><a href="._week41-bs081.html">82</a></li>
|
|
<li><a href="._week41-bs082.html">83</a></li>
|
|
<li><a href="._week41-bs083.html">84</a></li>
|
|
<li><a href="._week41-bs084.html">85</a></li>
|
|
<li><a href="._week41-bs085.html">86</a></li>
|
|
<li><a href="._week41-bs086.html">87</a></li>
|
|
<li><a href="._week41-bs087.html">88</a></li>
|
|
<li><a href="._week41-bs088.html">89</a></li>
|
|
<li><a href="._week41-bs089.html">90</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week41-bs102.html">103</a></li>
|
|
<li><a href="._week41-bs081.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|