Files
FYS-STK4155/doc/pub/week43/html/._week43-bs068.html
T
2020-10-26 08:08:50 +01:00

410 lines
26 KiB
HTML

<!--
Automatically generated HTML file from DocOnce source
(https://github.com/hplgit/doconce/)
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 43: Solving Differential Equations with Deep Learning and Dimensionality Reduction methods">
<title>Week 43: Solving Differential Equations with Deep Learning and Dimensionality Reduction methods</title>
<!-- Bootstrap style: bootstrap -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Plans for week 43', 2, None, '___sec0'),
('Recurrent Neural Networks', 2, None, '___sec1'),
('Solving ODEs with Deep Learning', 2, None, '___sec2'),
('Ordinary Differential Equations', 2, None, '___sec3'),
('The trial solution', 2, None, '___sec4'),
('Minimization process', 2, None, '___sec5'),
('Minimizing the cost function using gradient descent and '
'automatic differentiation',
2,
None,
'___sec6'),
('Example: Exponential decay', 2, None, '___sec7'),
('The function to solve for', 2, None, '___sec8'),
('The trial solution', 2, None, '___sec9'),
('Setup of Network', 2, None, '___sec10'),
('Reformulating the problem', 2, None, '___sec11'),
('More technicalities', 2, None, '___sec12'),
('More details', 2, None, '___sec13'),
('A possible implementation of a neural network',
2,
None,
'___sec14'),
('Technicalities', 2, None, '___sec15'),
('Final technicalities I', 2, None, '___sec16'),
('Final technicalities II', 2, None, '___sec17'),
('Final technicalities III', 2, None, '___sec18'),
('Final technicalities IV', 2, None, '___sec19'),
('Back propagation', 2, None, '___sec20'),
('Gradient descent', 2, None, '___sec21'),
('The code for solving the ODE', 2, None, '___sec22'),
('The network with one input layer, specified number of hidden '
'layers, and one output layer',
2,
None,
'___sec23'),
('Example: Population growth', 2, None, '___sec24'),
('Setting up the problem', 2, None, '___sec25'),
('The trial solution', 2, None, '___sec26'),
('The program using Autograd', 2, None, '___sec27'),
('Using forward Euler to solve the ODE', 2, None, '___sec28'),
('Example: Solving the one dimensional Poisson equation',
2,
None,
'___sec29'),
('The specific equation to solve for', 2, None, '___sec30'),
('Solving the equation using Autograd', 2, None, '___sec31'),
('Comparing with a numerical scheme', 2, None, '___sec32'),
('Setting up the code', 2, None, '___sec33'),
('Partial Differential Equations', 2, None, '___sec34'),
('Type of problem', 2, None, '___sec35'),
('Network requirements', 2, None, '___sec36'),
('More details', 2, None, '___sec37'),
('Example: The diffusion equation', 2, None, '___sec38'),
('Defining the problem', 2, None, '___sec39'),
('Setting up the network using Autograd', 2, None, '___sec40'),
('Setting up the network using Autograd; The trial solution',
2,
None,
'___sec41'),
('Why the jacobian?', 2, None, '___sec42'),
('Setting up the network using Autograd; The full program',
2,
None,
'___sec43'),
('Example: Solving the wave equation with Neural Networks',
2,
None,
'___sec44'),
('The problem to solve for', 2, None, '___sec45'),
('The trial solution', 2, None, '___sec46'),
('The analytical solution', 2, None, '___sec47'),
('Solving the wave equation - the full program using Autograd',
2,
None,
'___sec48'),
('Resources on differential equations and deep learning',
2,
None,
'___sec49'),
('Friday, Principal Component Analysis', 2, None, '___sec50'),
('Basic ideas of the Principal Component Analysis (PCA)',
2,
None,
'___sec51'),
('Introducing the Covariance and Correlation functions',
2,
None,
'___sec52'),
('Correlation Function and Design/Feature Matrix',
2,
None,
'___sec53'),
('Covariance Matrix Examples', 2, None, '___sec54'),
('Correlation Matrix', 2, None, '___sec55'),
('Correlation Matrix with Pandas', 2, None, '___sec56'),
('Correlation Matrix with Pandas and the Franke function',
2,
None,
'___sec57'),
('Rewriting the Covariance and/or Correlation Matrix',
2,
None,
'___sec58'),
('Towards the PCA theorem', 2, None, '___sec59'),
('The Algorithm before the Theorem', 2, None, '___sec60'),
('Writing our own PCA code', 2, None, '___sec61'),
('Compute the sample mean and center the data',
3,
None,
'___sec62'),
('Compute the sample covariance', 3, None, '___sec63'),
('Diagonalize the sample covariance matrix to obtain the '
'principal components',
3,
None,
'___sec64'),
('Classical PCA Theorem', 2, None, '___sec65'),
('Proof of the PCA Theorem', 2, None, '___sec66'),
('PCA Proof continued', 2, None, '___sec67'),
('The final step', 2, None, '___sec68'),
('Geometric Interpretation and link with Singular Value '
'Decomposition',
2,
None,
'___sec69'),
('Principal Component Analysis', 2, None, '___sec70'),
('PCA and scikit-learn', 2, None, '___sec71'),
('Back to the Cancer Data', 2, None, '___sec72'),
('More on the PCA', 2, None, '___sec73'),
('Incremental PCA', 2, None, '___sec74'),
('Randomized PCA', 2, None, '___sec75'),
('Kernel PCA', 2, None, '___sec76'),
('LLE', 2, None, '___sec77'),
('Other techniques', 2, None, '___sec78')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week43-bs.html">Week 43: Solving Differential Equations with Deep Learning and Dimensionality Reduction methods</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week43-bs001.html#___sec0" style="font-size: 80%;"><b>Plans for week 43</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs002.html#___sec1" style="font-size: 80%;"><b>Recurrent Neural Networks</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs003.html#___sec2" style="font-size: 80%;"><b>Solving ODEs with Deep Learning</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs004.html#___sec3" style="font-size: 80%;"><b>Ordinary Differential Equations</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs005.html#___sec4" style="font-size: 80%;"><b>The trial solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs006.html#___sec5" style="font-size: 80%;"><b>Minimization process</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs007.html#___sec6" style="font-size: 80%;"><b>Minimizing the cost function using gradient descent and automatic differentiation</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs008.html#___sec7" style="font-size: 80%;"><b>Example: Exponential decay</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs009.html#___sec8" style="font-size: 80%;"><b>The function to solve for</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs010.html#___sec9" style="font-size: 80%;"><b>The trial solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs011.html#___sec10" style="font-size: 80%;"><b>Setup of Network</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs012.html#___sec11" style="font-size: 80%;"><b>Reformulating the problem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs013.html#___sec12" style="font-size: 80%;"><b>More technicalities</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs014.html#___sec13" style="font-size: 80%;"><b>More details</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs015.html#___sec14" style="font-size: 80%;"><b>A possible implementation of a neural network</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs016.html#___sec15" style="font-size: 80%;"><b>Technicalities</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs017.html#___sec16" style="font-size: 80%;"><b>Final technicalities I</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs018.html#___sec17" style="font-size: 80%;"><b>Final technicalities II</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs019.html#___sec18" style="font-size: 80%;"><b>Final technicalities III</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs020.html#___sec19" style="font-size: 80%;"><b>Final technicalities IV</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs021.html#___sec20" style="font-size: 80%;"><b>Back propagation</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs022.html#___sec21" style="font-size: 80%;"><b>Gradient descent</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs023.html#___sec22" style="font-size: 80%;"><b>The code for solving the ODE</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs024.html#___sec23" style="font-size: 80%;"><b>The network with one input layer, specified number of hidden layers, and one output layer</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs025.html#___sec24" style="font-size: 80%;"><b>Example: Population growth</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs026.html#___sec25" style="font-size: 80%;"><b>Setting up the problem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs027.html#___sec26" style="font-size: 80%;"><b>The trial solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs028.html#___sec27" style="font-size: 80%;"><b>The program using Autograd</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs029.html#___sec28" style="font-size: 80%;"><b>Using forward Euler to solve the ODE</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs030.html#___sec29" style="font-size: 80%;"><b>Example: Solving the one dimensional Poisson equation</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs031.html#___sec30" style="font-size: 80%;"><b>The specific equation to solve for</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs032.html#___sec31" style="font-size: 80%;"><b>Solving the equation using Autograd</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs033.html#___sec32" style="font-size: 80%;"><b>Comparing with a numerical scheme</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs034.html#___sec33" style="font-size: 80%;"><b>Setting up the code</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs035.html#___sec34" style="font-size: 80%;"><b>Partial Differential Equations</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs036.html#___sec35" style="font-size: 80%;"><b>Type of problem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs037.html#___sec36" style="font-size: 80%;"><b>Network requirements</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs038.html#___sec37" style="font-size: 80%;"><b>More details</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs039.html#___sec38" style="font-size: 80%;"><b>Example: The diffusion equation</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs040.html#___sec39" style="font-size: 80%;"><b>Defining the problem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs041.html#___sec40" style="font-size: 80%;"><b>Setting up the network using Autograd</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs042.html#___sec41" style="font-size: 80%;"><b>Setting up the network using Autograd; The trial solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs043.html#___sec42" style="font-size: 80%;"><b>Why the jacobian?</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs044.html#___sec43" style="font-size: 80%;"><b>Setting up the network using Autograd; The full program</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs045.html#___sec44" style="font-size: 80%;"><b>Example: Solving the wave equation with Neural Networks</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs046.html#___sec45" style="font-size: 80%;"><b>The problem to solve for</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs047.html#___sec46" style="font-size: 80%;"><b>The trial solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs048.html#___sec47" style="font-size: 80%;"><b>The analytical solution</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs049.html#___sec48" style="font-size: 80%;"><b>Solving the wave equation - the full program using Autograd</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs050.html#___sec49" style="font-size: 80%;"><b>Resources on differential equations and deep learning</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs051.html#___sec50" style="font-size: 80%;"><b>Friday, Principal Component Analysis</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs052.html#___sec51" style="font-size: 80%;"><b>Basic ideas of the Principal Component Analysis (PCA)</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs053.html#___sec52" style="font-size: 80%;"><b>Introducing the Covariance and Correlation functions</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs054.html#___sec53" style="font-size: 80%;"><b>Correlation Function and Design/Feature Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs055.html#___sec54" style="font-size: 80%;"><b>Covariance Matrix Examples</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs056.html#___sec55" style="font-size: 80%;"><b>Correlation Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs057.html#___sec56" style="font-size: 80%;"><b>Correlation Matrix with Pandas</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs058.html#___sec57" style="font-size: 80%;"><b>Correlation Matrix with Pandas and the Franke function</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs059.html#___sec58" style="font-size: 80%;"><b>Rewriting the Covariance and/or Correlation Matrix</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs060.html#___sec59" style="font-size: 80%;"><b>Towards the PCA theorem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs061.html#___sec60" style="font-size: 80%;"><b>The Algorithm before the Theorem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs062.html#___sec61" style="font-size: 80%;"><b>Writing our own PCA code</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs062.html#___sec62" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Compute the sample mean and center the data</a></li>
<!-- navigation toc: --> <li><a href="._week43-bs062.html#___sec63" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Compute the sample covariance</a></li>
<!-- navigation toc: --> <li><a href="._week43-bs062.html#___sec64" style="font-size: 80%;">&nbsp;&nbsp;&nbsp;Diagonalize the sample covariance matrix to obtain the principal components</a></li>
<!-- navigation toc: --> <li><a href="._week43-bs063.html#___sec65" style="font-size: 80%;"><b>Classical PCA Theorem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs064.html#___sec66" style="font-size: 80%;"><b>Proof of the PCA Theorem</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs065.html#___sec67" style="font-size: 80%;"><b>PCA Proof continued</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs066.html#___sec68" style="font-size: 80%;"><b>The final step</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs067.html#___sec69" style="font-size: 80%;"><b>Geometric Interpretation and link with Singular Value Decomposition</b></a></li>
<!-- navigation toc: --> <li><a href="#___sec70" style="font-size: 80%;"><b>Principal Component Analysis</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs069.html#___sec71" style="font-size: 80%;"><b>PCA and scikit-learn</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs070.html#___sec72" style="font-size: 80%;"><b>Back to the Cancer Data</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs071.html#___sec73" style="font-size: 80%;"><b>More on the PCA</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs072.html#___sec74" style="font-size: 80%;"><b>Incremental PCA</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs073.html#___sec75" style="font-size: 80%;"><b>Randomized PCA</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs074.html#___sec76" style="font-size: 80%;"><b>Kernel PCA</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs075.html#___sec77" style="font-size: 80%;"><b>LLE</b></a></li>
<!-- navigation toc: --> <li><a href="._week43-bs076.html#___sec78" style="font-size: 80%;"><b>Other techniques</b></a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0068"></a>
<!-- !split -->
<h2 id="___sec70" class="anchor">Principal Component Analysis </h2>
<p>
Principal Component Analysis (PCA) is by far the most popular dimensionality reduction algorithm.
First it identifies the hyperplane that lies closest to the data, and then it projects the data onto it.
<p>
The following Python code uses NumPy&#8217;s <b>svd()</b> function to obtain all the principal components of the
training set, then extracts the first two principal components. First we center the data using either <b>pandas</b> or our own code
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> display
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">100</span>)
<span style="color: #408080; font-style: italic"># setting up a 10 x 5 vanilla matrix </span>
rows <span style="color: #666666">=</span> <span style="color: #666666">10</span>
cols <span style="color: #666666">=</span> <span style="color: #666666">5</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(rows,cols)
df <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(X)
<span style="color: #408080; font-style: italic"># Pandas does the centering for us</span>
df <span style="color: #666666">=</span> df <span style="color: #666666">-</span>df<span style="color: #666666">.</span>mean()
display(df)
<span style="color: #408080; font-style: italic"># we center it ourselves</span>
X_centered <span style="color: #666666">=</span> X <span style="color: #666666">-</span> X<span style="color: #666666">.</span>mean(axis<span style="color: #666666">=0</span>)
<span style="color: #408080; font-style: italic"># Then check the difference between pandas and our own set up</span>
<span style="color: #008000">print</span>(X_centered<span style="color: #666666">-</span>df)
<span style="color: #408080; font-style: italic">#Now we do an SVD</span>
U, s, V <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>svd(X_centered)
c1 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, <span style="color: #666666">0</span>]
c2 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, <span style="color: #666666">1</span>]
W2 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, :<span style="color: #666666">2</span>]
X2D <span style="color: #666666">=</span> X_centered<span style="color: #666666">.</span>dot(W2)
<span style="color: #008000">print</span>(X2D)
</pre></div>
<p>
PCA assumes that the dataset is centered around the origin. Scikit-Learn&#8217;s PCA classes take care of centering
the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don&#8217;t
forget to center the data first.
<p>
Once you have identified all the principal components, you can reduce the dimensionality of the dataset
down to \( d \) dimensions by projecting it onto the hyperplane defined by the first \( d \) principal components.
Selecting this hyperplane ensures that the projection will preserve as much variance as possible.
<p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span>W2 <span style="color: #666666">=</span> V<span style="color: #666666">.</span>T[:, :<span style="color: #666666">2</span>]
X2D <span style="color: #666666">=</span> X_centered<span style="color: #666666">.</span>dot(W2)
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week43-bs067.html">&laquo;</a></li>
<li><a href="._week43-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week43-bs060.html">61</a></li>
<li><a href="._week43-bs061.html">62</a></li>
<li><a href="._week43-bs062.html">63</a></li>
<li><a href="._week43-bs063.html">64</a></li>
<li><a href="._week43-bs064.html">65</a></li>
<li><a href="._week43-bs065.html">66</a></li>
<li><a href="._week43-bs066.html">67</a></li>
<li><a href="._week43-bs067.html">68</a></li>
<li class="active"><a href="._week43-bs068.html">69</a></li>
<li><a href="._week43-bs069.html">70</a></li>
<li><a href="._week43-bs070.html">71</a></li>
<li><a href="._week43-bs071.html">72</a></li>
<li><a href="._week43-bs072.html">73</a></li>
<li><a href="._week43-bs073.html">74</a></li>
<li><a href="._week43-bs074.html">75</a></li>
<li><a href="._week43-bs075.html">76</a></li>
<li><a href="._week43-bs076.html">77</a></li>
<li><a href="._week43-bs069.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="http://..."><img width="250" align=right src="http://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>