lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e7ec5ea971b9afd343204fb86027c1654e775a36
FYS-STK4155/doc/pub/week40/html/week40-reveal.html
T
Morten Hjorth-Jensen e7ec5ea971 update week 40
2025-09-28 10:58:16 +02:00

2432 lines
124 KiB
HTML
Raw Blame History

<!--
HTML file automatically generated from DocOnce source
(https://github.com/doconce/doconce/)
doconce format html week40-reveal.html week40-reveal reveal --html_slide_theme=beige
-->
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 40: Gradient descent methods (continued) and start Neural networks">
<title>Week 40: Gradient descent methods (continued) and start Neural networks</title>
<!-- reveal.js: https://lab.hakim.se/reveal-js/ -->
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
<link rel="stylesheet" href="reveal.js/css/reveal.css">
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
<!--
<link rel="stylesheet" href="reveal.js/css/reveal.css">
<link rel="stylesheet" href="reveal.js/css/theme/beige.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/beigesmall.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/solarized.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/serif.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/moon.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/sky.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/darkgray.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/default.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/cbc.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/simula.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/black.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/white.css" id="theme">
<link rel="stylesheet" href="reveal.js/css/theme/league.css" id="theme">
-->
<!-- For syntax highlighting -->
<link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
<!-- Printing and PDF exports -->
<script>
var link = document.createElement( 'link' );
link.rel = 'stylesheet';
link.type = 'text/css';
link.href = window.location.search.match( /print-pdf/gi ) ? 'css/print/pdf.css' : 'css/print/paper.css';
document.getElementsByTagName( 'head' )[0].appendChild( link );
</script>
<style type="text/css">
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
.reveal .alert-text-small { font-size: 80%; }
.reveal .alert-text-large { font-size: 130%; }
.reveal .alert-text-normal { font-size: 90%; }
.reveal .alert {
padding:8px 35px 8px 14px; margin-bottom:18px;
text-shadow:0 1px 0 rgba(255,255,255,0.5);
border:5px solid #bababa;
-webkit-border-radius: 14px; -moz-border-radius: 14px;
border-radius:14px;
background-position: 10px 10px;
background-repeat: no-repeat;
background-size: 38px;
padding-left: 30px; /* 55px; if icon */
}
.reveal .alert-block {padding-top:14px; padding-bottom:14px}
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
/*.reveal .alert li {margin-top: 1em}*/
.reveal .alert-block p+p {margin-top:5px}
/*.reveal .alert-notice { background-image: url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
.reveal .alert-summary { background-image:url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
.reveal .alert-warning { background-image: url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
.reveal .alert-question {background-image:url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
/* Override reveal.js table border */
.reveal table td {
border: 0;
}
<style type="text/css">
/* Override h1, h2, ... styles */
h1 { font-size: 2.8em; }
h2 { font-size: 1.5em; }
h3 { font-size: 1.4em; }
h4 { font-size: 1.3em; }
h1, h2, h3, h4 { font-weight: bold; line-height: 1.2; }
body { overflow: auto; } /* vertical scrolling */
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
p.caption { width: 80%; font-size: 60%; font-style: italic; text-align: left; }
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
.slide .alert-text-small { font-size: 80%; }
.slide .alert-text-large { font-size: 130%; }
.slide .alert-text-normal { font-size: 90%; }
.slide .alert {
padding:8px 35px 8px 14px; margin-bottom:18px;
text-shadow:0 1px 0 rgba(255,255,255,0.5);
border:5px solid #bababa;
-webkit-border-radius:14px; -moz-border-radius:14px;
border-radius:14px
background-position: 10px 10px;
background-repeat: no-repeat;
background-size: 38px;
padding-left: 30px; /* 55px; if icon */
}
.slide .alert-block {padding-top:14px; padding-bottom:14px}
.slide .alert-block > p, .alert-block > ul {margin-bottom:0}
/*.slide .alert li {margin-top: 1em}*/
.deck .alert-block p+p {margin-top:5px}
/*.slide .alert-notice { background-image: url(https://hplgit.github.io/doconce/
bundled/html_images//small_gray_notice.png); }
.slide .alert-summary { background-image:url(https://hplgit.github.io/doconce/
bundled/html_images//small_gray_summary.png); }
.slide .alert-warning { background-image: url(https://hplgit.github.io/doconce/
bundled/html_images//small_gray_warning.png); }
.slide .alert-question {background-image:url(https://hplgit.github.io/doconce/
bundled/html_images/small_gray_question.png); } */
.dotable table, .dotable th, .dotable tr, .dotable tr td {
border: 2px solid black;
border-collapse: collapse;
padding: 2px;
}
</style>
<!-- Styles for table layout of slides -->
<style type="text/css">
td.padding {
padding-top:20px;
padding-bottom:20px;
padding-right:50px;
padding-left:50px;
}
</style>
</head>
<body>
<div class="reveal">
<div class="slides">
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<section>
<!-- ------------------- main content ---------------------- -->
<center>
<h1 style="text-align: center;">Week 40: Gradient descent methods (continued) and start Neural networks</h1>
</center> <!-- document title -->
<!-- author(s): Morten Hjorth-Jensen -->
<center>
<b>Morten Hjorth-Jensen</b>
</center>
<!-- institution -->
<center>
<b>Department of Physics, University of Oslo, Norway</b>
</center>
<br>
<center>
<h4>September 29-October 3, 2025</h4>
</center> <!-- date -->
<br>
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
</center>
</section>
<section>
<h2 id="lecture-monday-september-30-2024">Lecture Monday September 30, 2024 </h2>
<div class="alert alert-block alert-block alert-text-normal">
<b></b>
<p>
<ol>
<p><li> Logistic regression and gradient descent, examples on how to code</li>
<p><li> Automatic differentiation and gradient descent, examples using Logistic regression</li>
<p><li> Start with the basics of Neural Networks, setting up the basic steps, from the simple perceptron model to the multi-layer perceptron model
<!-- o <a href="https://youtu.be/jdJoOrCIdII" target="_blank">Video of lecture</a> -->
<!-- o Whiteboard notes at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember30.pdf" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember30.pdf</tt></a> --></li>
</ol>
</div>
</section>
<section>
<h2 id="suggested-readings-and-videos">Suggested readings and videos </h2>
<div class="alert alert-block alert-block alert-text-normal">
<b>Readings and Videos:</b>
<p>
<ol>
<p><li> The lecture notes for week 40 (these notes)</li>
<p><li> For a good discussion on gradient methods, we would like to recommend Goodfellow et al section 4.3-4.5 and sections 8.3-8.6. We will come back to the latter chapter in our discussion of Neural networks as well.</li>
<p><li> For neural networks we recommend Goodfellow et al chapter 6 and Raschka et al chapter 2 (contains also material about gradient descent) and chapter 11 (we will use this next week)
<!-- o Video on gradient descent at <a href="https://www.youtube.com/watch?v=sDv4f4s2SB8" target="_blank"><tt>https://www.youtube.com/watch?v=sDv4f4s2SB8</tt></a> --></li>
<p><li> Video on automatic differentiation at <a href="https://www.youtube.com/watch?v=wG_nF1awSSY" target="_blank"><tt>https://www.youtube.com/watch?v=wG_nF1awSSY</tt></a></li>
<p><li> Neural Networks demystified at <a href="https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs" target="_blank"><tt>https://www.youtube.com/watch?v=bxe2T-V8XRs&list=PLiaHhY2iBX9hdHaRr6b7XevZtgZRa1PoU&ab_channel=WelchLabs</tt></a></li>
<p><li> Building Neural Networks from scratch at URL:https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex"</li>
</ol>
</div>
</section>
<section>
<h2 id="lab-sessions-tuesday-and-wednesday">Lab sessions Tuesday and Wednesday </h2>
<div class="alert alert-block alert-block alert-text-normal">
<b>Material for the active learning sessions on Tuesday and Wednesday</b>
<p>
<ul>
<p><li> Work on project 1 and discussions on how to structure your report</li>
<p><li> No weekly exercises for week 40, project work only</li>
<p><li> Video on how to write scientific reports recorded during one of the lab sessions at <a href="https://youtu.be/tVW1ZDmZnwM" target="_blank"><tt>https://youtu.be/tVW1ZDmZnwM</tt></a></li>
<p><li> A general guideline can be found at <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md" target="_blank"><tt>https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md</tt></a>.</li>
</ul>
</div>
</section>
<section>
<h2 id="automatic-differentiation">Automatic differentiation </h2>
<p><a href="https://en.wikipedia.org/wiki/Automatic_differentiation" target="_blank">Automatic differentiation (AD)</a>,
also called algorithmic
differentiation or computational differentiation,is a set of
techniques to numerically evaluate the derivative of a function
specified by a computer program. AD exploits the fact that every
computer program, no matter how complicated, executes a sequence of
elementary arithmetic operations (addition, subtraction,
multiplication, division, etc.) and elementary functions (exp, log,
sin, cos, etc.). By applying the chain rule repeatedly to these
operations, derivatives of arbitrary order can be computed
automatically, accurately to working precision, and using at most a
small constant factor more arithmetic operations than the original
program.
</p>
<p>Automatic differentiation is neither:</p>
<ul>
<p><li> Symbolic differentiation, nor</li>
<p><li> Numerical differentiation (the method of finite differences).</li>
</ul>
<p>
<p>Symbolic differentiation can lead to inefficient code and faces the
difficulty of converting a computer program into a single expression,
while numerical differentiation can introduce round-off errors in the
discretization process and cancellation
</p>
<p>Python has tools for so-called <b>automatic differentiation</b>.
Consider the following example
</p>
<p>&nbsp;<br>
$$
f(x) = \sin\left(2\pi x + x^2\right)
$$
<p>&nbsp;<br>
<p>which has the following derivative</p>
<p>&nbsp;<br>
$$
f'(x) = \cos\left(2\pi x + x^2\right)\left(2\pi + 2x\right)
$$
<p>&nbsp;<br>
<p>Using <b>autograd</b> we have</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #228B22"># To do elementwise differentiation:</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> elementwise_grad <span style="color: #8B008B; font-weight: bold">as</span> egrad
<span style="color: #228B22"># To plot:</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> np.sin(<span style="color: #B452CD">2</span>*np.pi*x + x**<span style="color: #B452CD">2</span>)
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f_grad_analytic</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> np.cos(<span style="color: #B452CD">2</span>*np.pi*x + x**<span style="color: #B452CD">2</span>)*(<span style="color: #B452CD">2</span>*np.pi + <span style="color: #B452CD">2</span>*x)
<span style="color: #228B22"># Do the comparison:</span>
x = np.linspace(<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>,<span style="color: #B452CD">1000</span>)
f_grad = egrad(f)
computed = f_grad(x)
analytic = f_grad_analytic(x)
plt.title(<span style="color: #CD5555">&#39;Derivative computed from Autograd compared with the analytical derivative&#39;</span>)
plt.plot(x,computed,label=<span style="color: #CD5555">&#39;autograd&#39;</span>)
plt.plot(x,analytic,label=<span style="color: #CD5555">&#39;analytic&#39;</span>)
plt.xlabel(<span style="color: #CD5555">&#39;x&#39;</span>)
plt.ylabel(<span style="color: #CD5555">&#39;y&#39;</span>)
plt.legend()
plt.show()
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The max absolute difference is: %g&quot;</span>%(np.max(np.abs(computed - analytic))))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="using-autograd">Using autograd </h2>
<p>Here we
experiment with what kind of functions Autograd is capable
of finding the gradient of. The following Python functions are just
meant to illustrate what Autograd can do, but please feel free to
experiment with other, possibly more complicated, functions as well.
</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f1</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">3</span> + <span style="color: #B452CD">1</span>
f1_grad = grad(f1)
<span style="color: #228B22"># Remember to send in float as argument to the computed gradient from Autograd!</span>
a = <span style="color: #B452CD">1.0</span>
<span style="color: #228B22"># See the evaluated gradient at a using autograd:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The gradient of f1 evaluated at a = %g using autograd is: %g&quot;</span>%(a,f1_grad(a)))
<span style="color: #228B22"># Compare with the analytical derivative, that is f1&#39;(x) = 3*x**2 </span>
grad_analytical = <span style="color: #B452CD">3</span>*a**<span style="color: #B452CD">2</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g&quot;</span>%(a,grad_analytical))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="autograd-with-more-complicated-functions">Autograd with more complicated functions </h2>
<p>To differentiate with respect to two (or more) arguments of a Python
function, Autograd need to know at which variable the function if
being differentiated with respect to.
</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f2</span>(x1,x2):
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">3</span>*x1**<span style="color: #B452CD">3</span> + x2*(x1 - <span style="color: #B452CD">5</span>) + <span style="color: #B452CD">1</span>
<span style="color: #228B22"># By sending the argument 0, Autograd will compute the derivative w.r.t the first variable, in this case x1</span>
f2_grad_x1 = grad(f2,<span style="color: #B452CD">0</span>)
<span style="color: #228B22"># ... and differentiate w.r.t x2 by sending 1 as an additional arugment to grad</span>
f2_grad_x2 = grad(f2,<span style="color: #B452CD">1</span>)
x1 = <span style="color: #B452CD">1.0</span>
x2 = <span style="color: #B452CD">3.0</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Evaluating at x1 = %g, x2 = %g&quot;</span>%(x1,x2))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;-&quot;</span>*<span style="color: #B452CD">30</span>)
<span style="color: #228B22"># Compare with the analytical derivatives:</span>
<span style="color: #228B22"># Derivative of f2 w.r.t x1 is: 9*x1**2 + x2:</span>
f2_grad_x1_analytical = <span style="color: #B452CD">9</span>*x1**<span style="color: #B452CD">2</span> + x2
<span style="color: #228B22"># Derivative of f2 w.r.t x2 is: x1 - 5:</span>
f2_grad_x2_analytical = x1 - <span style="color: #B452CD">5</span>
<span style="color: #228B22"># See the evaluated derivations:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The derivative of f2 w.r.t x1: %g&quot;</span>%( f2_grad_x1(x1,x2) ))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical derivative of f2 w.r.t x1: %g&quot;</span>%( f2_grad_x1(x1,x2) ))
<span style="color: #658b00">print</span>()
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The derivative of f2 w.r.t x2: %g&quot;</span>%( f2_grad_x2(x1,x2) ))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical derivative of f2 w.r.t x2: %g&quot;</span>%( f2_grad_x2(x1,x2) ))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable.</p>
</section>
<section>
<h2 id="more-complicated-functions-using-the-elements-of-their-arguments-directly">More complicated functions using the elements of their arguments directly </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f3</span>(x): <span style="color: #228B22"># Assumes x is an array of length 5 or higher</span>
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">2</span>*x[<span style="color: #B452CD">0</span>] + <span style="color: #B452CD">3</span>*x[<span style="color: #B452CD">1</span>] + <span style="color: #B452CD">5</span>*x[<span style="color: #B452CD">2</span>] + <span style="color: #B452CD">7</span>*x[<span style="color: #B452CD">3</span>] + <span style="color: #B452CD">11</span>*x[<span style="color: #B452CD">4</span>]**<span style="color: #B452CD">2</span>
f3_grad = grad(f3)
x = np.linspace(<span style="color: #B452CD">0</span>,<span style="color: #B452CD">4</span>,<span style="color: #B452CD">5</span>)
<span style="color: #228B22"># Print the computed gradient:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed gradient of f3 is: &quot;</span>, f3_grad(x))
<span style="color: #228B22"># The analytical gradient is: (2, 3, 5, 7, 22*x[4])</span>
f3_grad_analytical = np.array([<span style="color: #B452CD">2</span>, <span style="color: #B452CD">3</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">22</span>*x[<span style="color: #B452CD">4</span>]])
<span style="color: #228B22"># Print the analytical gradient:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical gradient of f3 is: &quot;</span>, f3_grad_analytical)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>Note that in this case, when sending an array as input argument, the
output from Autograd is another array. This is the true gradient of
the function, as opposed to the function in the previous example. By
using arrays to represent the variables, the output from Autograd
might be easier to work with, as the output is closer to what one
could expect form a gradient-evaluting function.
</p>
</section>
<section>
<h2 id="functions-using-mathematical-functions-from-numpy">Functions using mathematical functions from Numpy </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f4</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> np.sqrt(<span style="color: #B452CD">1</span>+x**<span style="color: #B452CD">2</span>) + np.exp(x) + np.sin(<span style="color: #B452CD">2</span>*np.pi*x)
f4_grad = grad(f4)
x = <span style="color: #B452CD">2.7</span>
<span style="color: #228B22"># Print the computed derivative:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed derivative of f4 at x = %g is: %g&quot;</span>%(x,f4_grad(x)))
<span style="color: #228B22"># The analytical derivative is: x/sqrt(1 + x**2) + exp(x) + cos(2*pi*x)*2*pi</span>
f4_grad_analytical = x/np.sqrt(<span style="color: #B452CD">1</span> + x**<span style="color: #B452CD">2</span>) + np.exp(x) + np.cos(<span style="color: #B452CD">2</span>*np.pi*x)*<span style="color: #B452CD">2</span>*np.pi
<span style="color: #228B22"># Print the analytical gradient:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical gradient of f4 at x = %g is: %g&quot;</span>%(x,f4_grad_analytical))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="more-autograd">More autograd </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f5</span>(x):
<span style="color: #8B008B; font-weight: bold">if</span> x &gt;= <span style="color: #B452CD">0</span>:
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2</span>
<span style="color: #8B008B; font-weight: bold">else</span>:
<span style="color: #8B008B; font-weight: bold">return</span> -<span style="color: #B452CD">3</span>*x + <span style="color: #B452CD">1</span>
f5_grad = grad(f5)
x = <span style="color: #B452CD">2.7</span>
<span style="color: #228B22"># Print the computed derivative:</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed derivative of f5 at x = %g is: %g&quot;</span>%(x,f5_grad(x)))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="and-with-loops">And with loops </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f6_for</span>(x):
val = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">10</span>):
val = val + x**i
<span style="color: #8B008B; font-weight: bold">return</span> val
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f6_while</span>(x):
val = <span style="color: #B452CD">0</span>
i = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">while</span> i &lt; <span style="color: #B452CD">10</span>:
val = val + x**i
i = i + <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">return</span> val
f6_for_grad = grad(f6_for)
f6_while_grad = grad(f6_while)
x = <span style="color: #B452CD">0.5</span>
<span style="color: #228B22"># Print the computed derivaties of f6_for and f6_while</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed derivative of f6_for at x = %g is: %g&quot;</span>%(x,f6_for_grad(x)))
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed derivative of f6_while at x = %g is: %g&quot;</span>%(x,f6_while_grad(x)))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Both of the functions are implementation of the sum: sum(x**i) for i = 0, ..., 9</span>
<span style="color: #228B22"># The analytical derivative is: sum(i*x**(i-1)) </span>
f6_grad_analytical = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">10</span>):
f6_grad_analytical += i*x**(i-<span style="color: #B452CD">1</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical derivative of f6 at x = %g is: %g&quot;</span>%(x,f6_grad_analytical))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="using-recursion">Using recursion </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">f7</span>(n): <span style="color: #228B22"># Assume that n is an integer</span>
<span style="color: #8B008B; font-weight: bold">if</span> n == <span style="color: #B452CD">1</span> <span style="color: #8B008B">or</span> n == <span style="color: #B452CD">0</span>:
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">else</span>:
<span style="color: #8B008B; font-weight: bold">return</span> n*f7(n-<span style="color: #B452CD">1</span>)
f7_grad = grad(f7)
n = <span style="color: #B452CD">2.0</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The computed derivative of f7 at n = %d is: %g&quot;</span>%(n,f7_grad(n)))
<span style="color: #228B22"># The function f7 is an implementation of the factorial of n.</span>
<span style="color: #228B22"># By using the product rule, one can find that the derivative is:</span>
f7_grad_analytical = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(n)-<span style="color: #B452CD">1</span>):
tmp = <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">for</span> k <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(n)-<span style="color: #B452CD">1</span>):
<span style="color: #8B008B; font-weight: bold">if</span> k != i:
tmp *= (n - k)
f7_grad_analytical += tmp
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;The analytical derivative of f7 at n = %d is: %g&quot;</span>%(n,f7_grad_analytical))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input.</p>
</section>
<section>
<h2 id="using-autograd-with-ols">Using Autograd with OLS </h2>
<p>We conclude the part on optmization by showing how we can make codes
for linear regression and logistic regression using <b>autograd</b>. The
first example shows results with ordinary leats squares.
</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients for OLS</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(beta):
<span style="color: #8B008B; font-weight: bold">return</span> (<span style="color: #B452CD">1.0</span>/n)*np.sum((y-X @ beta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #228B22"># define the gradient</span>
training_gradient = grad(CostOLS)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = training_gradient(theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
plt.plot(xnew, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(xnew, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
plt.xlabel(<span style="color: #CD5555">r&#39;$x$&#39;</span>)
plt.ylabel(<span style="color: #CD5555">r&#39;$y$&#39;</span>)
plt.title(<span style="color: #CD5555">r&#39;Random numbers &#39;</span>)
plt.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients for OLS</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(beta):
<span style="color: #8B008B; font-weight: bold">return</span> (<span style="color: #B452CD">1.0</span>/n)*np.sum((y-X @ beta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x<span style="color: #228B22">#+np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">30</span>
<span style="color: #228B22"># define the gradient</span>
training_gradient = grad(CostOLS)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = training_gradient(theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #658b00">iter</span>,gradients[<span style="color: #B452CD">0</span>],gradients[<span style="color: #B452CD">1</span>])
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
<span style="color: #228B22"># Now improve with momentum gradient descent</span>
change = <span style="color: #B452CD">0.0</span>
delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
<span style="color: #228B22"># calculate gradient</span>
gradients = training_gradient(theta)
<span style="color: #228B22"># calculate update</span>
new_change = eta*gradients+delta_momentum*change
<span style="color: #228B22"># take a step</span>
theta -= new_change
<span style="color: #228B22"># save the change</span>
change = new_change
<span style="color: #658b00">print</span>(<span style="color: #658b00">iter</span>,gradients[<span style="color: #B452CD">0</span>],gradients[<span style="color: #B452CD">1</span>])
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd wth momentum&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="including-stochastic-gradient-descent-with-autograd">Including Stochastic Gradient Descent with Autograd </h2>
<p>In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using <b>autograd</b>.</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
plt.plot(xnew, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(xnew, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
plt.xlabel(<span style="color: #CD5555">r&#39;$x$&#39;</span>)
plt.ylabel(<span style="color: #CD5555">r&#39;$y$&#39;</span>)
plt.title(<span style="color: #CD5555">r&#39;Random numbers &#39;</span>)
plt.show()
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">100</span>
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
change = <span style="color: #B452CD">0.0</span>
delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
eta = learning_schedule(epoch*m+i)
<span style="color: #228B22"># calculate update</span>
new_change = eta*gradients+delta_momentum*change
<span style="color: #228B22"># take a step</span>
theta -= new_change
<span style="color: #228B22"># save the change</span>
change = new_change
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg with momentum&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="similar-second-order-function-now-problem-but-now-with-adagrad">Similar (second order function now) problem but now with AdaGrad </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
Giter += gradients*gradients
update = gradients*eta/(delta+np.sqrt(Giter))
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
</section>
<section>
<h2 id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">RMSprop for adaptive learning rate with Stochastic Gradient Descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameter rho</span>
rho = <span style="color: #B452CD">0.99</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Accumulated gradient</span>
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
update = gradients*eta/(delta+np.sqrt(Giter))
<span style="color: #228B22"># Hadamard product</span>
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf">And finally <a href="https://arxiv.org/pdf/1412.6980.pdf" target="_blank">ADAM</a> </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980</span>
beta1 = <span style="color: #B452CD">0.9</span>
beta2 = <span style="color: #B452CD">0.999</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-7</span>
<span style="color: #658b00">iter</span> = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
first_moment = <span style="color: #B452CD">0.0</span>
second_moment = <span style="color: #B452CD">0.0</span>
<span style="color: #658b00">iter</span> += <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Computing moments first</span>
first_moment = beta1*first_moment + (<span style="color: #B452CD">1</span>-beta1)*gradients
second_moment = beta2*second_moment+(<span style="color: #B452CD">1</span>-beta2)*gradients*gradients
first_term = first_moment/(<span style="color: #B452CD">1.0</span>-beta1**<span style="color: #658b00">iter</span>)
second_term = second_moment/(<span style="color: #B452CD">1.0</span>-beta2**<span style="color: #658b00">iter</span>)
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
update = eta*first_term/(np.sqrt(second_term)+delta)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own ADAM&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sigmoid</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">0.5</span> * (np.tanh(x / <span style="color: #B452CD">2.</span>) + <span style="color: #B452CD">1</span>)
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">logistic_predictions</span>(weights, inputs):
<span style="color: #228B22"># Outputs probability of a label being true according to logistic model.</span>
<span style="color: #8B008B; font-weight: bold">return</span> sigmoid(np.dot(inputs, weights))
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">training_loss</span>(weights):
<span style="color: #228B22"># Training loss is the negative log-likelihood of the training labels.</span>
preds = logistic_predictions(weights, inputs)
label_probabilities = preds * targets + (<span style="color: #B452CD">1</span> - preds) * (<span style="color: #B452CD">1</span> - targets)
<span style="color: #8B008B; font-weight: bold">return</span> -np.sum(np.log(label_probabilities))
<span style="color: #228B22"># Build a toy dataset.</span>
inputs = np.array([[<span style="color: #B452CD">0.52</span>, <span style="color: #B452CD">1.12</span>, <span style="color: #B452CD">0.77</span>],
[<span style="color: #B452CD">0.88</span>, -<span style="color: #B452CD">1.08</span>, <span style="color: #B452CD">0.15</span>],
[<span style="color: #B452CD">0.52</span>, <span style="color: #B452CD">0.06</span>, -<span style="color: #B452CD">1.30</span>],
[<span style="color: #B452CD">0.74</span>, -<span style="color: #B452CD">2.49</span>, <span style="color: #B452CD">1.39</span>]])
targets = np.array([<span style="color: #8B008B; font-weight: bold">True</span>, <span style="color: #8B008B; font-weight: bold">True</span>, <span style="color: #8B008B; font-weight: bold">False</span>, <span style="color: #8B008B; font-weight: bold">True</span>])
<span style="color: #228B22"># Define a function that returns gradients of training loss using Autograd.</span>
training_gradient_fun = grad(training_loss)
<span style="color: #228B22"># Optimize weights using gradient descent.</span>
weights = np.array([<span style="color: #B452CD">0.0</span>, <span style="color: #B452CD">0.0</span>, <span style="color: #B452CD">0.0</span>])
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Initial loss:&quot;</span>, training_loss(weights))
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">100</span>):
weights -= training_gradient_fun(weights) * <span style="color: #B452CD">0.01</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Trained loss:&quot;</span>, training_loss(weights))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<h2 id="introducing-jax-https-jax-readthedocs-io-en-latest">Introducing <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a> </h2>
<p>Presently, instead of using <b>autograd</b>, we recommend using <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a></p>
<p><b>JAX</b> is Autograd and <a href="https://www.tensorflow.org/xla" target="_blank">XLA (Accelerated Linear Algebra))</a>,
brought together for high-performance numerical computing and machine learning research.
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.
</p>
<h3 id="getting-started-with-jax-note-the-way-we-import-numpy">Getting started with Jax, note the way we import numpy </h3>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">jax</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">jax.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">jnp</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">jax</span> <span style="color: #8B008B; font-weight: bold">import</span> grad <span style="color: #8B008B; font-weight: bold">as</span> jax_grad
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<h3 id="a-warm-up-example">A warm-up example </h3>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">function</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">analytical_gradient</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> <span style="color: #B452CD">2</span>*x
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(starting_point, learning_rate, num_iterations, solver=<span style="color: #CD5555">&quot;analytical&quot;</span>):
x = starting_point
trajectory_x = [x]
trajectory_y = [function(x)]
<span style="color: #8B008B; font-weight: bold">if</span> solver == <span style="color: #CD5555">&quot;analytical&quot;</span>:
grad = analytical_gradient
<span style="color: #8B008B; font-weight: bold">elif</span> solver == <span style="color: #CD5555">&quot;jax&quot;</span>:
grad = jax_grad(function)
x = jnp.float64(x)
learning_rate = jnp.float64(learning_rate)
<span style="color: #8B008B; font-weight: bold">for</span> _ <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(num_iterations):
x = x - learning_rate * grad(x)
trajectory_x.append(x)
trajectory_y.append(function(x))
<span style="color: #8B008B; font-weight: bold">return</span> trajectory_x, trajectory_y
x = np.linspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">100</span>)
plt.plot(x, function(x), label=<span style="color: #CD5555">&quot;f(x)&quot;</span>)
descent_x, descent_y = gradient_descent(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">10</span>, solver=<span style="color: #CD5555">&quot;analytical&quot;</span>)
jax_descend_x, jax_descend_y = gradient_descent(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">10</span>, solver=<span style="color: #CD5555">&quot;jax&quot;</span>)
plt.plot(descent_x, descent_y, label=<span style="color: #CD5555">&quot;Gradient descent&quot;</span>, marker=<span style="color: #CD5555">&quot;o&quot;</span>)
plt.plot(jax_descend_x, jax_descend_y, label=<span style="color: #CD5555">&quot;JAX&quot;</span>, marker=<span style="color: #CD5555">&quot;x&quot;</span>)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<h3 id="a-more-advanced-example">A more advanced example </h3>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;">backend = np
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">function</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x*backend.sin(x**<span style="color: #B452CD">2</span> + <span style="color: #B452CD">1</span>)
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">analytical_gradient</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> backend.sin(x**<span style="color: #B452CD">2</span> + <span style="color: #B452CD">1</span>) + <span style="color: #B452CD">2</span>*x**<span style="color: #B452CD">2</span>*backend.cos(x**<span style="color: #B452CD">2</span> + <span style="color: #B452CD">1</span>)
x = np.linspace(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">100</span>)
plt.plot(x, function(x), label=<span style="color: #CD5555">&quot;f(x)&quot;</span>)
descent_x, descent_y = gradient_descent(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">300</span>, solver=<span style="color: #CD5555">&quot;analytical&quot;</span>)
<span style="color: #228B22"># Change the backend to JAX</span>
backend = jnp
jax_descend_x, jax_descend_y = gradient_descent(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0.01</span>, <span style="color: #B452CD">300</span>, solver=<span style="color: #CD5555">&quot;jax&quot;</span>)
plt.scatter(descent_x, descent_y, label=<span style="color: #CD5555">&quot;Gradient descent&quot;</span>, marker=<span style="color: #CD5555">&quot;v&quot;</span>, s=<span style="color: #B452CD">10</span>, color=<span style="color: #CD5555">&quot;red&quot;</span>)
plt.scatter(jax_descend_x, jax_descend_y, label=<span style="color: #CD5555">&quot;JAX&quot;</span>, marker=<span style="color: #CD5555">&quot;x&quot;</span>, s=<span style="color: #B452CD">5</span>, color=<span style="color: #CD5555">&quot;black&quot;</span>)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="introduction-to-neural-networks">Introduction to Neural networks </h2>
<p>Artificial neural networks are computational systems that can learn to
perform tasks by considering examples, generally without being
programmed with any task-specific rules. It is supposed to mimic a
biological system, wherein neurons interact by sending signals in the
form of mathematical functions between layers. All layers can contain
an arbitrary number of neurons, and each connection is represented by
a weight variable.
</p>
</section>
<section>
<h2 id="artificial-neurons">Artificial neurons </h2>
<p>The field of artificial neural networks has a long history of
development, and is closely connected with the advancement of computer
science and computers in general. A model of artificial neurons was
first developed by McCulloch and Pitts in 1943 to study signal
processing in the brain and has later been refined by others. The
general idea is to mimic neural networks in the human brain, which is
composed of billions of neurons that communicate with each other by
sending electrical signals. Each neuron accumulates its incoming
signals, which must exceed an activation threshold to yield an
output. If the threshold is not overcome, the neuron remains inactive,
i.e. has zero output.
</p>
<p>This behaviour has inspired a simple mathematical model for an artificial neuron.</p>
<p>&nbsp;<br>
$$
\begin{equation}
y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
\tag{1}
\end{equation}
$$
<p>&nbsp;<br>
<p>Here, the output \( y \) of the neuron is the value of its activation function, which have as input
a weighted sum of signals \( x_i, \dots ,x_n \) received by \( n \) other neurons.
</p>
<p>Conceptually, it is helpful to divide neural networks into four
categories:
</p>
<ol>
<p><li> general purpose neural networks for supervised learning,</li>
<p><li> neural networks designed specifically for image processing, the most prominent example of this class being Convolutional Neural Networks (CNNs),</li>
<p><li> neural networks for sequential data such as Recurrent Neural Networks (RNNs), and</li>
<p><li> neural networks for unsupervised learning such as Deep Boltzmann Machines.</li>
</ol>
<p>
<p>In natural science, DNNs and CNNs have already found numerous
applications. In statistical physics, they have been applied to detect
phase transitions in 2D Ising and Potts models, lattice gauge
theories, and different phases of polymers, or solving the
Navier-Stokes equation in weather forecasting. Deep learning has also
found interesting applications in quantum physics. Various quantum
phase transitions can be detected and studied using DNNs and CNNs,
topological phases, and even non-equilibrium many-body
localization. Representing quantum states as DNNs quantum state
tomography are among some of the impressive achievements to reveal the
potential of DNNs to facilitate the study of quantum systems.
</p>
<p>In quantum information theory, it has been shown that one can perform
gate decompositions with the help of neural.
</p>
<p>The applications are not limited to the natural sciences. There is a
plethora of applications in essentially all disciplines, from the
humanities to life science and medicine.
</p>
</section>
<section>
<h2 id="neural-network-types">Neural network types </h2>
<p>An artificial neural network (ANN), is a computational model that
consists of layers of connected neurons, or nodes or units. We will
refer to these interchangeably as units or nodes, and sometimes as
neurons.
</p>
<p>It is supposed to mimic a biological nervous system by letting each
neuron interact with other neurons by sending signals in the form of
mathematical functions between layers. A wide variety of different
ANNs have been developed, but most of them consist of an input layer,
an output layer and eventual layers in-between, called <em>hidden
layers</em>. All layers can contain an arbitrary number of nodes, and each
connection between two nodes is associated with a weight variable.
</p>
<p>Neural networks (also called neural nets) are neural-inspired
nonlinear models for supervised learning. As we will see, neural nets
can be viewed as natural, more powerful extensions of supervised
learning methods such as linear and logistic regression and soft-max
methods we discussed earlier.
</p>
</section>
<section>
<h2 id="feed-forward-neural-networks">Feed-forward neural networks </h2>
<p>The feed-forward neural network (FFNN) was the first and simplest type
of ANNs that were devised. In this network, the information moves in
only one direction: forward through the layers.
</p>
<p>Nodes are represented by circles, while the arrows display the
connections between the nodes, including the direction of information
flow. Additionally, each arrow corresponds to a weight variable
(figure to come). We observe that each node in a layer is connected
to <em>all</em> nodes in the subsequent layer, making this a so-called
<em>fully-connected</em> FFNN.
</p>
</section>
<section>
<h2 id="convolutional-neural-network">Convolutional Neural Network </h2>
<p>A different variant of FFNNs are <em>convolutional neural networks</em>
(CNNs), which have a connectivity pattern inspired by the animal
visual cortex. Individual neurons in the visual cortex only respond to
stimuli from small sub-regions of the visual field, called a receptive
field. This makes the neurons well-suited to exploit the strong
spatially local correlation present in natural images. The response of
each neuron can be approximated mathematically as a convolution
operation. (figure to come)
</p>
<p>Convolutional neural networks emulate the behaviour of neurons in the
visual cortex by enforcing a <em>local</em> connectivity pattern between
nodes of adjacent layers: Each node in a convolutional layer is
connected only to a subset of the nodes in the previous layer, in
contrast to the fully-connected FFNN. Often, CNNs consist of several
convolutional layers that learn local features of the input, with a
fully-connected layer at the end, which gathers all the local data and
produces the outputs. They have wide applications in image and video
recognition.
</p>
</section>
<section>
<h2 id="recurrent-neural-networks">Recurrent neural networks </h2>
<p>So far we have only mentioned ANNs where information flows in one
direction: forward. <em>Recurrent neural networks</em> on the other hand,
have connections between nodes that form directed <em>cycles</em>. This
creates a form of internal memory which are able to capture
information on what has been calculated before; the output is
dependent on the previous computations. Recurrent NNs make use of
sequential information by performing the same task for every element
in a sequence, where each element depends on previous elements. An
example of such information is sentences, making recurrent NNs
especially well-suited for handwriting and speech recognition.
</p>
</section>
<section>
<h2 id="other-types-of-networks">Other types of networks </h2>
<p>There are many other kinds of ANNs that have been developed. One type
that is specifically designed for interpolation in multidimensional
space is the radial basis function (RBF) network. RBFs are typically
made up of three layers: an input layer, a hidden layer with
non-linear radial symmetric activation functions and a linear output
layer (''linear'' here means that each node in the output layer has a
linear activation function). The layers are normally fully-connected
and there are no cycles, thus RBFs can be viewed as a type of
fully-connected FFNN. They are however usually treated as a separate
type of NN due the unusual activation functions.
</p>
</section>
<section>
<h2 id="multilayer-perceptrons">Multilayer perceptrons </h2>
<p>One uses often so-called fully-connected feed-forward neural networks
with three or more layers (an input layer, one or more hidden layers
and an output layer) consisting of neurons that have non-linear
activation functions.
</p>
<p>Such networks are often called <em>multilayer perceptrons</em> (MLPs).</p>
</section>
<section>
<h2 id="why-multilayer-perceptrons">Why multilayer perceptrons? </h2>
<p>According to the <em>Universal approximation theorem</em>, a feed-forward
neural network with just a single hidden layer containing a finite
number of neurons can approximate a continuous multidimensional
function to arbitrary accuracy, assuming the activation function for
the hidden layer is a <b>non-constant, bounded and
monotonically-increasing continuous function</b>.
</p>
<p>Note that the requirements on the activation function only applies to
the hidden layer, the output nodes are always assumed to be linear, so
as to not restrict the range of output values.
</p>
</section>
<section>
<h2 id="illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model">Illustration of a single perceptron model and a multi-perceptron model </h2>
<center> <!-- FIGURE -->
<hr class="figure">
<center>
<p class="caption">Figure 1: In a) we show a single perceptron model while in b) we dispay a network with two hidden layers, an input layer and an output layer. </p>
</center>
<p><img src="figures/nns.png" width="600" align="bottom"></p>
</center>
</section>
<section>
<h2 id="examples-of-xor-or-and-and-gates">Examples of XOR, OR and AND gates </h2>
<p>Let us first try to fit various gates using standard linear
regression. The gates we are thinking of are the classical XOR, OR and
AND gates, well-known elements in computer science. The tables here
show how we can set up the inputs \( x_1 \) and \( x_2 \) in order to yield a
specific target \( y_i \).
</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
<span style="color: #CD5555">Simple code that tests XOR, OR and AND gates with linear regression</span>
<span style="color: #CD5555">&quot;&quot;&quot;</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #228B22"># Design matrix</span>
X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
Xinv = np.linalg.pinv(X.T @ X)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The XOR gate </span>
yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
ThetaXOR = Xinv @ X.T @ yXOR
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The OR gate </span>
yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
ThetaOR = Xinv @ X.T @ yOR
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The OR gate </span>
yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
ThetaAND = Xinv @ X.T @ yAND
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>What is happening here?</p>
</section>
<section>
<h2 id="does-logistic-regression-do-a-better-job">Does Logistic Regression do a better Job? </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;</span>
<span style="color: #CD5555">Simple code that tests XOR and OR gates with linear regression</span>
<span style="color: #CD5555">and logistic regression</span>
<span style="color: #CD5555">&quot;&quot;&quot;</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #228B22"># Design matrix</span>
X = np.array([ [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>], [<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>],[<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>]],dtype=np.float64)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The X.TX matrix:{</span>X.T @ X<span style="color: #CD5555">}&quot;</span>)
Xinv = np.linalg.pinv(X.T @ X)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The invers of X.TX matrix:{</span>Xinv<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The XOR gate </span>
yXOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">0</span>])
ThetaXOR = Xinv @ X.T @ yXOR
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the XOR gate:{</span>ThetaXOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the XOR gate:{</span>X @ ThetaXOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The OR gate </span>
yOR = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span> ,<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>])
ThetaOR = Xinv @ X.T @ yOR
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the OR gate:{</span>ThetaOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the OR gate:{</span>X @ ThetaOR<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># The OR gate </span>
yAND = np.array( [ <span style="color: #B452CD">0</span>, <span style="color: #B452CD">0</span> ,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>])
ThetaAND = Xinv @ X.T @ yAND
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The values of theta for the AND gate:{</span>ThetaAND<span style="color: #CD5555">}&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;The linear regression prediction for the AND gate:{</span>X @ ThetaAND<span style="color: #CD5555">}&quot;</span>)
<span style="color: #228B22"># Now we change to logistic regression</span>
<span style="color: #228B22"># Logistic Regression</span>
logreg = LogisticRegression()
logreg.fit(X, yOR)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for OR gate: {:.2f}&quot;</span>.format(logreg.score(X,yOR)))
logreg.fit(X, yXOR)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for XOR gate: {:.2f}&quot;</span>.format(logreg.score(X,yXOR)))
logreg.fit(X, yAND)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Test set accuracy with Logistic Regression for AND gate: {:.2f}&quot;</span>.format(logreg.score(X,yAND)))
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<p>Not exactly impressive, but somewhat better.</p>
</section>
<section>
<h2 id="adding-neural-networks">Adding Neural Networks </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># and now neural networks with Scikit-Learn and the XOR</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.neural_network</span> <span style="color: #8B008B; font-weight: bold">import</span> MLPClassifier
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_classification
X, yXOR = make_classification(n_samples=<span style="color: #B452CD">100</span>, random_state=<span style="color: #B452CD">1</span>)
FFNN = MLPClassifier(random_state=<span style="color: #B452CD">1</span>, max_iter=<span style="color: #B452CD">300</span>).fit(X, yXOR)
FFNN.predict_proba(X)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Test set accuracy with Feed Forward Neural Network for XOR gate:{</span>FFNN.score(X, yXOR)<span style="color: #CD5555">}&quot;</span>)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="mathematical-model">Mathematical model </h2>
<p>The output \( y \) is produced via the activation function \( f \)</p>
<p>&nbsp;<br>
$$
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
$$
<p>&nbsp;<br>
<p>This function receives \( x_i \) as inputs.
Here the activation \( z=(\sum_{i=1}^n w_ix_i+b_i) \).
In an FFNN of such neurons, the <em>inputs</em> \( x_i \) are the <em>outputs</em> of
the neurons in the preceding layer. Furthermore, an MLP is
fully-connected, which means that each neuron receives a weighted sum
of the outputs of <em>all</em> neurons in the previous layer.
</p>
</section>
<section>
<h2 id="mathematical-model">Mathematical model </h2>
<p>First, for each node \( i \) in the first hidden layer, we calculate a weighted sum \( z_i^1 \) of the input coordinates \( x_j \),</p>
<p>&nbsp;<br>
$$
\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
\tag{2}
\end{equation}
$$
<p>&nbsp;<br>
<p>Here \( b_i \) is the so-called bias which is normally needed in
case of zero activation weights or inputs. How to fix the biases and
the weights will be discussed below. The value of \( z_i^1 \) is the
argument to the activation function \( f_i \) of each node \( i \), The
variable \( M \) stands for all possible inputs to a given node \( i \) in the
first layer. We define the output \( y_i^1 \) of all neurons in layer 1 as
</p>
<p>&nbsp;<br>
$$
\begin{equation}
y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j + b_i^1\right)
\tag{3}
\end{equation}
$$
<p>&nbsp;<br>
<p>where we assume that all nodes in the same layer have identical
activation functions, hence the notation \( f \). In general, we could assume in the more general case that different layers have different activation functions.
In this case we would identify these functions with a superscript \( l \) for the \( l \)-th layer,
</p>
<p>&nbsp;<br>
$$
\begin{equation}
y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
\tag{4}
\end{equation}
$$
<p>&nbsp;<br>
<p>where \( N_l \) is the number of nodes in layer \( l \). When the output of
all the nodes in the first hidden layer are computed, the values of
the subsequent layer can be calculated and so forth until the output
is obtained.
</p>
</section>
<section>
<h2 id="mathematical-model">Mathematical model </h2>
<p>The output of neuron \( i \) in layer 2 is thus,</p>
<p>&nbsp;<br>
$$
\begin{align}
y_i^2 &= f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right)
\tag{5}\\
&= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
\tag{6}
\end{align}
$$
<p>&nbsp;<br>
<p>where we have substituted \( y_k^1 \) with the inputs \( x_k \). Finally, the ANN output reads</p>
<p>&nbsp;<br>
$$
\begin{align}
y_i^3 &= f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right)
\tag{7}\\
&= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
+ b_1^3\right]
\tag{8}
\end{align}
$$
<p>&nbsp;<br>
</section>
<section>
<h2 id="mathematical-model">Mathematical model </h2>
<p>We can generalize this expression to an MLP with \( l \) hidden
layers. The complete functional form is,
</p>
<p>&nbsp;<br>
$$
\begin{align}
&y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] &&
\tag{9}
\end{align}
$$
<p>&nbsp;<br>
<p>which illustrates a basic property of MLPs: The only independent
variables are the input values \( x_n \).
</p>
</section>
<section>
<h2 id="mathematical-model">Mathematical model </h2>
<p>This confirms that an MLP, despite its quite convoluted mathematical
form, is nothing more than an analytic function, specifically a
mapping of real-valued vectors \( \hat{x} \in \mathbb{R}^n \rightarrow
\hat{y} \in \mathbb{R}^m \).
</p>
<p>Furthermore, the flexibility and universality of an MLP can be
illustrated by realizing that the expression is essentially a nested
sum of scaled activation functions of the form
</p>
<p>&nbsp;<br>
$$
\begin{equation}
f(x) = c_1 f(c_2 x + c_3) + c_4
\tag{10}
\end{equation}
$$
<p>&nbsp;<br>
<p>where the parameters \( c_i \) are weights and biases. By adjusting these
parameters, the activation functions can be shifted up and down or
left and right, change slope or be rescaled which is the key to the
flexibility of a neural network.
</p>
</section>
<section>
<h3 id="matrix-vector-notation">Matrix-vector notation </h3>
<p>We can introduce a more convenient notation for the activations in an A NN. </p>
<p>Additionally, we can represent the biases and activations
as layer-wise column vectors \( \hat{b}_l \) and \( \hat{y}_l \), so that the \( i \)-th element of each vector
is the bias \( b_i^l \) and activation \( y_i^l \) of node \( i \) in layer \( l \) respectively.
</p>
<p>We have that \( \mathrm{W}_l \) is an \( N_{l-1} \times N_l \) matrix, while \( \hat{b}_l \) and \( \hat{y}_l \) are \( N_l \times 1 \) column vectors.
With this notation, the sum becomes a matrix-vector multiplication, and we can write
the equation for the activations of hidden layer 2 (assuming three nodes for simplicity) as
</p>
<p>&nbsp;<br>
$$
\begin{equation}
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
f_2\left(\left[\begin{array}{ccc}
w^2_{11} &w^2_{12} &w^2_{13} \\
w^2_{21} &w^2_{22} &w^2_{23} \\
w^2_{31} &w^2_{32} &w^2_{33} \\
\end{array} \right] \cdot
\left[\begin{array}{c}
y^1_1 \\
y^1_2 \\
y^1_3 \\
\end{array}\right] +
\left[\begin{array}{c}
b^2_1 \\
b^2_2 \\
b^2_3 \\
\end{array}\right]\right).
\tag{11}
\end{equation}
$$
<p>&nbsp;<br>
</section>
<section>
<h3 id="matrix-vector-notation-and-activation">Matrix-vector notation and activation </h3>
<p>The activation of node \( i \) in layer 2 is</p>
<p>&nbsp;<br>
$$
\begin{equation}
y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) =
f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
\tag{12}
\end{equation}
$$
<p>&nbsp;<br>
<p>This is not just a convenient and compact notation, but also a useful
and intuitive way to think about MLPs: The output is calculated by a
series of matrix-vector multiplications and vector additions that are
used as input to the activation functions. For each operation
\( \mathrm{W}_l \hat{y}_{l-1} \) we move forward one layer.
</p>
</section>
<section>
<h3 id="activation-functions">Activation functions </h3>
<p>A property that characterizes a neural network, other than its
connectivity, is the choice of activation function(s). As described
in, the following restrictions are imposed on an activation function
for a FFNN to fulfill the universal approximation theorem
</p>
<ul>
<p><li> Non-constant</li>
<p><li> Bounded</li>
<p><li> Monotonically-increasing</li>
<p><li> Continuous</li>
</ul>
</section>
<section>
<h3 id="activation-functions-logistic-and-hyperbolic-ones">Activation functions, Logistic and Hyperbolic ones </h3>
<p>The second requirement excludes all linear functions. Furthermore, in
a MLP with only linear activation functions, each layer simply
performs a linear transformation of its inputs.
</p>
<p>Regardless of the number of layers, the output of the NN will be
nothing but a linear function of the inputs. Thus we need to introduce
some kind of non-linearity to the NN to be able to fit non-linear
functions Typical examples are the logistic <em>Sigmoid</em>
</p>
<p>&nbsp;<br>
$$
f(x) = \frac{1}{1 + e^{-x}},
$$
<p>&nbsp;<br>
<p>and the <em>hyperbolic tangent</em> function</p>
<p>&nbsp;<br>
$$
f(x) = \tanh(x)
$$
<p>&nbsp;<br>
</section>
<section>
<h3 id="relevance">Relevance </h3>
<p>The <em>sigmoid</em> function are more biologically plausible because the
output of inactive neurons are zero. Such activation function are
called <em>one-sided</em>. However, it has been shown that the hyperbolic
tangent performs better than the sigmoid for training MLPs. has
become the most popular for <em>deep neural networks</em>
</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #CD5555">&quot;&quot;&quot;The sigmoid function (or the logistic curve) is a </span>
<span style="color: #CD5555">function that takes any real number, z, and outputs a number (0,1).</span>
<span style="color: #CD5555">It is useful in neural networks for assigning weights on a relative scale.</span>
<span style="color: #CD5555">The value z is the weighted sum of parameters involved in the learning algorithm.&quot;&quot;&quot;</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">mt</span>
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.1</span>)
sigma_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1</span>/(<span style="color: #B452CD">1</span>+numpy.exp(-z)))
sigma = sigma_fn(z)
fig = plt.figure()
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
ax.plot(z, sigma)
ax.set_ylim([-<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">1.1</span>])
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
ax.set_title(<span style="color: #CD5555">&#39;sigmoid function&#39;</span>)
plt.show()
<span style="color: #CD5555">&quot;&quot;&quot;Step Function&quot;&quot;&quot;</span>
z = numpy.arange(-<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>, <span style="color: #B452CD">.02</span>)
step_fn = numpy.vectorize(<span style="color: #8B008B; font-weight: bold">lambda</span> z: <span style="color: #B452CD">1.0</span> <span style="color: #8B008B; font-weight: bold">if</span> z &gt;= <span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">else</span> <span style="color: #B452CD">0.0</span>)
step = step_fn(z)
fig = plt.figure()
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
ax.plot(z, step)
ax.set_ylim([-<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">1.5</span>])
ax.set_xlim([-<span style="color: #B452CD">5</span>,<span style="color: #B452CD">5</span>])
ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
ax.set_title(<span style="color: #CD5555">&#39;step function&#39;</span>)
plt.show()
<span style="color: #CD5555">&quot;&quot;&quot;Sine Function&quot;&quot;&quot;</span>
z = numpy.arange(-<span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">2</span>*mt.pi, <span style="color: #B452CD">0.1</span>)
t = numpy.sin(z)
fig = plt.figure()
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
ax.plot(z, t)
ax.set_ylim([-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>])
ax.set_xlim([-<span style="color: #B452CD">2</span>*mt.pi,<span style="color: #B452CD">2</span>*mt.pi])
ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
ax.set_title(<span style="color: #CD5555">&#39;sine function&#39;</span>)
plt.show()
<span style="color: #CD5555">&quot;&quot;&quot;Plots a graph of the squashing function used by a rectified linear</span>
<span style="color: #CD5555">unit&quot;&quot;&quot;</span>
z = numpy.arange(-<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>, <span style="color: #B452CD">.1</span>)
zero = numpy.zeros(<span style="color: #658b00">len</span>(z))
y = numpy.max([zero, z], axis=<span style="color: #B452CD">0</span>)
fig = plt.figure()
ax = fig.add_subplot(<span style="color: #B452CD">111</span>)
ax.plot(z, y)
ax.set_ylim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
ax.set_xlim([-<span style="color: #B452CD">2.0</span>, <span style="color: #B452CD">2.0</span>])
ax.grid(<span style="color: #8B008B; font-weight: bold">True</span>)
ax.set_xlabel(<span style="color: #CD5555">&#39;z&#39;</span>)
ax.set_title(<span style="color: #CD5555">&#39;Rectified linear unit&#39;</span>)
plt.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
</div> <!-- class="slides" -->
</div> <!-- class="reveal" -->
<script src="reveal.js/lib/js/head.min.js"></script>
<script src="reveal.js/js/reveal.js"></script>
<script>
// Full list of configuration options available here:
// https://github.com/hakimel/reveal.js#configuration
Reveal.initialize({
// Display navigation controls in the bottom right corner
controls: true,
// Display progress bar (below the horiz. slider)
progress: true,
// Display the page number of the current slide
slideNumber: true,
// Push each slide change to the browser history
history: false,
// Enable keyboard shortcuts for navigation
keyboard: true,
// Enable the slide overview mode
overview: true,
// Vertical centering of slides
//center: true,
center: false,
// Enables touch navigation on devices with touch input
touch: true,
// Loop the presentation
loop: false,
// Change the presentation direction to be RTL
rtl: false,
// Turns fragments on and off globally
fragments: true,
// Flags if the presentation is running in an embedded mode,
// i.e. contained within a limited portion of the screen
embedded: false,
// Number of milliseconds between automatically proceeding to the
// next slide, disabled when set to 0, this value can be overwritten
// by using a data-autoslide attribute on your slides
autoSlide: 0,
// Stop auto-sliding after user input
autoSlideStoppable: true,
// Enable slide navigation via mouse wheel
mouseWheel: false,
// Hides the address bar on mobile devices
hideAddressBar: true,
// Opens links in an iframe preview overlay
previewLinks: false,
// Transition style
transition: 'default', // default/cube/page/concave/zoom/linear/fade/none
// Transition speed
transitionSpeed: 'default', // default/fast/slow
// Transition style for full page slide backgrounds
backgroundTransition: 'default', // default/none/slide/concave/convex/zoom
// Number of slides away from the current that are visible
viewDistance: 3,
// Parallax background image
//parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
// Parallax background size
//parallaxBackgroundSize: '' // CSS syntax, e.g. "2100px 900px"
theme: Reveal.getQueryHash().theme, // available themes are in reveal.js/css/theme
transition: Reveal.getQueryHash().transition || 'none', // default/cube/page/concave/zoom/linear/none
});
Reveal.initialize({
dependencies: [
// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
{ src: 'reveal.js/lib/js/classList.js', condition: function() { return !document.body.classList; } },
// Interpret Markdown in <section> elements
{ src: 'reveal.js/plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
// Syntax highlight for <code> elements
{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
// Zoom in and out with Alt+click
{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: function() { return !!document.body.classList; } },
// Speaker notes
{ src: 'reveal.js/plugin/notes/notes.js', async: true, condition: function() { return !!document.body.classList; } },
// Remote control your reveal.js presentation using a touch device
//{ src: 'reveal.js/plugin/remotes/remotes.js', async: true, condition: function() { return !!document.body.classList; } },
// MathJax
//{ src: 'reveal.js/plugin/math/math.js', async: true }
]
});
Reveal.initialize({
// The "normal" size of the presentation, aspect ratio will be preserved
// when the presentation is scaled to fit different resolutions. Can be
// specified using percentage units.
width: 1170, // original: 960,
height: 700,
// Factor of the display size that should remain empty around the content
margin: 0.1,
// Bounds for smallest/largest possible scale to apply to content
minScale: 0.2,
maxScale: 1.0
});
</script>
<!-- begin footer logo
<div style="position: absolute; bottom: 0px; left: 0; margin-left: 0px">
<img src="somelogo.png">
</div>
end footer logo -->
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink