lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update week 39

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-09-29 05:27:01 +02:00
parent 6a86dbcaaa
commit ef56fde917
6 changed files with 719 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-bs.html
+47 -37
View File
@@ -152,6 +152,14 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'using-gradient-descent-methods-limitations'),
('Improving gradient descent with momentum',
2,
None,
'improving-gradient-descent-with-momentum'),
('Same code but now with momentum gradient descent',
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Overview video on Stochastic Gradient Descent',
2,
None,
@@ -310,41 +318,43 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week39-bs041.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs042.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs043.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs044.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs045.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs046.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs047.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs048.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs049.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs050.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs051.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs052.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs053.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs054.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs055.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs056.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs057.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs058.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs059.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs060.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs061.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs062.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs063.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs064.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs065.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs066.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs067.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs068.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs069.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs070.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs071.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs072.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs073.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs074.html#recommended-to-avoid" style="font-size: 80%;">Recommended to avoid</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs075.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs076.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs077.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs078.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs044.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs045.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs046.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs047.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs048.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs049.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs050.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs051.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs052.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs053.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs054.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs055.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs056.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs057.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs058.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs059.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs060.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs061.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs062.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs063.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs064.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs065.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs066.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs067.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs068.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs069.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs070.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs071.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs072.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs073.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs074.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs075.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs076.html#recommended-to-avoid" style="font-size: 80%;">Recommended to avoid</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs077.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs078.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs079.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs080.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
</ul>
</li>
@@ -374,7 +384,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 27, 2022</h4>
<h4>Sep 29, 2022</h4>
</center> <!-- date -->
<br>
@@ -399,7 +409,7 @@ MathJax.Hub.Config({
<li><a href="._week39-bs008.html">9</a></li>
<li><a href="._week39-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._week39-bs078.html">79</a></li>
<li><a href="._week39-bs080.html">81</a></li>
<li><a href="._week39-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/week39/html/week39-reveal.html
+173 -1
View File
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 27, 2022</h4>
<h4>Sep 29, 2022</h4>
</center> <!-- date -->
<br>
@@ -1621,6 +1621,178 @@ plt.show()
</ul>
</section>
<section>
<h2 id="improving-gradient-descent-with-momentum">Improving gradient descent with momentum </h2>
<p>We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
<span style="color: #228B22"># objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># derivative of objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># gradient descent algorithm</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size):
<span style="color: #228B22"># track all solutions</span>
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
<span style="color: #228B22"># generate an initial point</span>
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
<span style="color: #228B22"># run the gradient descent</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
<span style="color: #228B22"># calculate gradient</span>
gradient = derivative(solution)
<span style="color: #228B22"># take a step</span>
solution = solution - step_size * gradient
<span style="color: #228B22"># evaluate candidate point</span>
solution_eval = objective(solution)
<span style="color: #228B22"># store solution</span>
solutions.append(solution)
scores.append(solution_eval)
<span style="color: #228B22"># report progress</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;&gt;%d f(%s) = %.5f&#39;</span> % (i, solution, solution_eval))
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
<span style="color: #228B22"># seed the pseudo random number generator</span>
seed(<span style="color: #B452CD">4</span>)
<span style="color: #228B22"># define range for input</span>
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
<span style="color: #228B22"># define the total iterations</span>
n_iter = <span style="color: #B452CD">30</span>
<span style="color: #228B22"># define the step size</span>
step_size = <span style="color: #B452CD">0.1</span>
<span style="color: #228B22"># perform the gradient descent search</span>
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
<span style="color: #228B22"># compute targets</span>
results = objective(inputs)
<span style="color: #228B22"># create a line plot of input vs result</span>
pyplot.plot(inputs, results)
<span style="color: #228B22"># plot the solutions found</span>
pyplot.plot(solutions, scores, <span style="color: #CD5555">&#39;.-&#39;</span>, color=<span style="color: #CD5555">&#39;red&#39;</span>)
<span style="color: #228B22"># show the plot</span>
pyplot.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
<span style="color: #228B22"># objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># derivative of objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># gradient descent algorithm</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size, momentum):
<span style="color: #228B22"># track all solutions</span>
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
<span style="color: #228B22"># generate an initial point</span>
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
<span style="color: #228B22"># keep track of the change</span>
change = <span style="color: #B452CD">0.0</span>
<span style="color: #228B22"># run the gradient descent</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
<span style="color: #228B22"># calculate gradient</span>
gradient = derivative(solution)
<span style="color: #228B22"># calculate update</span>
new_change = step_size * gradient + momentum * change
<span style="color: #228B22"># take a step</span>
solution = solution - new_change
<span style="color: #228B22"># save the change</span>
change = new_change
<span style="color: #228B22"># evaluate candidate point</span>
solution_eval = objective(solution)
<span style="color: #228B22"># store solution</span>
solutions.append(solution)
scores.append(solution_eval)
<span style="color: #228B22"># report progress</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;&gt;%d f(%s) = %.5f&#39;</span> % (i, solution, solution_eval))
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
<span style="color: #228B22"># seed the pseudo random number generator</span>
seed(<span style="color: #B452CD">4</span>)
<span style="color: #228B22"># define range for input</span>
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
<span style="color: #228B22"># define the total iterations</span>
n_iter = <span style="color: #B452CD">30</span>
<span style="color: #228B22"># define the step size</span>
step_size = <span style="color: #B452CD">0.1</span>
<span style="color: #228B22"># define momentum</span>
momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #228B22"># perform the gradient descent search with momentum</span>
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
<span style="color: #228B22"># compute targets</span>
results = objective(inputs)
<span style="color: #228B22"># create a line plot of input vs result</span>
pyplot.plot(inputs, results)
<span style="color: #228B22"># plot the solutions found</span>
pyplot.plot(solutions, scores, <span style="color: #CD5555">&#39;.-&#39;</span>, color=<span style="color: #CD5555">&#39;red&#39;</span>)
<span style="color: #228B22"># show the plot</span>
pyplot.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="overview-video-on-stochastic-gradient-descent">Overview video on Stochastic Gradient Descent </h2>
doc/pub/week39/html/week39-solarized.html
+181 -1
View File
@@ -179,6 +179,14 @@ div.toc p,a {
2,
None,
'using-gradient-descent-methods-limitations'),
('Improving gradient descent with momentum',
2,
None,
'improving-gradient-descent-with-momentum'),
('Same code but now with momentum gradient descent',
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Overview video on Stochastic Gradient Descent',
2,
None,
@@ -297,7 +305,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 27, 2022</h4>
<h4>Sep 29, 2022</h4>
</center> <!-- date -->
<br>
@@ -1579,6 +1587,178 @@ plt.show()
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
</ul>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="improving-gradient-descent-with-momentum">Improving gradient descent with momentum </h2>
<p>We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.</p>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
<span style="color: #228B22"># objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># derivative of objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># gradient descent algorithm</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size):
<span style="color: #228B22"># track all solutions</span>
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
<span style="color: #228B22"># generate an initial point</span>
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
<span style="color: #228B22"># run the gradient descent</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
<span style="color: #228B22"># calculate gradient</span>
gradient = derivative(solution)
<span style="color: #228B22"># take a step</span>
solution = solution - step_size * gradient
<span style="color: #228B22"># evaluate candidate point</span>
solution_eval = objective(solution)
<span style="color: #228B22"># store solution</span>
solutions.append(solution)
scores.append(solution_eval)
<span style="color: #228B22"># report progress</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;&gt;%d f(%s) = %.5f&#39;</span> % (i, solution, solution_eval))
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
<span style="color: #228B22"># seed the pseudo random number generator</span>
seed(<span style="color: #B452CD">4</span>)
<span style="color: #228B22"># define range for input</span>
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
<span style="color: #228B22"># define the total iterations</span>
n_iter = <span style="color: #B452CD">30</span>
<span style="color: #228B22"># define the step size</span>
step_size = <span style="color: #B452CD">0.1</span>
<span style="color: #228B22"># perform the gradient descent search</span>
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
<span style="color: #228B22"># compute targets</span>
results = objective(inputs)
<span style="color: #228B22"># create a line plot of input vs result</span>
pyplot.plot(inputs, results)
<span style="color: #228B22"># plot the solutions found</span>
pyplot.plot(solutions, scores, <span style="color: #CD5555">&#39;.-&#39;</span>, color=<span style="color: #CD5555">&#39;red&#39;</span>)
<span style="color: #228B22"># show the plot</span>
pyplot.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> asarray
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">import</span> arange
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> rand
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">numpy.random</span> <span style="color: #8B008B; font-weight: bold">import</span> seed
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib</span> <span style="color: #8B008B; font-weight: bold">import</span> pyplot
<span style="color: #228B22"># objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">objective</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x**<span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># derivative of objective function</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">derivative</span>(x):
<span style="color: #8B008B; font-weight: bold">return</span> x * <span style="color: #B452CD">2.0</span>
<span style="color: #228B22"># gradient descent algorithm</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size, momentum):
<span style="color: #228B22"># track all solutions</span>
solutions, scores = <span style="color: #658b00">list</span>(), <span style="color: #658b00">list</span>()
<span style="color: #228B22"># generate an initial point</span>
solution = bounds[:, <span style="color: #B452CD">0</span>] + rand(<span style="color: #658b00">len</span>(bounds)) * (bounds[:, <span style="color: #B452CD">1</span>] - bounds[:, <span style="color: #B452CD">0</span>])
<span style="color: #228B22"># keep track of the change</span>
change = <span style="color: #B452CD">0.0</span>
<span style="color: #228B22"># run the gradient descent</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_iter):
<span style="color: #228B22"># calculate gradient</span>
gradient = derivative(solution)
<span style="color: #228B22"># calculate update</span>
new_change = step_size * gradient + momentum * change
<span style="color: #228B22"># take a step</span>
solution = solution - new_change
<span style="color: #228B22"># save the change</span>
change = new_change
<span style="color: #228B22"># evaluate candidate point</span>
solution_eval = objective(solution)
<span style="color: #228B22"># store solution</span>
solutions.append(solution)
scores.append(solution_eval)
<span style="color: #228B22"># report progress</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&#39;&gt;%d f(%s) = %.5f&#39;</span> % (i, solution, solution_eval))
<span style="color: #8B008B; font-weight: bold">return</span> [solutions, scores]
<span style="color: #228B22"># seed the pseudo random number generator</span>
seed(<span style="color: #B452CD">4</span>)
<span style="color: #228B22"># define range for input</span>
bounds = asarray([[-<span style="color: #B452CD">1.0</span>, <span style="color: #B452CD">1.0</span>]])
<span style="color: #228B22"># define the total iterations</span>
n_iter = <span style="color: #B452CD">30</span>
<span style="color: #228B22"># define the step size</span>
step_size = <span style="color: #B452CD">0.1</span>
<span style="color: #228B22"># define momentum</span>
momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #228B22"># perform the gradient descent search with momentum</span>
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
<span style="color: #228B22"># sample input range uniformly at 0.1 increments</span>
inputs = arange(bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">0</span>], bounds[<span style="color: #B452CD">0</span>,<span style="color: #B452CD">1</span>]+<span style="color: #B452CD">0.1</span>, <span style="color: #B452CD">0.1</span>)
<span style="color: #228B22"># compute targets</span>
results = objective(inputs)
<span style="color: #228B22"># create a line plot of input vs result</span>
pyplot.plot(inputs, results)
<span style="color: #228B22"># plot the solutions found</span>
pyplot.plot(solutions, scores, <span style="color: #CD5555">&#39;.-&#39;</span>, color=<span style="color: #CD5555">&#39;red&#39;</span>)
<span style="color: #228B22"># show the plot</span>
pyplot.show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="overview-video-on-stochastic-gradient-descent">Overview video on Stochastic Gradient Descent </h2>
doc/pub/week39/html/week39.html
+181 -1
View File
@@ -256,6 +256,14 @@ div.toc p,a {
2,
None,
'using-gradient-descent-methods-limitations'),
('Improving gradient descent with momentum',
2,
None,
'improving-gradient-descent-with-momentum'),
('Same code but now with momentum gradient descent',
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Overview video on Stochastic Gradient Descent',
2,
None,
@@ -374,7 +382,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 27, 2022</h4>
<h4>Sep 29, 2022</h4>
</center> <!-- date -->
<br>
@@ -1656,6 +1664,178 @@ plt<span style="color: #666666">.</span>show()
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
</ul>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="improving-gradient-descent-with-momentum">Improving gradient descent with momentum </h2>
<p>We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.</p>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">import</span> asarray
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">import</span> arange
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy.random</span> <span style="color: #008000; font-weight: bold">import</span> rand
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy.random</span> <span style="color: #008000; font-weight: bold">import</span> seed
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> pyplot
<span style="color: #408080; font-style: italic"># objective function</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">objective</span>(x):
<span style="color: #008000; font-weight: bold">return</span> x<span style="color: #666666">**2.0</span>
<span style="color: #408080; font-style: italic"># derivative of objective function</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">derivative</span>(x):
<span style="color: #008000; font-weight: bold">return</span> x <span style="color: #666666">*</span> <span style="color: #666666">2.0</span>
<span style="color: #408080; font-style: italic"># gradient descent algorithm</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size):
<span style="color: #408080; font-style: italic"># track all solutions</span>
solutions, scores <span style="color: #666666">=</span> <span style="color: #008000">list</span>(), <span style="color: #008000">list</span>()
<span style="color: #408080; font-style: italic"># generate an initial point</span>
solution <span style="color: #666666">=</span> bounds[:, <span style="color: #666666">0</span>] <span style="color: #666666">+</span> rand(<span style="color: #008000">len</span>(bounds)) <span style="color: #666666">*</span> (bounds[:, <span style="color: #666666">1</span>] <span style="color: #666666">-</span> bounds[:, <span style="color: #666666">0</span>])
<span style="color: #408080; font-style: italic"># run the gradient descent</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_iter):
<span style="color: #408080; font-style: italic"># calculate gradient</span>
gradient <span style="color: #666666">=</span> derivative(solution)
<span style="color: #408080; font-style: italic"># take a step</span>
solution <span style="color: #666666">=</span> solution <span style="color: #666666">-</span> step_size <span style="color: #666666">*</span> gradient
<span style="color: #408080; font-style: italic"># evaluate candidate point</span>
solution_eval <span style="color: #666666">=</span> objective(solution)
<span style="color: #408080; font-style: italic"># store solution</span>
solutions<span style="color: #666666">.</span>append(solution)
scores<span style="color: #666666">.</span>append(solution_eval)
<span style="color: #408080; font-style: italic"># report progress</span>
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;&gt;</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> f(</span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">) = </span><span style="color: #BB6688; font-weight: bold">%.5f</span><span style="color: #BA2121">&#39;</span> <span style="color: #666666">%</span> (i, solution, solution_eval))
<span style="color: #008000; font-weight: bold">return</span> [solutions, scores]
<span style="color: #408080; font-style: italic"># seed the pseudo random number generator</span>
seed(<span style="color: #666666">4</span>)
<span style="color: #408080; font-style: italic"># define range for input</span>
bounds <span style="color: #666666">=</span> asarray([[<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>]])
<span style="color: #408080; font-style: italic"># define the total iterations</span>
n_iter <span style="color: #666666">=</span> <span style="color: #666666">30</span>
<span style="color: #408080; font-style: italic"># define the step size</span>
step_size <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
<span style="color: #408080; font-style: italic"># perform the gradient descent search</span>
solutions, scores <span style="color: #666666">=</span> gradient_descent(objective, derivative, bounds, n_iter, step_size)
<span style="color: #408080; font-style: italic"># sample input range uniformly at 0.1 increments</span>
inputs <span style="color: #666666">=</span> arange(bounds[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>], bounds[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]<span style="color: #666666">+0.1</span>, <span style="color: #666666">0.1</span>)
<span style="color: #408080; font-style: italic"># compute targets</span>
results <span style="color: #666666">=</span> objective(inputs)
<span style="color: #408080; font-style: italic"># create a line plot of input vs result</span>
pyplot<span style="color: #666666">.</span>plot(inputs, results)
<span style="color: #408080; font-style: italic"># plot the solutions found</span>
pyplot<span style="color: #666666">.</span>plot(solutions, scores, <span style="color: #BA2121">&#39;.-&#39;</span>, color<span style="color: #666666">=</span><span style="color: #BA2121">&#39;red&#39;</span>)
<span style="color: #408080; font-style: italic"># show the plot</span>
pyplot<span style="color: #666666">.</span>show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">import</span> asarray
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">import</span> arange
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy.random</span> <span style="color: #008000; font-weight: bold">import</span> rand
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">numpy.random</span> <span style="color: #008000; font-weight: bold">import</span> seed
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">matplotlib</span> <span style="color: #008000; font-weight: bold">import</span> pyplot
<span style="color: #408080; font-style: italic"># objective function</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">objective</span>(x):
<span style="color: #008000; font-weight: bold">return</span> x<span style="color: #666666">**2.0</span>
<span style="color: #408080; font-style: italic"># derivative of objective function</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">derivative</span>(x):
<span style="color: #008000; font-weight: bold">return</span> x <span style="color: #666666">*</span> <span style="color: #666666">2.0</span>
<span style="color: #408080; font-style: italic"># gradient descent algorithm</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">gradient_descent</span>(objective, derivative, bounds, n_iter, step_size, momentum):
<span style="color: #408080; font-style: italic"># track all solutions</span>
solutions, scores <span style="color: #666666">=</span> <span style="color: #008000">list</span>(), <span style="color: #008000">list</span>()
<span style="color: #408080; font-style: italic"># generate an initial point</span>
solution <span style="color: #666666">=</span> bounds[:, <span style="color: #666666">0</span>] <span style="color: #666666">+</span> rand(<span style="color: #008000">len</span>(bounds)) <span style="color: #666666">*</span> (bounds[:, <span style="color: #666666">1</span>] <span style="color: #666666">-</span> bounds[:, <span style="color: #666666">0</span>])
<span style="color: #408080; font-style: italic"># keep track of the change</span>
change <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
<span style="color: #408080; font-style: italic"># run the gradient descent</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_iter):
<span style="color: #408080; font-style: italic"># calculate gradient</span>
gradient <span style="color: #666666">=</span> derivative(solution)
<span style="color: #408080; font-style: italic"># calculate update</span>
new_change <span style="color: #666666">=</span> step_size <span style="color: #666666">*</span> gradient <span style="color: #666666">+</span> momentum <span style="color: #666666">*</span> change
<span style="color: #408080; font-style: italic"># take a step</span>
solution <span style="color: #666666">=</span> solution <span style="color: #666666">-</span> new_change
<span style="color: #408080; font-style: italic"># save the change</span>
change <span style="color: #666666">=</span> new_change
<span style="color: #408080; font-style: italic"># evaluate candidate point</span>
solution_eval <span style="color: #666666">=</span> objective(solution)
<span style="color: #408080; font-style: italic"># store solution</span>
solutions<span style="color: #666666">.</span>append(solution)
scores<span style="color: #666666">.</span>append(solution_eval)
<span style="color: #408080; font-style: italic"># report progress</span>
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;&gt;</span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> f(</span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">) = </span><span style="color: #BB6688; font-weight: bold">%.5f</span><span style="color: #BA2121">&#39;</span> <span style="color: #666666">%</span> (i, solution, solution_eval))
<span style="color: #008000; font-weight: bold">return</span> [solutions, scores]
<span style="color: #408080; font-style: italic"># seed the pseudo random number generator</span>
seed(<span style="color: #666666">4</span>)
<span style="color: #408080; font-style: italic"># define range for input</span>
bounds <span style="color: #666666">=</span> asarray([[<span style="color: #666666">-1.0</span>, <span style="color: #666666">1.0</span>]])
<span style="color: #408080; font-style: italic"># define the total iterations</span>
n_iter <span style="color: #666666">=</span> <span style="color: #666666">30</span>
<span style="color: #408080; font-style: italic"># define the step size</span>
step_size <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
<span style="color: #408080; font-style: italic"># define momentum</span>
momentum <span style="color: #666666">=</span> <span style="color: #666666">0.3</span>
<span style="color: #408080; font-style: italic"># perform the gradient descent search with momentum</span>
solutions, scores <span style="color: #666666">=</span> gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
<span style="color: #408080; font-style: italic"># sample input range uniformly at 0.1 increments</span>
inputs <span style="color: #666666">=</span> arange(bounds[<span style="color: #666666">0</span>,<span style="color: #666666">0</span>], bounds[<span style="color: #666666">0</span>,<span style="color: #666666">1</span>]<span style="color: #666666">+0.1</span>, <span style="color: #666666">0.1</span>)
<span style="color: #408080; font-style: italic"># compute targets</span>
results <span style="color: #666666">=</span> objective(inputs)
<span style="color: #408080; font-style: italic"># create a line plot of input vs result</span>
pyplot<span style="color: #666666">.</span>plot(inputs, results)
<span style="color: #408080; font-style: italic"># plot the solutions found</span>
pyplot<span style="color: #666666">.</span>plot(solutions, scores, <span style="color: #BA2121">&#39;.-&#39;</span>, color<span style="color: #666666">=</span><span style="color: #BA2121">&#39;red&#39;</span>)
<span style="color: #408080; font-style: italic"># show the plot</span>
pyplot<span style="color: #666666">.</span>show()
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="overview-video-on-stochastic-gradient-descent">Overview video on Stochastic Gradient Descent </h2>
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/src/week39/week39.do.txt
+137 -1
View File
@@ -1064,7 +1064,143 @@ plt.show()
* _GD treats all directions in parameter space uniformly._ Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.
* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.
!split
===== Improving gradient descent with momentum =====
We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.
!bc pycod
from numpy import asarray
from numpy import arange
from numpy.random import rand
from numpy.random import seed
from matplotlib import pyplot
# objective function
def objective(x):
return x**2.0
# derivative of objective function
def derivative(x):
return x * 2.0
# gradient descent algorithm
def gradient_descent(objective, derivative, bounds, n_iter, step_size):
# track all solutions
solutions, scores = list(), list()
# generate an initial point
solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
# run the gradient descent
for i in range(n_iter):
# calculate gradient
gradient = derivative(solution)
# take a step
solution = solution - step_size * gradient
# evaluate candidate point
solution_eval = objective(solution)
# store solution
solutions.append(solution)
scores.append(solution_eval)
# report progress
print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
return [solutions, scores]
# seed the pseudo random number generator
seed(4)
# define range for input
bounds = asarray([[-1.0, 1.0]])
# define the total iterations
n_iter = 30
# define the step size
step_size = 0.1
# perform the gradient descent search
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
# sample input range uniformly at 0.1 increments
inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
# compute targets
results = objective(inputs)
# create a line plot of input vs result
pyplot.plot(inputs, results)
# plot the solutions found
pyplot.plot(solutions, scores, '.-', color='red')
# show the plot
pyplot.show()
!ec
!split
===== Same code but now with momentum gradient descent =====
!bc pycod
from numpy import asarray
from numpy import arange
from numpy.random import rand
from numpy.random import seed
from matplotlib import pyplot
# objective function
def objective(x):
return x**2.0
# derivative of objective function
def derivative(x):
return x * 2.0
# gradient descent algorithm
def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):
# track all solutions
solutions, scores = list(), list()
# generate an initial point
solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
# keep track of the change
change = 0.0
# run the gradient descent
for i in range(n_iter):
# calculate gradient
gradient = derivative(solution)
# calculate update
new_change = step_size * gradient + momentum * change
# take a step
solution = solution - new_change
# save the change
change = new_change
# evaluate candidate point
solution_eval = objective(solution)
# store solution
solutions.append(solution)
scores.append(solution_eval)
# report progress
print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
return [solutions, scores]
# seed the pseudo random number generator
seed(4)
# define range for input
bounds = asarray([[-1.0, 1.0]])
# define the total iterations
n_iter = 30
# define the step size
step_size = 0.1
# define momentum
momentum = 0.3
# perform the gradient descent search with momentum
solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
# sample input range uniformly at 0.1 increments
inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
# compute targets
results = objective(inputs)
# create a line plot of input vs result
pyplot.plot(inputs, results)
# plot the solutions found
pyplot.plot(solutions, scores, '.-', color='red')
# show the plot
pyplot.show()
!ec
!split
===== Overview video on Stochastic Gradient Descent =====