updated book
This commit is contained in:
Morten Hjorth-Jensen
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -397,6 +397,7 @@ document.write(`
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-hessian-matrix-for-ridge-regression">The Hessian matrix for Ridge Regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#program-example-for-gradient-descent-with-ridge-regression">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#using-gradient-descent-methods-limitations">Using gradient descent methods, limitations</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-based-gd">Momentum based GD</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#improving-gradient-descent-with-momentum">Improving gradient descent with momentum</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview-video-on-stochastic-gradient-descent-sgd">Overview video on Stochastic Gradient Descent (SGD)</a></li>
|
||||
@@ -415,23 +416,25 @@ document.write(`
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#time-decay-rate">Time decay rate</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#replace-or-not">Replace or not</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-based-gd">Momentum based GD</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-on-momentum-based-approaches">More on momentum based approaches</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-parameter">Momentum parameter</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#second-moment-of-the-gradient">Second moment of the gradient</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#challenge-choosing-a-fixed-learning-rate">Challenge: Choosing a Fixed Learning Rate</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#motivation-for-adaptive-step-sizes">Motivation for Adaptive Step Sizes</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">AdaGrad algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#derivation-of-the-adagrad-algorithm">Derivation of the AdaGrad Algorithm</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-update-rule-derivation">AdaGrad Update Rule Derivation</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-properties">AdaGrad Properties</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rmsprop-adaptive-learning-rates">RMSProp: Adaptive Learning Rates</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-optimizer">Adam Optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rms-prop">RMS prop</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">ADAM optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">AdaGrad algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rmsprop-algorithm-taken-from-goodfellow-et-al">RMSProp algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-optimizer">Adam Optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">ADAM optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-combine-momentum-and-rmsprop">Why Combine Momentum and RMSProp?</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-exponential-moving-averages-moments">Adam: Exponential Moving Averages (Moments)</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-bias-correction">Adam: Bias Correction</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-update-rule-derivation">Adam: Update Rule Derivation</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-vs-adagrad-and-rmsprop">Adam vs. AdaGrad and RMSProp</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adaptivity-across-dimensions">Adaptivity Across Dimensions</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-algorithm-taken-from-goodfellow-et-al">ADAM algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#practical-tips">Practical tips</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#sneaking-in-automatic-differentiation-using-autograd">Sneaking in automatic differentiation using Autograd</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id3">Same code but now with momentum gradient descent</a></li>
|
||||
@@ -501,7 +504,8 @@ desirable properties such as:</p>
|
||||
<p>We revisit an example similar to what we had in the first homework set. We have a function of the type</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>x = 2*np.random.rand(m,1)
|
||||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import numpy as np
|
||||
x = 2*np.random.rand(m,1)
|
||||
y = 4+3*x+np.random.randn(m,1)
|
||||
</pre></div>
|
||||
</div>
|
||||
@@ -531,7 +535,7 @@ X \equiv \begin{bmatrix}
|
||||
1 & x_{100} & \\
|
||||
\end{bmatrix}.
|
||||
\end{split}\]</div>
|
||||
<p>The cost/loss/risk function is given by (</p>
|
||||
<p>The cost/loss/risk function is given by</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
C(\theta) = \frac{1}{n}||X\theta-\mathbf{y}||_{2}^{2} = \frac{1}{n}\sum_{i=1}^{100}\left[ (\theta_0 + \theta_1 x_i)^2 - 2 y_i (\theta_0 + \theta_1 x_i) + y_i^2\right]
|
||||
@@ -563,13 +567,13 @@ C(\theta) = \frac{1}{n}||X\theta-\mathbf{y}||_{2}^{2} = \frac{1}{n}\sum_{i=1}^{1
|
||||
</section>
|
||||
<section id="simple-program">
|
||||
<h2>Simple program<a class="headerlink" href="#simple-program" title="Link to this heading">#</a></h2>
|
||||
<p>We can now write a program that minimizes <span class="math notranslate nohighlight">\(C(\theta)\)</span> using the gradient descent method with a constant learning rate <span class="math notranslate nohighlight">\(\gamma\)</span> according to</p>
|
||||
<p>We can now write a program that minimizes <span class="math notranslate nohighlight">\(C(\theta)\)</span> using the gradient descent method with a constant learning rate <span class="math notranslate nohighlight">\(\eta\)</span> according to</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\theta_{k+1} = \theta_k - \gamma \nabla_\theta C(\theta_k), \ k=0,1,\cdots
|
||||
\theta_{k+1} = \theta_k - \eta \nabla_\theta C(\theta_k), \ k=0,1,\cdots
|
||||
\]</div>
|
||||
<p>We can use the expression we computed for the gradient and let use a
|
||||
<span class="math notranslate nohighlight">\(\theta_0\)</span> be chosen randomly and let <span class="math notranslate nohighlight">\(\gamma = 0.001\)</span>. Stop iterating
|
||||
<span class="math notranslate nohighlight">\(\theta_0\)</span> be chosen randomly and let <span class="math notranslate nohighlight">\(\eta = 0.001\)</span>. Stop iterating
|
||||
when <span class="math notranslate nohighlight">\(||\nabla_\theta C(\theta_k) || \leq \epsilon = 10^{-8}\)</span>. <strong>Note that the code below does not include the latter stop criterion</strong>.</p>
|
||||
<p>And finally we can compare our solution for <span class="math notranslate nohighlight">\(\theta\)</span> with the analytic result given by
|
||||
<span class="math notranslate nohighlight">\(\theta= (X^TX)^{-1} X^T \mathbf{y}\)</span>.</p>
|
||||
@@ -738,9 +742,15 @@ plt.show()
|
||||
<li><p>GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="momentum-based-gd">
|
||||
<h2>Momentum based GD<a class="headerlink" href="#momentum-based-gd" title="Link to this heading">#</a></h2>
|
||||
<p>We discuss here some simple examples where we introduce what is called
|
||||
‘memory’about previous steps, or what is normally called momentum
|
||||
gradient descent.
|
||||
For the mathematical details, see whiteboad notes from lecture on September 8, 2025.</p>
|
||||
</section>
|
||||
<section id="improving-gradient-descent-with-momentum">
|
||||
<h2>Improving gradient descent with momentum<a class="headerlink" href="#improving-gradient-descent-with-momentum" title="Link to this heading">#</a></h2>
|
||||
<p>We discuss here some simple examples where we introduce what is called ‘memory’about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from numpy import asarray
|
||||
@@ -961,8 +971,8 @@ function, which we want to minimize, can almost always be written as a
|
||||
sum over <span class="math notranslate nohighlight">\(n\)</span> data points <span class="math notranslate nohighlight">\(\{\mathbf{x}_i\}_{i=1}^n\)</span>,</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
C(\mathbf{\theta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\theta}).
|
||||
\]</div>
|
||||
</section>
|
||||
<section id="computation-of-gradients">
|
||||
@@ -971,8 +981,8 @@ C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
computed as a sum over <span class="math notranslate nohighlight">\(i\)</span>-gradients</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
\nabla_\theta C(\mathbf{\theta}) = \sum_i^n \nabla_\theta c_i(\mathbf{x}_i,
|
||||
\mathbf{\theta}).
|
||||
\]</div>
|
||||
<p>Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are <span class="math notranslate nohighlight">\(n\)</span>
|
||||
@@ -995,10 +1005,10 @@ all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
\nabla_{\theta}
|
||||
C(\mathbf{\theta}) = \sum_{i=1}^n \nabla_\theta c_i(\mathbf{x}_i,
|
||||
\mathbf{\theta}) \rightarrow \sum_{i \in B_k}^n \nabla_\theta
|
||||
c_i(\mathbf{x}_i, \mathbf{\theta}).
|
||||
\]</div>
|
||||
</section>
|
||||
<section id="the-gradient-step">
|
||||
@@ -1006,8 +1016,8 @@ c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
<p>Thus a gradient descent step now looks like</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
\theta_{j+1} = \theta_j - \eta_j \sum_{i \in B_k}^n \nabla_\theta c_i(\mathbf{x}_i,
|
||||
\mathbf{\theta})
|
||||
\]</div>
|
||||
<p>where <span class="math notranslate nohighlight">\(k\)</span> is picked at random with equal
|
||||
probability from <span class="math notranslate nohighlight">\([1,n/M]\)</span>. An iteration over the number of
|
||||
@@ -1055,12 +1065,12 @@ is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the <span class="math notranslate nohighlight">\(\beta\)</span> that
|
||||
compare the values of the cost function and keep the <span class="math notranslate nohighlight">\(\theta\)</span> that
|
||||
gave the lowest value.</p>
|
||||
</section>
|
||||
<section id="slightly-different-approach">
|
||||
<h2>Slightly different approach<a class="headerlink" href="#slightly-different-approach" title="Link to this heading">#</a></h2>
|
||||
<p>Another approach is to let the step length <span class="math notranslate nohighlight">\(\gamma_j\)</span> depend on the
|
||||
<p>Another approach is to let the step length <span class="math notranslate nohighlight">\(\eta_j\)</span> depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all. Such approaches are
|
||||
also called scaling. There are many such ways to <a class="reference external" href="https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1">scale the learning
|
||||
@@ -1072,11 +1082,11 @@ for a discussion of different scaling functions for the learning rate.</p>
|
||||
</section>
|
||||
<section id="time-decay-rate">
|
||||
<h2>Time decay rate<a class="headerlink" href="#time-decay-rate" title="Link to this heading">#</a></h2>
|
||||
<p>As an example, let <span class="math notranslate nohighlight">\(e = 0,1,2,3,\cdots\)</span> denote the current epoch and let <span class="math notranslate nohighlight">\(t_0, t_1 > 0\)</span> be two fixed numbers. Furthermore, let <span class="math notranslate nohighlight">\(t = e \cdot m + i\)</span> where <span class="math notranslate nohighlight">\(m\)</span> is the number of minibatches and <span class="math notranslate nohighlight">\(i=0,\cdots,m-1\)</span>. Then the function $<span class="math notranslate nohighlight">\(\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} \)</span><span class="math notranslate nohighlight">\( goes to zero as the number of epochs gets large. I.e. we start with a step length \)</span>\gamma_j (0; t_0, t_1) = t_0/t_1<span class="math notranslate nohighlight">\( which decays in *time* \)</span>t$.</p>
|
||||
<p>In this way we can fix the number of epochs, compute <span class="math notranslate nohighlight">\(\beta\)</span> and
|
||||
<p>As an example, let <span class="math notranslate nohighlight">\(e = 0,1,2,3,\cdots\)</span> denote the current epoch and let <span class="math notranslate nohighlight">\(t_0, t_1 > 0\)</span> be two fixed numbers. Furthermore, let <span class="math notranslate nohighlight">\(t = e \cdot m + i\)</span> where <span class="math notranslate nohighlight">\(m\)</span> is the number of minibatches and <span class="math notranslate nohighlight">\(i=0,\cdots,m-1\)</span>. Then the function $<span class="math notranslate nohighlight">\(\eta_j(t; t_0, t_1) = \frac{t_0}{t+t_1} \)</span><span class="math notranslate nohighlight">\( goes to zero as the number of epochs gets large. I.e. we start with a step length \)</span>\eta_j (0; t_0, t_1) = t_0/t_1<span class="math notranslate nohighlight">\( which decays in *time* \)</span>t$.</p>
|
||||
<p>In this way we can fix the number of epochs, compute <span class="math notranslate nohighlight">\(\theta\)</span> and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final <span class="math notranslate nohighlight">\(\beta\)</span> that gives the lowest value of the cost
|
||||
pick the final <span class="math notranslate nohighlight">\(\theta\)</span> that gives the lowest value of the cost
|
||||
function.</p>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
@@ -1092,18 +1102,18 @@ n_epochs = 500 #number of epochs
|
||||
t0 = 1.0
|
||||
t1 = 10
|
||||
|
||||
gamma_j = t0/t1
|
||||
eta_j = t0/t1
|
||||
j = 0
|
||||
for epoch in range(1,n_epochs+1):
|
||||
for i in range(m):
|
||||
k = np.random.randint(m) #Pick the k-th minibatch at random
|
||||
#Compute the gradient using the data in minibatch Bk
|
||||
#Compute new suggestion for beta
|
||||
#Compute new suggestion for theta
|
||||
t = epoch*m+i
|
||||
gamma_j = step_length(t,t0,t1)
|
||||
eta_j = step_length(t,t0,t1)
|
||||
j += 1
|
||||
|
||||
print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
|
||||
print("eta_j after %d epochs: %g" % (n_epochs,eta_j))
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1192,111 +1202,6 @@ mini-batches. The discussion
|
||||
<a class="reference external" href="https://sebastianraschka.com/faq/docs/sgd-methods.html">here</a> may be
|
||||
useful.</p>
|
||||
</section>
|
||||
<section id="momentum-based-gd">
|
||||
<h2>Momentum based GD<a class="headerlink" href="#momentum-based-gd" title="Link to this heading">#</a></h2>
|
||||
<p>The stochastic gradient descent (SGD) is almost always used with a
|
||||
<em>momentum</em> or inertia term that serves as a memory of the direction we
|
||||
are moving in parameter space. This is typically implemented as
|
||||
follows</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{v}_{t}=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t) \nonumber
|
||||
\]</div>
|
||||
<!-- Equation labels as ordinary links -->
|
||||
<div id="_auto1"></div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\begin{equation}
|
||||
\boldsymbol{\theta}_{t+1}= \boldsymbol{\theta}_t -\mathbf{v}_{t},
|
||||
\label{_auto1} \tag{1}
|
||||
\end{equation}
|
||||
\]</div>
|
||||
<p>where we have introduced a momentum parameter <span class="math notranslate nohighlight">\(\gamma\)</span>, with
|
||||
<span class="math notranslate nohighlight">\(0\le\gamma\le 1\)</span>, and for brevity we dropped the explicit notation to
|
||||
indicate the gradient is to be taken over a different mini-batch at
|
||||
each step. We call this algorithm gradient descent with momentum
|
||||
(GDM). From these equations, it is clear that <span class="math notranslate nohighlight">\(\mathbf{v}_t\)</span> is a
|
||||
running average of recently encountered gradients and
|
||||
<span class="math notranslate nohighlight">\((1-\gamma)^{-1}\)</span> sets the characteristic time scale for the memory
|
||||
used in the averaging procedure. Consistent with this, when
|
||||
<span class="math notranslate nohighlight">\(\gamma=0\)</span>, this just reduces down to ordinary SGD as discussed
|
||||
earlier. An equivalent way of writing the updates is</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\Delta \boldsymbol{\theta}_{t+1} = \gamma \Delta \boldsymbol{\theta}_t -\ \eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t),
|
||||
\]</div>
|
||||
<p>where we have defined <span class="math notranslate nohighlight">\(\Delta \boldsymbol{\theta}_{t}= \boldsymbol{\theta}_t-\boldsymbol{\theta}_{t-1}\)</span>.</p>
|
||||
</section>
|
||||
<section id="more-on-momentum-based-approaches">
|
||||
<h2>More on momentum based approaches<a class="headerlink" href="#more-on-momentum-based-approaches" title="Link to this heading">#</a></h2>
|
||||
<p>Let us try to get more intuition from these equations. It is helpful
|
||||
to consider a simple physical analogy with a particle of mass <span class="math notranslate nohighlight">\(m\)</span>
|
||||
moving in a viscous medium with drag coefficient <span class="math notranslate nohighlight">\(\mu\)</span> and potential
|
||||
<span class="math notranslate nohighlight">\(E(\mathbf{w})\)</span>. If we denote the particle’s position by <span class="math notranslate nohighlight">\(\mathbf{w}\)</span>,
|
||||
then its motion is described by</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
m {d^2 \mathbf{w} \over dt^2} + \mu {d \mathbf{w} \over dt }= -\nabla_w E(\mathbf{w}).
|
||||
\]</div>
|
||||
<p>We can discretize this equation in the usual way to get</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
m { \mathbf{w}_{t+\Delta t}-2 \mathbf{w}_{t} +\mathbf{w}_{t-\Delta t} \over (\Delta t)^2}+\mu {\mathbf{w}_{t+\Delta t}- \mathbf{w}_{t} \over \Delta t} = -\nabla_w E(\mathbf{w}).
|
||||
\]</div>
|
||||
<p>Rearranging this equation, we can rewrite this as</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\Delta \mathbf{w}_{t +\Delta t}= - { (\Delta t)^2 \over m +\mu \Delta t} \nabla_w E(\mathbf{w})+ {m \over m +\mu \Delta t} \Delta \mathbf{w}_t.
|
||||
\]</div>
|
||||
</section>
|
||||
<section id="momentum-parameter">
|
||||
<h2>Momentum parameter<a class="headerlink" href="#momentum-parameter" title="Link to this heading">#</a></h2>
|
||||
<p>Notice that this equation is identical to previous one if we identify
|
||||
the position of the particle, <span class="math notranslate nohighlight">\(\mathbf{w}\)</span>, with the parameters
|
||||
<span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. This allows us to identify the momentum
|
||||
parameter and learning rate with the mass of the particle and the
|
||||
viscous drag as:</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\gamma= {m \over m +\mu \Delta t }, \qquad \eta = {(\Delta t)^2 \over m +\mu \Delta t}.
|
||||
\]</div>
|
||||
<p>Thus, as the name suggests, the momentum parameter is proportional to
|
||||
the mass of the particle and effectively provides inertia.
|
||||
Furthermore, in the large viscosity/small learning rate limit, our
|
||||
memory time scales as <span class="math notranslate nohighlight">\((1-\gamma)^{-1} \approx m/(\mu \Delta t)\)</span>.</p>
|
||||
<p>Why is momentum useful? SGD momentum helps the gradient descent
|
||||
algorithm gain speed in directions with persistent but small gradients
|
||||
even in the presence of stochasticity, while suppressing oscillations
|
||||
in high-curvature directions. This becomes especially important in
|
||||
situations where the landscape is shallow and flat in some directions
|
||||
and narrow and steep in others. It has been argued that first-order
|
||||
methods (with appropriate initial conditions) can perform comparable
|
||||
to more expensive second order methods, especially in the context of
|
||||
complex deep learning models.</p>
|
||||
<p>These beneficial properties of momentum can sometimes become even more
|
||||
pronounced by using a slight modification of the classical momentum
|
||||
algorithm called Nesterov Accelerated Gradient (NAG).</p>
|
||||
<p>In the NAG algorithm, rather than calculating the gradient at the
|
||||
current parameters, <span class="math notranslate nohighlight">\(\nabla_\theta E(\boldsymbol{\theta}_t)\)</span>, one
|
||||
calculates the gradient at the expected value of the parameters given
|
||||
our current momentum, <span class="math notranslate nohighlight">\(\nabla_\theta E(\boldsymbol{\theta}_t +\gamma
|
||||
\mathbf{v}_{t-1})\)</span>. This yields the NAG update rule</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{v}_{t}=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \mathbf{v}_{t-1}) \nonumber
|
||||
\]</div>
|
||||
<!-- Equation labels as ordinary links -->
|
||||
<div id="_auto2"></div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\begin{equation}
|
||||
\boldsymbol{\theta}_{t+1}= \boldsymbol{\theta}_t -\mathbf{v}_{t}.
|
||||
\label{_auto2} \tag{2}
|
||||
\end{equation}
|
||||
\]</div>
|
||||
<p>One of the major advantages of NAG is that it allows for the use of a
|
||||
larger learning rate than GDM for the same choice of <span class="math notranslate nohighlight">\(\gamma\)</span>.</p>
|
||||
</section>
|
||||
<section id="second-moment-of-the-gradient">
|
||||
<h2>Second moment of the gradient<a class="headerlink" href="#second-moment-of-the-gradient" title="Link to this heading">#</a></h2>
|
||||
<p>In stochastic gradient descent, with and without momentum, we still
|
||||
@@ -1337,29 +1242,35 @@ For a function with steep directions and flat directions, a single global <span
|
||||
<section id="motivation-for-adaptive-step-sizes">
|
||||
<h2>Motivation for Adaptive Step Sizes<a class="headerlink" href="#motivation-for-adaptive-step-sizes" title="Link to this heading">#</a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Instead of a fixed global <span class="math notranslate nohighlight">\(\eta\)</span>, use an \textbf{adaptive learning rate} for each parameter that depends on the history of gradients.</p></li>
|
||||
<li><p>Instead of a fixed global <span class="math notranslate nohighlight">\(\eta\)</span>, use an <strong>adaptive learning rate</strong> for each parameter that depends on the history of gradients.</p></li>
|
||||
<li><p>Parameters that have large accumulated gradient magnitude should get smaller steps (they’ve been changing a lot), whereas parameters with small or infrequent gradients can have larger relative steps.</p></li>
|
||||
<li><p>This is especially useful for sparse features: Rarely active features accumulate little gradient, so their learning rate remains comparatively high, ensuring they are not neglected</p></li>
|
||||
<li><p>Conversely, frequently active features accumulate large gradient sums, and their learning rate automatically decreases, preventing too-large updates</p></li>
|
||||
<li><p>Several algorithms implement this idea (AdaGrad, RMSProp, AdaDelta, Adam, etc.). We will derive <strong>AdaGrad</strong>, one of the first adaptive methods.</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="adagrad-algorithm-taken-from-goodfellow-et-al">
|
||||
<h2>AdaGrad algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#adagrad-algorithm-taken-from-goodfellow-et-al" title="Link to this heading">#</a></h2>
|
||||
<!-- dom:FIGURE: [figures/adagrad.png, width=600 frac=0.8] -->
|
||||
<!-- begin figure -->
|
||||
<p><img src="figures/adagrad.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||||
<!-- end figure --></section>
|
||||
<section id="derivation-of-the-adagrad-algorithm">
|
||||
<h2>Derivation of the AdaGrad Algorithm<a class="headerlink" href="#derivation-of-the-adagrad-algorithm" title="Link to this heading">#</a></h2>
|
||||
<p><strong>Accumulating Gradient History.</strong></p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>AdaGrad maintains a running sum of squared gradients for each parameter (coordinate)</p></li>
|
||||
<li><p>Let <span class="math notranslate nohighlight">\(g_t = \nabla \ell_{i_t}(x_t)\)</span> be the gradient at step <span class="math notranslate nohighlight">\(t\)</span> (or a subgradient for nondifferentiable cases).</p></li>
|
||||
<li><p>Initialize <span class="math notranslate nohighlight">\(h_0 = 0\)</span> (an all-zero vector in <span class="math notranslate nohighlight">\(\mathbb{R}^d\)</span>).</p></li>
|
||||
<li><p>Let <span class="math notranslate nohighlight">\(g_t = \nabla C_{i_t}(x_t)\)</span> be the gradient at step <span class="math notranslate nohighlight">\(t\)</span> (or a subgradient for nondifferentiable cases).</p></li>
|
||||
<li><p>Initialize <span class="math notranslate nohighlight">\(r_0 = 0\)</span> (an all-zero vector in <span class="math notranslate nohighlight">\(\mathbb{R}^d\)</span>).</p></li>
|
||||
<li><p>At each iteration <span class="math notranslate nohighlight">\(t\)</span>, update the accumulation:</p></li>
|
||||
</ol>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
h_t \;=\; h_{t-1} + g_t \circ g_t,
|
||||
r_t = r_{t-1} + g_t \circ g_t,
|
||||
\]</div>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Here <span class="math notranslate nohighlight">\(g_t \circ g_t\)</span> denotes element-wise square of the gradient vector. <span class="math notranslate nohighlight">\(h_t^{(j)} = h_{t-1}^{(j)} + (g_{t,j})^2\)</span> for each parameter <span class="math notranslate nohighlight">\(j\)</span>.</p></li>
|
||||
<li><p>We can view <span class="math notranslate nohighlight">\(H_t = \mathrm{diag}(h_t)\)</span> as a diagonal matrix of past squared gradients. Initially <span class="math notranslate nohighlight">\(H_0 = 0\)</span>.</p></li>
|
||||
<li><p>Here <span class="math notranslate nohighlight">\(g_t \circ g_t\)</span> denotes element-wise square of the gradient vector. <span class="math notranslate nohighlight">\(g_t^{(j)} = g_{t-1}^{(j)} + (g_{t,j})^2\)</span> for each parameter <span class="math notranslate nohighlight">\(j\)</span>.</p></li>
|
||||
<li><p>We can view <span class="math notranslate nohighlight">\(H_t = \mathrm{diag}(r_t)\)</span> as a diagonal matrix of past squared gradients. Initially <span class="math notranslate nohighlight">\(H_0 = 0\)</span>.</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="adagrad-update-rule-derivation">
|
||||
@@ -1367,32 +1278,32 @@ h_t \;=\; h_{t-1} + g_t \circ g_t,
|
||||
<p>We scale the gradient by the inverse square root of the accumulated matrix <span class="math notranslate nohighlight">\(H_t\)</span>. The AdaGrad update at step <span class="math notranslate nohighlight">\(t\)</span> is:</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
x_{t+1} \;=\; x_t \;-\; \eta\, H_t^{-1/2}\, g_t,
|
||||
\theta_{t+1} =\theta_t - \eta H_t^{-1/2} g_t,
|
||||
\]</div>
|
||||
<p>where <span class="math notranslate nohighlight">\(H_t^{-1/2}\)</span> is the diagonal matrix with entries <span class="math notranslate nohighlight">\((h_{t}^{(1)})^{-1/2}, \dots, (h_{t}^{(d)})^{-1/2}\)</span>
|
||||
<p>where <span class="math notranslate nohighlight">\(H_t^{-1/2}\)</span> is the diagonal matrix with entries <span class="math notranslate nohighlight">\((r_{t}^{(1)})^{-1/2}, \dots, (r_{t}^{(d)})^{-1/2}\)</span>
|
||||
In coordinates, this means each parameter <span class="math notranslate nohighlight">\(j\)</span> has an individual step size:</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
x_{t+1,j} =x_{t,j} -\frac{\eta}{\sqrt{h_{t,j}}}g_{t,j}.
|
||||
\theta_{t+1,j} =\theta_{t,j} -\frac{\eta}{\sqrt{r_{t,j}}}g_{t,j}.
|
||||
\]</div>
|
||||
<p>In practice we add a small constant <span class="math notranslate nohighlight">\(\epsilon\)</span> in the denominator for numerical stability to avoid division by zero:</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
x_{t+1,j}= x_{t,j}-\frac{\eta}{\sqrt{\epsilon + h_{t,j}}}g_{t,j}.
|
||||
\theta_{t+1,j}= \theta_{t,j}-\frac{\eta}{\sqrt{\epsilon + r_{t,j}}}g_{t,j}.
|
||||
\]</div>
|
||||
<p>Equivalently, the effective learning rate for parameter <span class="math notranslate nohighlight">\(j\)</span> at time <span class="math notranslate nohighlight">\(t\)</span> is <span class="math notranslate nohighlight">\(\displaystyle \alpha_{t,j} = \frac{\eta}{\sqrt{\epsilon + h_{t,j}}}\)</span>. This decreases over time as <span class="math notranslate nohighlight">\(h_{t,j}\)</span> grows.</p>
|
||||
<p>Equivalently, the effective learning rate for parameter <span class="math notranslate nohighlight">\(j\)</span> at time <span class="math notranslate nohighlight">\(t\)</span> is <span class="math notranslate nohighlight">\(\displaystyle \alpha_{t,j} = \frac{\eta}{\sqrt{\epsilon + r_{t,j}}}\)</span>. This decreases over time as <span class="math notranslate nohighlight">\(r_{t,j}\)</span> grows.</p>
|
||||
</section>
|
||||
<section id="adagrad-properties">
|
||||
<h2>AdaGrad Properties<a class="headerlink" href="#adagrad-properties" title="Link to this heading">#</a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>AdaGrad automatically tunes the step size for each parameter. Parameters with more <strong>volatile or large gradients</strong> get smaller steps, and those with <strong>small or infrequent gradients</strong> get relatively larger steps</p></li>
|
||||
<li><p>No manual schedule needed: The accumulation <span class="math notranslate nohighlight">\(h_t\)</span> keeps increasing (or stays the same if gradient is zero), so step sizes <span class="math notranslate nohighlight">\(\eta/\sqrt{h_t}\)</span> are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.</p></li>
|
||||
<li><p>Sparse data benefit: For very sparse features, <span class="math notranslate nohighlight">\(h_{t,j}\)</span> grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal</p></li>
|
||||
<li><p>Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate (e.g. <span class="math notranslate nohighlight">\(O(1/\sqrt{T})\)</span> regret bound) comparable to the best fixed learning rate tuned for the problem</p></li>
|
||||
<li><p>AdaGrad automatically tunes the step size for each parameter. Parameters with more <em>volatile or large gradients</em> get smaller steps, and those with <em>small or infrequent gradients</em> get relatively larger steps</p></li>
|
||||
<li><p>No manual schedule needed: The accumulation <span class="math notranslate nohighlight">\(r_t\)</span> keeps increasing (or stays the same if gradient is zero), so step sizes <span class="math notranslate nohighlight">\(\eta/\sqrt{r_t}\)</span> are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.</p></li>
|
||||
<li><p>Sparse data benefit: For very sparse features, <span class="math notranslate nohighlight">\(r_{t,j}\)</span> grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal</p></li>
|
||||
<li><p>Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate comparable to the best fixed learning rate tuned for the problem</p></li>
|
||||
</ol>
|
||||
<p>It effectively reduces the need to tune <span class="math notranslate nohighlight">\(\eta\)</span> by hand.</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Limitations: Because <span class="math notranslate nohighlight">\(h_t\)</span> accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)</p></li>
|
||||
<li><p>Limitations: Because <span class="math notranslate nohighlight">\(r_t\)</span> accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="rmsprop-adaptive-learning-rates">
|
||||
@@ -1401,19 +1312,25 @@ x_{t+1,j}= x_{t,j}-\frac{\eta}{\sqrt{\epsilon + h_{t,j}}}g_{t,j}.
|
||||
Uses a decaying average of squared gradients (instead of a cumulative sum):</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
v_t = \beta_2\, v_{t-1} + (1-\beta_2)\, (\nabla L(w_t))^2,
|
||||
v_t = \rho v_{t-1} + (1-\rho)(\nabla C(\theta_t))^2,
|
||||
\]</div>
|
||||
<p>with <span class="math notranslate nohighlight">\(\beta_2\)</span> typically <span class="math notranslate nohighlight">\(0.9\)</span> (or <span class="math notranslate nohighlight">\(0.99\)</span>).</p>
|
||||
<p>with <span class="math notranslate nohighlight">\(\rho\)</span> typically <span class="math notranslate nohighlight">\(0.9\)</span> (or <span class="math notranslate nohighlight">\(0.99\)</span>).</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Update: <span class="math notranslate nohighlight">\(w_{t+1} = w_t - \frac{\alpha}{\sqrt{v_t + \epsilon}} \nabla L(w_t)\)</span>.</p></li>
|
||||
<li><p>Update: <span class="math notranslate nohighlight">\(\theta_{t+1} = \theta_t - \frac{\eta}{\sqrt{v_t + \epsilon}} \nabla C(\theta_t)\)</span>.</p></li>
|
||||
<li><p>Recent gradients have more weight, so <span class="math notranslate nohighlight">\(v_t\)</span> adapts to the current landscape.</p></li>
|
||||
<li><p>Avoids AdaGrad’s “infinite memory” problem – learning rate does not continuously decay to zero.</p></li>
|
||||
</ol>
|
||||
<p>RMSProp was first proposed in lecture notes by Geoff Hinton, 2012 - unpublished.)</p>
|
||||
</section>
|
||||
<section id="rmsprop-algorithm-taken-from-goodfellow-et-al">
|
||||
<h2>RMSProp algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#rmsprop-algorithm-taken-from-goodfellow-et-al" title="Link to this heading">#</a></h2>
|
||||
<!-- dom:FIGURE: [figures/rmsprop.png, width=600 frac=0.8] -->
|
||||
<!-- begin figure -->
|
||||
<p><img src="figures/rmsprop.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||||
<!-- end figure --></section>
|
||||
<section id="adam-optimizer">
|
||||
<h2>Adam Optimizer<a class="headerlink" href="#adam-optimizer" title="Link to this heading">#</a></h2>
|
||||
<p>Why Combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.</p>
|
||||
<p>Why combine Momentum and RMSProp? Motivation for Adam: Adaptive Moment Estimation (Adam) was introduced by Kingma an Ba (2014) to combine the benefits of momentum and RMSProp.</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Fast convergence by smoothing gradients (accelerates in long-term gradient direction).</p></li>
|
||||
<li><p>Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).</p></li>
|
||||
@@ -1422,132 +1339,108 @@ v_t = \beta_2\, v_{t-1} + (1-\beta_2)\, (\nabla L(w_t))^2,
|
||||
</ol>
|
||||
<p><strong>Result</strong>: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice.</p>
|
||||
</section>
|
||||
<section id="rms-prop">
|
||||
<h2>RMS prop<a class="headerlink" href="#rms-prop" title="Link to this heading">#</a></h2>
|
||||
<p>In RMS prop, in addition to keeping a running average of the first
|
||||
moment of the gradient, we also keep track of the second moment
|
||||
denoted by <span class="math notranslate nohighlight">\(\mathbf{s}_t=\mathbb{E}[\mathbf{g}_t^2]\)</span>. The update rule
|
||||
for RMS prop is given by</p>
|
||||
<!-- Equation labels as ordinary links -->
|
||||
<div id="_auto3"></div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\begin{equation}
|
||||
\mathbf{g}_t = \nabla_\theta E(\boldsymbol{\theta})
|
||||
\label{_auto3} \tag{3}
|
||||
\end{equation}
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{s}_t =\beta \mathbf{s}_{t-1} +(1-\beta)\mathbf{g}_t^2 \nonumber
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{\theta}_{t+1}=\boldsymbol{\theta}_t - \eta_t { \mathbf{g}_t \over \sqrt{\mathbf{s}_t +\epsilon}}, \nonumber
|
||||
\]</div>
|
||||
<p>where <span class="math notranslate nohighlight">\(\beta\)</span> controls the averaging time of the second moment and is
|
||||
typically taken to be about <span class="math notranslate nohighlight">\(\beta=0.9\)</span>, <span class="math notranslate nohighlight">\(\eta_t\)</span> is a learning rate
|
||||
typically chosen to be <span class="math notranslate nohighlight">\(10^{-3}\)</span>, and <span class="math notranslate nohighlight">\(\epsilon\sim 10^{-8} \)</span> is a
|
||||
small regularization constant to prevent divergences. Multiplication
|
||||
and division by vectors is understood as an element-wise operation. It
|
||||
is clear from this formula that the learning rate is reduced in
|
||||
directions where the norm of the gradient is consistently large. This
|
||||
greatly speeds up the convergence by allowing us to use a larger
|
||||
learning rate for flat directions.</p>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h2><a class="reference external" href="https://arxiv.org/abs/1412.6980">ADAM optimizer</a><a class="headerlink" href="#id2" title="Link to this heading">#</a></h2>
|
||||
<p>A related algorithm is the ADAM optimizer. In
|
||||
<a class="reference external" href="https://arxiv.org/abs/1412.6980">ADAM</a>, we keep a running average of
|
||||
<p>In <a class="reference external" href="https://arxiv.org/abs/1412.6980">ADAM</a>, we keep a running average of
|
||||
both the first and second moment of the gradient and use this
|
||||
information to adaptively change the learning rate for different
|
||||
parameters. The method isefficient when working with large
|
||||
parameters. The method is efficient when working with large
|
||||
problems involving lots data and/or parameters. It is a combination of the
|
||||
gradient descent with momentum algorithm and the RMSprop algorithm
|
||||
discussed above.</p>
|
||||
<p>In addition to keeping a running average of the first and
|
||||
second moments of the gradient
|
||||
(i.e. <span class="math notranslate nohighlight">\(\mathbf{m}_t=\mathbb{E}[\mathbf{g}_t]\)</span> and
|
||||
<span class="math notranslate nohighlight">\(\mathbf{s}_t=\mathbb{E}[\mathbf{g}^2_t]\)</span>, respectively), ADAM
|
||||
performs an additional bias correction to account for the fact that we
|
||||
are estimating the first two moments of the gradient using a running
|
||||
average (denoted by the hats in the update rule below). The update
|
||||
rule for ADAM is given by (where multiplication and division are once
|
||||
again understood to be element-wise operations below)</p>
|
||||
<!-- Equation labels as ordinary links -->
|
||||
<div id="_auto4"></div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\begin{equation}
|
||||
\mathbf{g}_t = \nabla_\theta E(\boldsymbol{\theta})
|
||||
\label{_auto4} \tag{4}
|
||||
\end{equation}
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{m}_t = \beta_1 \mathbf{m}_{t-1} + (1-\beta_1) \mathbf{g}_t \nonumber
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\mathbf{s}_t =\beta_2 \mathbf{s}_{t-1} +(1-\beta_2)\mathbf{g}_t^2 \nonumber
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{\mathbf{m}}_t={\mathbf{m}_t \over 1-\beta_1^t} \nonumber
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{\mathbf{s}}_t ={\mathbf{s}_t \over1-\beta_2^t} \nonumber
|
||||
\]</div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\boldsymbol{\theta}_{t+1}=\boldsymbol{\theta}_t - \eta_t { \boldsymbol{\mathbf{m}}_t \over \sqrt{\boldsymbol{\mathbf{s}}_t} +\epsilon}, \nonumber
|
||||
\]</div>
|
||||
<!-- Equation labels as ordinary links -->
|
||||
<div id="_auto5"></div>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\begin{equation}
|
||||
\label{_auto5} \tag{5}
|
||||
\end{equation}
|
||||
\]</div>
|
||||
<p>where <span class="math notranslate nohighlight">\(\beta_1\)</span> and <span class="math notranslate nohighlight">\(\beta_2\)</span> set the memory lifetime of the first and
|
||||
second moment and are typically taken to be <span class="math notranslate nohighlight">\(0.9\)</span> and <span class="math notranslate nohighlight">\(0.99\)</span>
|
||||
respectively, and <span class="math notranslate nohighlight">\(\eta\)</span> and <span class="math notranslate nohighlight">\(\epsilon\)</span> are identical to RMSprop.</p>
|
||||
<p>Like in RMSprop, the effective step size of a parameter depends on the
|
||||
magnitude of its gradient squared. To understand this better, let us
|
||||
rewrite this expression in terms of the variance
|
||||
<span class="math notranslate nohighlight">\(\boldsymbol{\sigma}_t^2 = \boldsymbol{\mathbf{s}}_t -
|
||||
(\boldsymbol{\mathbf{m}}_t)^2\)</span>. Consider a single parameter <span class="math notranslate nohighlight">\(\theta_t\)</span>. The
|
||||
update rule for this parameter is given by</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\Delta \theta_{t+1}= -\eta_t { \boldsymbol{m}_t \over \sqrt{\sigma_t^2 + m_t^2 }+\epsilon}.
|
||||
\]</div>
|
||||
</section>
|
||||
<section id="algorithms-and-codes-for-adagrad-rmsprop-and-adam">
|
||||
<h2>Algorithms and codes for Adagrad, RMSprop and Adam<a class="headerlink" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam" title="Link to this heading">#</a></h2>
|
||||
<p>The algorithms we have implemented are well described in the text by <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow, Bengio and Courville, chapter 8</a>.</p>
|
||||
<p>The codes which implement these algorithms are discussed after our presentation of automatic differentiation.</p>
|
||||
<section id="why-combine-momentum-and-rmsprop">
|
||||
<h2>Why Combine Momentum and RMSProp?<a class="headerlink" href="#why-combine-momentum-and-rmsprop" title="Link to this heading">#</a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Momentum: Fast convergence by smoothing gradients (accelerates in long-term gradient direction).</p></li>
|
||||
<li><p>Adaptive rates (RMSProp): Per-dimension learning rate scaling for stability (handles different feature scales, sparse gradients).</p></li>
|
||||
<li><p>Adam uses both: maintains moving averages of both first moment (gradients) and second moment (squared gradients)</p></li>
|
||||
<li><p>Additionally, includes a mechanism to correct the bias in these moving averages (crucial in early iterations)</p></li>
|
||||
</ol>
|
||||
<p>Result: Adam is robust, achieves faster convergence with less tuning, and often outperforms SGD (with momentum) in practice</p>
|
||||
</section>
|
||||
<section id="adam-exponential-moving-averages-moments">
|
||||
<h2>Adam: Exponential Moving Averages (Moments)<a class="headerlink" href="#adam-exponential-moving-averages-moments" title="Link to this heading">#</a></h2>
|
||||
<p>Adam maintains two moving averages at each time step <span class="math notranslate nohighlight">\(t\)</span> for each parameter <span class="math notranslate nohighlight">\(w\)</span>:
|
||||
<strong>First moment (mean) <span class="math notranslate nohighlight">\(m_t\)</span>.</strong></p>
|
||||
<p>The Momentum term</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
m_t = \beta_1m_{t-1} + (1-\beta_1)\, \nabla C(\theta_t),
|
||||
\]</div>
|
||||
<p><strong>Second moment (uncentered variance) <span class="math notranslate nohighlight">\(v_t\)</span>.</strong></p>
|
||||
<p>The RMS term</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
v_t = \beta_2v_{t-1} + (1-\beta_2)(\nabla C(\theta_t))^2,
|
||||
\]</div>
|
||||
<p>with typical <span class="math notranslate nohighlight">\(\beta_1 = 0.9\)</span>, <span class="math notranslate nohighlight">\(\beta_2 = 0.999\)</span>. Initialize <span class="math notranslate nohighlight">\(m_0 = 0\)</span>, <span class="math notranslate nohighlight">\(v_0 = 0\)</span>.</p>
|
||||
<p>These are <strong>biased</strong> estimators of the true first and second moment of the gradients, especially at the start (since <span class="math notranslate nohighlight">\(m_0,v_0\)</span> are zero)</p>
|
||||
</section>
|
||||
<section id="adam-bias-correction">
|
||||
<h2>Adam: Bias Correction<a class="headerlink" href="#adam-bias-correction" title="Link to this heading">#</a></h2>
|
||||
<p>To counteract initialization bias in <span class="math notranslate nohighlight">\(m_t, v_t\)</span>, Adam computes bias-corrected estimates</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\hat{m}_t = \frac{m_t}{1 - \beta_1^t}, \qquad \hat{v}_t = \frac{v_t}{1 - \beta_2^t}.
|
||||
\]</div>
|
||||
<ul class="simple">
|
||||
<li><p>When <span class="math notranslate nohighlight">\(t\)</span> is small, <span class="math notranslate nohighlight">\(1-\beta_i^t \approx 0\)</span>, so <span class="math notranslate nohighlight">\(\hat{m}_t, \hat{v}_t\)</span> significantly larger than raw <span class="math notranslate nohighlight">\(m_t, v_t\)</span>, compensating for the initial zero bias.</p></li>
|
||||
<li><p>As <span class="math notranslate nohighlight">\(t\)</span> increases, <span class="math notranslate nohighlight">\(1-\beta_i^t \to 1\)</span>, and <span class="math notranslate nohighlight">\(\hat{m}_t, \hat{v}_t\)</span> converge to <span class="math notranslate nohighlight">\(m_t, v_t\)</span>.</p></li>
|
||||
<li><p>Bias correction is important for Adam’s stability in early iterations</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="adam-update-rule-derivation">
|
||||
<h2>Adam: Update Rule Derivation<a class="headerlink" href="#adam-update-rule-derivation" title="Link to this heading">#</a></h2>
|
||||
<p>Finally, Adam updates parameters using the bias-corrected moments:</p>
|
||||
<div class="math notranslate nohighlight">
|
||||
\[
|
||||
\theta_{t+1} =\theta_t -\frac{\alpha}{\sqrt{\hat{v}_t} + \epsilon}\hat{m}_t,
|
||||
\]</div>
|
||||
<p>where <span class="math notranslate nohighlight">\(\epsilon\)</span> is a small constant (e.g. <span class="math notranslate nohighlight">\(10^{-8}\)</span>) to prevent division by zero.
|
||||
Breaking it down:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Compute gradient <span class="math notranslate nohighlight">\(\nabla C(\theta_t)\)</span>.</p></li>
|
||||
<li><p>Update first moment <span class="math notranslate nohighlight">\(m_t\)</span> and second moment <span class="math notranslate nohighlight">\(v_t\)</span> (exponential moving averages).</p></li>
|
||||
<li><p>Bias-correct: <span class="math notranslate nohighlight">\(\hat{m}_t = m_t/(1-\beta_1^t)\)</span>, <span class="math notranslate nohighlight">\(\; \hat{v}_t = v_t/(1-\beta_2^t)\)</span>.</p></li>
|
||||
<li><p>Compute step: <span class="math notranslate nohighlight">\(\Delta \theta_t = \frac{\hat{m}_t}{\sqrt{\hat{v}_t} + \epsilon}\)</span>.</p></li>
|
||||
<li><p>Update parameters: <span class="math notranslate nohighlight">\(\theta_{t+1} = \theta_t - \alpha\, \Delta \theta_t\)</span>.</p></li>
|
||||
</ol>
|
||||
<p>This is the Adam update rule as given in the original paper.</p>
|
||||
</section>
|
||||
<section id="adam-vs-adagrad-and-rmsprop">
|
||||
<h2>Adam vs. AdaGrad and RMSProp<a class="headerlink" href="#adam-vs-adagrad-and-rmsprop" title="Link to this heading">#</a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>AdaGrad: Uses per-coordinate scaling like Adam, but no momentum. Tends to slow down too much due to cumulative history (no forgetting)</p></li>
|
||||
<li><p>RMSProp: Uses moving average of squared gradients (like Adam’s <span class="math notranslate nohighlight">\(v_t\)</span>) to maintain adaptive learning rates, but does not include momentum or bias-correction.</p></li>
|
||||
<li><p>Adam: Effectively RMSProp + Momentum + Bias-correction</p></li>
|
||||
</ol>
|
||||
<ul class="simple">
|
||||
<li><p>Momentum (<span class="math notranslate nohighlight">\(m_t\)</span>) provides acceleration and smoother convergence.</p></li>
|
||||
<li><p>Adaptive <span class="math notranslate nohighlight">\(v_t\)</span> scaling moderates the step size per dimension.</p></li>
|
||||
<li><p>Bias correction (absent in AdaGrad/RMSProp) ensures robust estimates early on.</p></li>
|
||||
</ul>
|
||||
<p>In practice, Adam often yields faster convergence and better tuning stability than RMSProp or AdaGrad alone</p>
|
||||
</section>
|
||||
<section id="adaptivity-across-dimensions">
|
||||
<h2>Adaptivity Across Dimensions<a class="headerlink" href="#adaptivity-across-dimensions" title="Link to this heading">#</a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Adam adapts the step size \emph{per coordinate}: parameters with larger gradient variance get smaller effective steps, those with smaller or sparse gradients get larger steps.</p></li>
|
||||
<li><p>This per-dimension adaptivity is inherited from AdaGrad/RMSProp and helps handle ill-conditioned or sparse problems.</p></li>
|
||||
<li><p>Meanwhile, momentum (first moment) allows Adam to continue making progress even if gradients become small or noisy, by leveraging accumulated direction.</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="adagrad-algorithm-taken-from-goodfellow-et-al">
|
||||
<h2>AdaGrad algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#adagrad-algorithm-taken-from-goodfellow-et-al" title="Link to this heading">#</a></h2>
|
||||
<!-- dom:FIGURE: [figures/adagrad.png, width=600 frac=0.8] -->
|
||||
<!-- begin figure -->
|
||||
<p><img src="figures/adagrad.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||||
<!-- end figure --></section>
|
||||
<section id="rmsprop-algorithm-taken-from-goodfellow-et-al">
|
||||
<h2>RMSProp algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#rmsprop-algorithm-taken-from-goodfellow-et-al" title="Link to this heading">#</a></h2>
|
||||
<!-- dom:FIGURE: [figures/rmsprop.png, width=600 frac=0.8] -->
|
||||
<!-- begin figure -->
|
||||
<p><img src="figures/rmsprop.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||||
<!-- end figure --></section>
|
||||
<section id="adam-algorithm-taken-from-goodfellow-et-al">
|
||||
<h2>ADAM algorithm, taken from <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow et al</a><a class="headerlink" href="#adam-algorithm-taken-from-goodfellow-et-al" title="Link to this heading">#</a></h2>
|
||||
<!-- dom:FIGURE: [figures/adam.png, width=600 frac=0.8] -->
|
||||
<!-- begin figure -->
|
||||
<p><img src="figures/adam.png" width="600"><p style="font-size: 0.9em"><i>Figure 1: </i></p></p>
|
||||
<!-- end figure --></section>
|
||||
<section id="algorithms-and-codes-for-adagrad-rmsprop-and-adam">
|
||||
<h2>Algorithms and codes for Adagrad, RMSprop and Adam<a class="headerlink" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam" title="Link to this heading">#</a></h2>
|
||||
<p>The algorithms we have implemented are well described in the text by <a class="reference external" href="https://www.deeplearningbook.org/contents/optimization.html">Goodfellow, Bengio and Courville, chapter 8</a>.</p>
|
||||
<p>The codes which implement these algorithms are discussed below here.</p>
|
||||
</section>
|
||||
<section id="practical-tips">
|
||||
<h2>Practical tips<a class="headerlink" href="#practical-tips" title="Link to this heading">#</a></h2>
|
||||
<ul class="simple">
|
||||
@@ -1576,8 +1469,8 @@ import autograd.numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from autograd import grad
|
||||
|
||||
def CostOLS(beta):
|
||||
return (1.0/n)*np.sum((y-X @ beta)**2)
|
||||
def CostOLS(theta):
|
||||
return (1.0/n)*np.sum((y-X @ theta)**2)
|
||||
|
||||
n = 100
|
||||
x = 2*np.random.rand(n,1)
|
||||
@@ -1634,8 +1527,8 @@ import autograd.numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from autograd import grad
|
||||
|
||||
def CostOLS(beta):
|
||||
return (1.0/n)*np.sum((y-X @ beta)**2)
|
||||
def CostOLS(theta):
|
||||
return (1.0/n)*np.sum((y-X @ theta)**2)
|
||||
|
||||
n = 100
|
||||
x = 2*np.random.rand(n,1)
|
||||
@@ -1857,8 +1750,8 @@ import numpy as np
|
||||
import autograd.numpy as np
|
||||
from autograd import grad
|
||||
|
||||
def CostOLS(beta):
|
||||
return (1.0/n)*np.sum((y-X @ beta)**2)
|
||||
def CostOLS(theta):
|
||||
return (1.0/n)*np.sum((y-X @ theta)**2)
|
||||
|
||||
n = 100
|
||||
x = 2*np.random.rand(n,1)
|
||||
@@ -1866,24 +1759,24 @@ y = 4+3*x+5*x*x
|
||||
|
||||
X = np.c_[np.ones((n,1)), x, x*x]
|
||||
XT_X = X.T @ X
|
||||
beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
||||
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
|
||||
print("Own inversion")
|
||||
print(beta_linreg)
|
||||
print(theta_linreg)
|
||||
# Hessian matrix
|
||||
H = (2.0/n)* XT_X
|
||||
# Note that here the Hessian does not depend on the parameters beta
|
||||
# Note that here the Hessian does not depend on the parameters theta
|
||||
invH = np.linalg.pinv(H)
|
||||
beta = np.random.randn(3,1)
|
||||
theta = np.random.randn(3,1)
|
||||
Niterations = 5
|
||||
# define the gradient
|
||||
training_gradient = grad(CostOLS)
|
||||
|
||||
for iter in range(Niterations):
|
||||
gradients = training_gradient(beta)
|
||||
beta -= invH @ gradients
|
||||
gradients = training_gradient(theta)
|
||||
theta -= invH @ gradients
|
||||
print(iter,gradients[0],gradients[1])
|
||||
print("beta from own Newton code")
|
||||
print(beta)
|
||||
print("theta from own Newton code")
|
||||
print(theta)
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -2048,9 +1941,9 @@ theta = np.random.randn(3,1)
|
||||
|
||||
# Value for learning rate
|
||||
eta = 0.01
|
||||
# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
|
||||
beta1 = 0.9
|
||||
beta2 = 0.999
|
||||
# Value for parameters theta1 and theta2, see https://arxiv.org/abs/1412.6980
|
||||
theta1 = 0.9
|
||||
theta2 = 0.999
|
||||
# Including AdaGrad parameter to avoid possible division by zero
|
||||
delta = 1e-7
|
||||
iter = 0
|
||||
@@ -2064,10 +1957,10 @@ for epoch in range(n_epochs):
|
||||
yi = y[random_index:random_index+M]
|
||||
gradients = (1.0/M)*training_gradient(yi, xi, theta)
|
||||
# Computing moments first
|
||||
first_moment = beta1*first_moment + (1-beta1)*gradients
|
||||
second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
|
||||
first_term = first_moment/(1.0-beta1**iter)
|
||||
second_term = second_moment/(1.0-beta2**iter)
|
||||
first_moment = theta1*first_moment + (1-theta1)*gradients
|
||||
second_moment = theta2*second_moment+(1-theta2)*gradients*gradients
|
||||
first_term = first_moment/(1.0-theta1**iter)
|
||||
second_term = second_moment/(1.0-theta2**iter)
|
||||
# Scaling with rho the new and the previous results
|
||||
update = eta*first_term/(np.sqrt(second_term)+delta)
|
||||
theta -= update
|
||||
@@ -2085,9 +1978,7 @@ print(theta)
|
||||
<li><p>Work on project 1</p></li>
|
||||
</ol>
|
||||
<!-- * [Video of exercise sessions week 37](https://youtu.be/bK4AEcTu-oM) -->
|
||||
<ul class="simple">
|
||||
<li><p>For more discussions of Ridge regression and calculation of averages, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.</p></li>
|
||||
</ul>
|
||||
<p>For more discussions of Ridge regression and calculation of averages, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.</p>
|
||||
</section>
|
||||
<section id="reminder-on-different-scaling-methods">
|
||||
<h2>Reminder on different scaling methods<a class="headerlink" href="#reminder-on-different-scaling-methods" title="Link to this heading">#</a></h2>
|
||||
@@ -2650,6 +2541,7 @@ centered matrix and/or vector that enter the fitting procedure.</p>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-hessian-matrix-for-ridge-regression">The Hessian matrix for Ridge Regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#program-example-for-gradient-descent-with-ridge-regression">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#using-gradient-descent-methods-limitations">Using gradient descent methods, limitations</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-based-gd">Momentum based GD</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#improving-gradient-descent-with-momentum">Improving gradient descent with momentum</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#same-code-but-now-with-momentum-gradient-descent">Same code but now with momentum gradient descent</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overview-video-on-stochastic-gradient-descent-sgd">Overview video on Stochastic Gradient Descent (SGD)</a></li>
|
||||
@@ -2668,23 +2560,25 @@ centered matrix and/or vector that enter the fitting procedure.</p>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#time-decay-rate">Time decay rate</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#replace-or-not">Replace or not</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-based-gd">Momentum based GD</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-on-momentum-based-approaches">More on momentum based approaches</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#momentum-parameter">Momentum parameter</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#second-moment-of-the-gradient">Second moment of the gradient</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#challenge-choosing-a-fixed-learning-rate">Challenge: Choosing a Fixed Learning Rate</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#motivation-for-adaptive-step-sizes">Motivation for Adaptive Step Sizes</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">AdaGrad algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#derivation-of-the-adagrad-algorithm">Derivation of the AdaGrad Algorithm</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-update-rule-derivation">AdaGrad Update Rule Derivation</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-properties">AdaGrad Properties</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rmsprop-adaptive-learning-rates">RMSProp: Adaptive Learning Rates</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-optimizer">Adam Optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rms-prop">RMS prop</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">ADAM optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adagrad-algorithm-taken-from-goodfellow-et-al">AdaGrad algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rmsprop-algorithm-taken-from-goodfellow-et-al">RMSProp algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-optimizer">Adam Optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">ADAM optimizer</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-combine-momentum-and-rmsprop">Why Combine Momentum and RMSProp?</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-exponential-moving-averages-moments">Adam: Exponential Moving Averages (Moments)</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-bias-correction">Adam: Bias Correction</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-update-rule-derivation">Adam: Update Rule Derivation</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-vs-adagrad-and-rmsprop">Adam vs. AdaGrad and RMSProp</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adaptivity-across-dimensions">Adaptivity Across Dimensions</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adam-algorithm-taken-from-goodfellow-et-al">ADAM algorithm, taken from Goodfellow et al</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#algorithms-and-codes-for-adagrad-rmsprop-and-adam">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#practical-tips">Practical tips</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#sneaking-in-automatic-differentiation-using-autograd">Sneaking in automatic differentiation using Autograd</a></li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id3">Same code but now with momentum gradient descent</a></li>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+652
-904
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user