update

doc/src/week39/codes/AdamSGD.py

@@ -1,4 +1,4 @@
-# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# Using Autograd to calculate gradients using Adam and Stochastic Gradient descent
 # OLS example
 from random import random, seed
 import numpy as np
@@ -32,19 +32,30 @@ theta = np.random.randn(3,1)
 
 # Value for learning rate
 eta = 0.01
+rho1 = 0.9
+rho2 = 0.99
 # Including AdaGrad parameter to avoid possible division by zero
 delta  = 1e-8
 for epoch in range(n_epochs):
     Giter = np.zeros(shape=(3,3))
+    t = 0
+    s = np.zeros(shape=(3,1))
     for i in range(m):
         random_index = M*np.random.randint(m)
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        t += 1
+        Previous = Giter
         Giter +=gradients @ gradients.T
-        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
-        update = np.multiply(Ginverse,gradients)
-        theta -= update
-print("theta from own AdaGrad")
+        Gnew = (rho2*Previous+(1-rho2)*Giter)
+        Gnew = Gnew#/(1.0-rho2*t)
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+        s += rho1*s+(1-rho1)*gradients
+#        snew = s/(1.0-rho1*t)
+        theta -= Ginverse.T @ s
+print("theta from own Adam")
 print(theta)
 
+
+from math import sqrt

doc/src/week39/codes/rmspropSGD.py

@@ -1,4 +1,4 @@
-# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent
 # OLS example
 from random import random, seed
 import numpy as np
@@ -48,5 +48,5 @@ for epoch in range(n_epochs):
         Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
         update = np.multiply(Ginverse,gradients)
         theta -= update
-print("theta from own AdaGrad")
+print("theta from own RMSprop")
 print(theta)

doc/src/week39/week39.do.txt

@@ -1676,10 +1676,16 @@ learning rate for flat directions.
 !split
 ===== "ADAM optimizer":"https://arxiv.org/abs/1412.6980" =====
 
-A related algorithm is the ADAM optimizer. In "ADAM":"https://arxiv.org/abs/1412.6980", we keep a running
-average of both the first and second moment of the gradient and use
-this information to adaptively change the learning rate for different
-parameters. In addition to keeping a running average of the first and
+A related algorithm is the ADAM optimizer. In
+"ADAM":"https://arxiv.org/abs/1412.6980", we keep a running average of
+both the first and second moment of the gradient and use this
+information to adaptively change the learning rate for different
+parameters.  The method isefficient when working with large
+problems involving lots data and/or parameters.  It is a combination of the
+gradient descent with momentum algorithm and the RMSprop algorithm
+discussed above.
+
+In addition to keeping a running average of the first and
 second moments of the gradient
 (i.e. $\mathbf{m}_t=\mathbb{E}[\mathbf{g}_t]$ and
 $\mathbf{s}_t=\mathbb{E}[\mathbf{g}^2_t]$, respectively), ADAM
@@ -1717,7 +1723,12 @@ update rule for this parameter is given by
 \]
 !et
 
+!split
+===== Algorithms and codes for Adagrad, RMSprop and Adam =====
 
+The algorithms we have implemented are well described in the text by "Goodfellow, Bengio and Courville, chapter 8":"https://www.deeplearningbook.org/contents/optimization.html".
+
+The codes which implement these algorithms are discussed after our presentation of automatic differentiation.
 
 
 !split