added adam, had forgotten and updated RMSprop

doc/src/week39/adam.py

@@ -0,0 +1,59 @@
+# Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)

doc/src/week39/rmsprop.py

@@ -43,16 +43,14 @@ for epoch in range(n_epochs):
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
-	# Previous value for the outer product of gradients
-        Previous = Giter
 	# Accumulated gradient
-#        Giter +=gradients @ gradients.T
 	# Scaling with rho the new and the previous results
         Giter = (rho*Giter+(1-rho)*gradients*gradients)
 	# Taking the diagonal only and inverting
         Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
 	# Hadamard product
-        update = np.multiply(Ginverse,gradients)
+        update = Ginverse*gradients
+#        update = np.multiply(Ginverse,gradients)
         theta -= update
 print("theta from own RMSprop")
 print(theta)

doc/src/week39/week39.do.txt

@@ -2550,21 +2550,82 @@ for epoch in range(n_epochs):
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
-	# Previous value for the outer product of gradients
-        Previous = Giter
-	# Accumulated gradient
-#        Giter +=gradients @ gradients.T
 	# Scaling with rho the new and the previous results
         Giter = (rho*Giter+(1-rho)*gradients*gradients)
 	# Taking the diagonal only and inverting
         Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
 	# Hadamard product
-        update = np.multiply(Ginverse,gradients)
+        update = Ginverse*gradients
         theta -= update
 print("theta from own RMSprop")
 print(theta)
+
 !ec
 
+!split
+===== And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf" =====
+
+!bc pycod
+# Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
+!ec
 
 !split
 ===== And Logistic Regression =====
@@ -2630,11 +2691,4 @@ print(derivative_fn(x_small))
 
 !ec
 
-!split
-===== Weekend challenge =====
-
-* Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.
-* Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods
-* Extend from linear regression to logistic regression.
-