Update adagrad.py

doc/src/week39/adagrad.py

@@ -1,4 +1,4 @@
-# Using Autograd to calculate gradients using SGD
+# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
 # OLS example
 from random import random, seed
 import numpy as np
@@ -10,42 +10,33 @@ from autograd import grad
 def CostOLS(y,X,theta):
     return np.sum((y-X @ theta)**2)
 
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
 
-X = np.c_[np.ones((n,1)), x]
+X = np.c_[np.ones((n,1)), x, x*x]
 XT_X = X.T @ X
 theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
 print("Own inversion")
 print(theta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
 
-theta = np.random.randn(2,1)
-eta = 1.0/np.max(EigValues)
-Niterations = 100
 
 # Note that we request the derivative wrt third argument (theta, 2 here)
 training_gradient = grad(CostOLS,2)
-
-for iter in range(Niterations):
-    gradients = (1.0/n)*training_gradient(y, X, theta)
-    theta -= eta*gradients
-print("theta from own gd")
-print(theta)
-print(np.size(gradients))
-
+# Define parameters for Stochastic Gradient Descent
 n_epochs = 50
 M = 5   #size of each minibatch
 m = int(n/M) #number of minibatches
-theta = np.random.randn(2,1)
-
-# Including AdaGrad
-delta  = 0.000001
-r = [0.0 for _ in range(gradients.shape[0])]
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+gradients = np.zeros(theta.shape)
+r = gradients*gradients
+print(r.shape)
+print(gradients.shape)
+# Value for learning rate
+eta = 0.01
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
 for epoch in range(n_epochs):
     for i in range(m):
         random_index = M*np.random.randint(m)
@@ -53,7 +44,8 @@ for epoch in range(n_epochs):
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
         # calculate squared gradient by Hadamard multiplication
-        r -= gradients*gradients
+#        r += (gradients*gradients)
+        r = np.sum(gradients*gradients)        
         # compute update
         update = 1.0/(delta+np.sqrt(r))*gradients
         theta -= eta*update
@@ -62,6 +54,3 @@ print(theta)
 
 
 
-
-
-