added rmsprop

doc/src/week39/adagrad.py

@@ -1,31 +0,0 @@
-# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
-# OLS example
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-
-n = 10000
-x = np.random.rand(n,1)
-y = 4*x+3*x*x
-# Setting up Design matrix
-X = np.c_[np.ones((n,1)), x, x*x]
-XTX = X.T @ X
-XTy = X.T @ y
-theta_linreg = np.linalg.pinv(XTX) @ (XTy)
-print("Own inversion")
-print(theta_linreg)
-
-
-beta = np.random.randn(3,1)
-eta = 0.01
-delta = 1e-8
-Niterations = 10000
-Giter = np.zeros(shape=(3,3))
-for iter in range(Niterations):
-    gradient = (2.0/n)*(XTX @ beta - XTy)
-    Giter +=gradient @ gradient.T
-    Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
-    beta -= np.multiply(Ginverse,gradient)
-
-print("Optimal parameters with AdaGrad",beta)

doc/src/week39/adagradSGD.py

@@ -43,11 +43,6 @@ for epoch in range(n_epochs):
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
         Giter +=gradients @ gradients.T
         Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
-
-        # calculate squared gradient by Hadamard multiplication
-#        r += (gradients*gradients)
-#        r = np.sum(gradients*gradients)        
-        # compute update
         update = np.multiply(Ginverse,gradients)
         theta -= update
 print("theta from own AdaGrad")

doc/src/week39/codes/adagradSGD.py

@@ -0,0 +1,50 @@
+# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        Giter +=gradients @ gradients.T
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own AdaGrad")
+print(theta)
+

doc/src/week39/codes/rmspronoplot.py

@@ -0,0 +1,52 @@
+# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        Previous = Giter
+        Giter +=gradients @ gradients.T
+        Gnew = (rho*Previous+(1-rho)*Giter)
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own AdaGrad")
+print(theta)

doc/src/week39/rmsprop.py

@@ -0,0 +1,52 @@
+# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        Previous = Giter
+        Giter +=gradients @ gradients.T
+        Gnew = (rho*Previous+(1-rho)*Giter)
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own AdaGrad")
+print(theta)

doc/src/week39/week39.do.txt

@@ -2491,6 +2491,70 @@ print(theta)
 
 Running this code we note an almost perfect agreement with the results from matrix inversion.
 
+!split
+=====  RMSprop for adaptive learning rate with Stochastic Gradient Descent =====
+!bc pycod
+# Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Previous value for the outer product of gradients
+        Previous = Giter
+	# Accumulated gradient
+        Giter +=gradients @ gradients.T
+	# Scaling with rho the new and the previous results
+        Gnew = (rho*Previous+(1-rho)*Giter)
+	# Taking the diagonal only and inverting
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+	# Hadamard product
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+!ec
+
+
 !split
 ===== And Logistic Regression =====