From 170dde1c94222632ec1df60ac13adff346b6ebef Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Tue, 4 Oct 2022 14:29:48 +0200 Subject: [PATCH] added rmsprop --- doc/pub/week39/html/week39-bs.html | 14 +- doc/pub/week39/html/week39-reveal.html | 83 +++ doc/pub/week39/html/week39-solarized.html | 88 +++ doc/pub/week39/html/week39.html | 88 +++ doc/pub/week39/ipynb/ipynb-week39-src.tar.gz | Bin 193 -> 193 bytes doc/pub/week39/ipynb/week39.ipynb | 642 +++++++++++-------- doc/src/week39/adagrad.py | 31 - doc/src/week39/adagradSGD.py | 5 - doc/src/week39/codes/adagradSGD.py | 50 ++ doc/src/week39/codes/rmspronoplot.py | 52 ++ doc/src/week39/rmsprop.py | 52 ++ doc/src/week39/week39.do.txt | 64 ++ 12 files changed, 848 insertions(+), 321 deletions(-) delete mode 100644 doc/src/week39/adagrad.py create mode 100644 doc/src/week39/codes/adagradSGD.py create mode 100644 doc/src/week39/codes/rmspronoplot.py create mode 100644 doc/src/week39/rmsprop.py diff --git a/doc/pub/week39/html/week39-bs.html b/doc/pub/week39/html/week39-bs.html index 011e4abdf..8da1f374f 100644 --- a/doc/pub/week39/html/week39-bs.html +++ b/doc/pub/week39/html/week39-bs.html @@ -253,6 +253,11 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'similar-second-order-function-now-problem-but-now-with-adagrad'), + ('RMSprop for adaptive learning rate with Stochastic Gradient ' + 'Descent', + 2, + None, + 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, @@ -375,9 +380,10 @@ MathJax.Hub.Config({
  • Including Stochastic Gradient Descent with Autograd
  • Same code but now with momentum gradient descent
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • And Logistic Regression
    • -
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • -
  • Weekend challenge
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And Logistic Regression
    • +
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • +
  • Weekend challenge
    • @@ -432,7 +438,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 86
    • +
  • 87
  • »
    • diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index 4c46ed246..612a3d1a6 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -3565,6 +3565,89 @@ delta = 1e-8

    Running this code we note an almost perfect agreement with the results from matrix inversion.

    +
    +

    RMSprop for adaptive learning rate with Stochastic Gradient Descent

    + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Previous value for the outer product of gradients
+        Previous = Giter
+	# Accumulated gradient
+        Giter +=gradients @ gradients.T
+	# Scaling with rho the new and the previous results
+        Gnew = (rho*Previous+(1-rho)*Giter)
+	# Taking the diagonal only and inverting
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+	# Hadamard product
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    And Logistic Regression

    diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index 57730e30e..5eceeedf7 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -280,6 +280,11 @@ div.toc p,a { 2, None, 'similar-second-order-function-now-problem-but-now-with-adagrad'), + ('RMSprop for adaptive learning rate with Stochastic Gradient ' + 'Descent', + 2, + None, + 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, @@ -3492,6 +3497,89 @@ delta = 1e-8

    Running this code we note an almost perfect agreement with the results from matrix inversion.

    +









    +

    RMSprop for adaptive learning rate with Stochastic Gradient Descent

    + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Previous value for the outer product of gradients
+        Previous = Giter
+	# Accumulated gradient
+        Giter +=gradients @ gradients.T
+	# Scaling with rho the new and the previous results
+        Gnew = (rho*Previous+(1-rho)*Giter)
+	# Taking the diagonal only and inverting
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+	# Hadamard product
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    And Logistic Regression

    diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 511b312c4..7f1b69db8 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -357,6 +357,11 @@ div.toc p,a { 2, None, 'similar-second-order-function-now-problem-but-now-with-adagrad'), + ('RMSprop for adaptive learning rate with Stochastic Gradient ' + 'Descent', + 2, + None, + 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, @@ -3569,6 +3574,89 @@ delta = 1e-8Running this code we note an almost perfect agreement with the results from matrix inversion.

    +









    +

    RMSprop for adaptive learning rate with Stochastic Gradient Descent

    + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 10000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = np.zeros(shape=(3,3))
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Previous value for the outer product of gradients
+        Previous = Giter
+	# Accumulated gradient
+        Giter +=gradients @ gradients.T
+	# Scaling with rho the new and the previous results
+        Gnew = (rho*Previous+(1-rho)*Giter)
+	# Taking the diagonal only and inverting
+        Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+	# Hadamard product
+        update = np.multiply(Ginverse,gradients)
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    And Logistic Regression

    diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 09221a7db531592c79986a567c27583ff619abcc..d0530ffbeca756a2e50fc6529c09e2800c8fa5cc 100644 GIT binary patch literal 193 zcmV;y06za8iwFP(Cp=>S1MSaC3c@fD2H>uHia9|^nxw_0U>7a~5igL^)W+JRCMnw6 z+Xv`MaZ^OdxA_@n7-kOHdb7(ScXz>J5JCy1FlL&tDN#Jv6O0*PN^pu|LJ7jUsTIG3tZ~2wR}F00;m8#M4&R literal 193 zcmV;y06za8iwFQVBRpdO1MSbv3W7io2XN0m#XN!R>Z)`M^3WlO=ml0Bb2E2sccpy$ z{D3+YT_l42UH%L+3^Rvpz1d}fy<2ZFgphij6G~&CETSn1DN7O{G>XI2 z1G3ynFP*Vm52rNM8Kpt_Zf+PW%MW|zSKyg{;#dg-+kI~