From 9bb691b916b326401e27b3a342799f1ebb5b113e Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Mon, 31 Oct 2022 16:26:24 +0100 Subject: [PATCH] added adam, had forgotten and updated RMSprop --- doc/pub/week39/html/week39-bs.html | 13 +- doc/pub/week39/html/week39-reveal.html | 101 ++- doc/pub/week39/html/week39-solarized.html | 106 ++- doc/pub/week39/html/week39.html | 106 ++- doc/pub/week39/ipynb/ipynb-week39-src.tar.gz | Bin 193 -> 192 bytes doc/pub/week39/ipynb/week39.ipynb | 669 ++++++++++--------- doc/src/week39/adam.py | 59 ++ doc/src/week39/rmsprop.py | 6 +- doc/src/week39/week39.do.txt | 78 ++- 9 files changed, 768 insertions(+), 370 deletions(-) create mode 100644 doc/src/week39/adam.py diff --git a/doc/pub/week39/html/week39-bs.html b/doc/pub/week39/html/week39-bs.html index 3b9e06844..03f5751cc 100644 --- a/doc/pub/week39/html/week39-bs.html +++ b/doc/pub/week39/html/week39-bs.html @@ -262,12 +262,15 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), + ('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"', + 2, + None, + 'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, None, - 'introducing-jax-https-jax-readthedocs-io-en-latest'), - ('Weekend challenge', 2, None, 'weekend-challenge')]} + 'introducing-jax-https-jax-readthedocs-io-en-latest')]} end of tocinfo --> @@ -386,9 +389,9 @@ MathJax.Hub.Config({
  • Same code but now with momentum gradient descent
  • Similar (second order function now) problem but now with AdaGrad
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And Logistic Regression
    • -
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • -
  • Weekend challenge
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • And Logistic Regression
    • +
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index 426497b98..4e0ff1b3e 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -3639,16 +3639,12 @@ delta = 1e-8 xi = X[random_index:random_index+M] yi = y[random_index:random_index+M] gradients = (1.0/M)*training_gradient(yi, xi, theta) - # Previous value for the outer product of gradients - Previous = Giter - # Accumulated gradient -# Giter +=gradients @ gradients.T # Scaling with rho the new and the previous results Giter = (rho*Giter+(1-rho)*gradients*gradients) # Taking the diagonal only and inverting Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))] # Hadamard product - update = np.multiply(Ginverse,gradients) + update = Ginverse*gradients theta -= update print("theta from own RMSprop") print(theta) @@ -3668,6 +3664,91 @@ delta = 1e-8 +
    +

    And finally ADAM

    + + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    And Logistic Regression

    @@ -3771,16 +3852,6 @@ derivative_fn = grad(sum_logistic)
    -
    -

    Weekend challenge

    - - -
    - diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index da9a76f76..b917a62e8 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -289,12 +289,15 @@ div.toc p,a { 2, None, 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), + ('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"', + 2, + None, + 'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, None, - 'introducing-jax-https-jax-readthedocs-io-en-latest'), - ('Weekend challenge', 2, None, 'weekend-challenge')]} + 'introducing-jax-https-jax-readthedocs-io-en-latest')]} end of tocinfo --> @@ -3572,16 +3575,12 @@ delta = 1e-8 xi = X[random_index:random_index+M] yi = y[random_index:random_index+M] gradients = (1.0/M)*training_gradient(yi, xi, theta) - # Previous value for the outer product of gradients - Previous = Giter - # Accumulated gradient -# Giter +=gradients @ gradients.T # Scaling with rho the new and the previous results Giter = (rho*Giter+(1-rho)*gradients*gradients) # Taking the diagonal only and inverting Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))] # Hadamard product - update = np.multiply(Ginverse,gradients) + update = Ginverse*gradients theta -= update print("theta from own RMSprop") print(theta) @@ -3601,6 +3600,91 @@ delta = 1e-8 +









    +

    And finally ADAM

    + + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    And Logistic Regression

    @@ -3704,14 +3788,6 @@ derivative_fn = grad(sum_logistic) -









    -

    Weekend challenge

    - -
    © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 1f134d2a7..fd81e7f3c 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -366,12 +366,15 @@ div.toc p,a { 2, None, 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), + ('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"', + 2, + None, + 'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'), ('And Logistic Regression', 2, None, 'and-logistic-regression'), ('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"', 2, None, - 'introducing-jax-https-jax-readthedocs-io-en-latest'), - ('Weekend challenge', 2, None, 'weekend-challenge')]} + 'introducing-jax-https-jax-readthedocs-io-en-latest')]} end of tocinfo --> @@ -3649,16 +3652,12 @@ delta = 1e-8= X[random_index:random_index+M] yi = y[random_index:random_index+M] gradients = (1.0/M)*training_gradient(yi, xi, theta) - # Previous value for the outer product of gradients - Previous = Giter - # Accumulated gradient -# Giter +=gradients @ gradients.T # Scaling with rho the new and the previous results Giter = (rho*Giter+(1-rho)*gradients*gradients) # Taking the diagonal only and inverting Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))] # Hadamard product - update = np.multiply(Ginverse,gradients) + update = Ginverse*gradients theta -= update print("theta from own RMSprop") print(theta) @@ -3678,6 +3677,91 @@ delta = 1e-8 +









    +

    And finally ADAM

    + + + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    And Logistic Regression

    @@ -3781,14 +3865,6 @@ derivative_fn = grad(sum_logistic) -









    -

    Weekend challenge

    - -
      -
    • Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.
      • -
    • Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods
      • -
    • Extend from linear regression to logistic regression.
      • -
    © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index cc1d3ae5d043a32f3d7a6ac4c9b991ec46210d25..594eee53119c0b2adbfd24ce5f38324e13e9850c 100644 GIT binary patch literal 192 zcmV;x06+g9iwFP->0e_21MSaC3c@fD2H>uHia9|^nm%wT*o6y0#0#W!YGZ9ulN9ak z?E`eBxG5s!+x&$22{VUmz1d}fy<2ZFgpeePV9Yd$rzGM0o=_SCjaZnbLyb}(l*KIa zfGoGtOJ^+C!zxX6MrlyK>kYNC{P53w3Ow^q94ld9yYFqKBuHg3SE_~^V$D{8XnUDM uq0o#Y(0J{HM&PmsUKGL#CHci~wK{3un85$_F^=On&et9c)n{1%2mk=t`B`}Y literal 193 zcmV;y06za8iwFQMxL;!c1MSaC3c@fD2H>uHia9|^nxw_0U>7a~5igL^)W+JRCMnw6 z+Xv`MaZ^OdxA_@n7-kOHdb7(ScXz>J5JCy1FlL&tDN#Jv6O0*PO5!vn6fp(}6Gm77 zXt|SKI_tO@PHC#MP*$jSbHi9!e%Ld=0?+&t$4Xk*?t52h1xh=SYhA-lh?=Y-+4d@j vLJK>#z=&%njR3BC;6)*w)QVri*65Srjg7)zKjV3x=Y8z~-|XS500;m8E%;W^ diff --git a/doc/pub/week39/ipynb/week39.ipynb b/doc/pub/week39/ipynb/week39.ipynb index a241169a5..6057b1837 100644 --- a/doc/pub/week39/ipynb/week39.ipynb +++ b/doc/pub/week39/ipynb/week39.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "583ac34f", + "id": "fb5f2222", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "7f768498", + "id": "7626a4e2", "metadata": { "editable": true }, @@ -29,7 +29,7 @@ }, { "cell_type": "markdown", - "id": "bccd38a9", + "id": "4c820ce4", "metadata": { "editable": true }, @@ -57,7 +57,7 @@ }, { "cell_type": "markdown", - "id": "1415ce8c", + "id": "a226bde2", "metadata": { "editable": true }, @@ -78,7 +78,7 @@ }, { "cell_type": "markdown", - "id": "1682f0f0", + "id": "080fefe2", "metadata": { "editable": true }, @@ -95,7 +95,7 @@ }, { "cell_type": "markdown", - "id": "22720198", + "id": "9be084a5", "metadata": { "editable": true }, @@ -110,7 +110,7 @@ }, { "cell_type": "markdown", - "id": "82b4e195", + "id": "f05fc9f2", "metadata": { "editable": true }, @@ -120,7 +120,7 @@ }, { "cell_type": "markdown", - "id": "2e5ded0f", + "id": "8d31a5e1", "metadata": { "editable": true }, @@ -136,7 +136,7 @@ }, { "cell_type": "markdown", - "id": "4c31ba97", + "id": "4d08c022", "metadata": { "editable": true }, @@ -148,7 +148,7 @@ }, { "cell_type": "markdown", - "id": "75e132d2", + "id": "40d8474b", "metadata": { "editable": true }, @@ -159,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "a0fd914f", + "id": "91d36d0b", "metadata": { "editable": true }, @@ -171,7 +171,7 @@ }, { "cell_type": "markdown", - "id": "666409d6", + "id": "349f8798", "metadata": { "editable": true }, @@ -181,7 +181,7 @@ }, { "cell_type": "markdown", - "id": "23e12354", + "id": "bdd74730", "metadata": { "editable": true }, @@ -195,7 +195,7 @@ }, { "cell_type": "markdown", - "id": "e711613c", + "id": "16ec380a", "metadata": { "editable": true }, @@ -207,7 +207,7 @@ }, { "cell_type": "markdown", - "id": "e0da6e71", + "id": "1d2847b3", "metadata": { "editable": true }, @@ -217,7 +217,7 @@ }, { "cell_type": "markdown", - "id": "c8d5e4e0", + "id": "5083c393", "metadata": { "editable": true }, @@ -229,7 +229,7 @@ }, { "cell_type": "markdown", - "id": "7c307d5b", + "id": "8cc7ae75", "metadata": { "editable": true }, @@ -241,7 +241,7 @@ }, { "cell_type": "markdown", - "id": "96cac52e", + "id": "0c704002", "metadata": { "editable": true }, @@ -261,7 +261,7 @@ }, { "cell_type": "markdown", - "id": "f14922a3", + "id": "0979b7e5", "metadata": { "editable": true }, @@ -277,7 +277,7 @@ }, { "cell_type": "markdown", - "id": "83676afb", + "id": "4fa81efe", "metadata": { "editable": true }, @@ -293,7 +293,7 @@ }, { "cell_type": "markdown", - "id": "54afff8c", + "id": "a72f0602", "metadata": { "editable": true }, @@ -304,7 +304,7 @@ }, { "cell_type": "markdown", - "id": "ce1efd26", + "id": "312e71a3", "metadata": { "editable": true }, @@ -316,7 +316,7 @@ }, { "cell_type": "markdown", - "id": "587d4556", + "id": "e212d683", "metadata": { "editable": true }, @@ -326,7 +326,7 @@ }, { "cell_type": "markdown", - "id": "f5b04720", + "id": "c61d2ec6", "metadata": { "editable": true }, @@ -338,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "d084b40b", + "id": "e4e17d1e", "metadata": { "editable": true }, @@ -348,7 +348,7 @@ }, { "cell_type": "markdown", - "id": "4ac0298a", + "id": "a321226a", "metadata": { "editable": true }, @@ -360,7 +360,7 @@ }, { "cell_type": "markdown", - "id": "20ca5b5b", + "id": "7fbca8b5", "metadata": { "editable": true }, @@ -382,7 +382,7 @@ }, { "cell_type": "markdown", - "id": "6c9d98dc", + "id": "a5c5f928", "metadata": { "editable": true }, @@ -395,7 +395,7 @@ }, { "cell_type": "markdown", - "id": "b5f4a3b9", + "id": "dda8547c", "metadata": { "editable": true }, @@ -408,7 +408,7 @@ }, { "cell_type": "markdown", - "id": "7216495f", + "id": "a9c136b2", "metadata": { "editable": true }, @@ -418,7 +418,7 @@ }, { "cell_type": "markdown", - "id": "3e3b0da1", + "id": "3dfb3256", "metadata": { "editable": true }, @@ -436,7 +436,7 @@ }, { "cell_type": "markdown", - "id": "b818c377", + "id": "c9646465", "metadata": { "editable": true }, @@ -446,7 +446,7 @@ }, { "cell_type": "markdown", - "id": "802fdbb2", + "id": "43695470", "metadata": { "editable": true }, @@ -461,7 +461,7 @@ }, { "cell_type": "markdown", - "id": "5d6e301d", + "id": "7aec92b8", "metadata": { "editable": true }, @@ -471,7 +471,7 @@ }, { "cell_type": "markdown", - "id": "5f915b85", + "id": "5aacc2d2", "metadata": { "editable": true }, @@ -485,7 +485,7 @@ }, { "cell_type": "markdown", - "id": "faf0adb4", + "id": "c1b231de", "metadata": { "editable": true }, @@ -495,7 +495,7 @@ }, { "cell_type": "markdown", - "id": "9ebe1dcc", + "id": "3a58969a", "metadata": { "editable": true }, @@ -509,7 +509,7 @@ }, { "cell_type": "markdown", - "id": "83db9607", + "id": "8ec5ee80", "metadata": { "editable": true }, @@ -524,7 +524,7 @@ }, { "cell_type": "markdown", - "id": "21cf9b71", + "id": "67d0192e", "metadata": { "editable": true }, @@ -541,7 +541,7 @@ }, { "cell_type": "markdown", - "id": "38ecb654", + "id": "c5413071", "metadata": { "editable": true }, @@ -553,7 +553,7 @@ }, { "cell_type": "markdown", - "id": "a9ea135f", + "id": "ad4cbada", "metadata": { "editable": true }, @@ -567,7 +567,7 @@ }, { "cell_type": "markdown", - "id": "0dd6ca78", + "id": "7f1d1d0c", "metadata": { "editable": true }, @@ -582,7 +582,7 @@ }, { "cell_type": "markdown", - "id": "4746eef0", + "id": "1ec87dad", "metadata": { "editable": true }, @@ -594,7 +594,7 @@ }, { "cell_type": "markdown", - "id": "8142e86d", + "id": "36d8f48f", "metadata": { "editable": true }, @@ -605,7 +605,7 @@ }, { "cell_type": "markdown", - "id": "bb6aa138", + "id": "af64d3f3", "metadata": { "editable": true }, @@ -633,7 +633,7 @@ }, { "cell_type": "markdown", - "id": "377a7c58", + "id": "bb5e0f73", "metadata": { "editable": true }, @@ -655,7 +655,7 @@ }, { "cell_type": "markdown", - "id": "b1532ef1", + "id": "1a121737", "metadata": { "editable": true }, @@ -677,7 +677,7 @@ }, { "cell_type": "markdown", - "id": "ebcfcc2a", + "id": "01601485", "metadata": { "editable": true }, @@ -689,7 +689,7 @@ }, { "cell_type": "markdown", - "id": "568ce124", + "id": "8d354c77", "metadata": { "editable": true }, @@ -726,7 +726,7 @@ }, { "cell_type": "markdown", - "id": "73988f05", + "id": "a64e34a9", "metadata": { "editable": true }, @@ -754,7 +754,7 @@ }, { "cell_type": "markdown", - "id": "32029fd7", + "id": "388506dc", "metadata": { "editable": true }, @@ -784,7 +784,7 @@ }, { "cell_type": "markdown", - "id": "37adc473", + "id": "d6c3cefb", "metadata": { "editable": true }, @@ -804,7 +804,7 @@ }, { "cell_type": "markdown", - "id": "197f049d", + "id": "bb8a0c06", "metadata": { "editable": true }, @@ -816,7 +816,7 @@ }, { "cell_type": "markdown", - "id": "6c3d1d0a", + "id": "428a2130", "metadata": { "editable": true }, @@ -826,7 +826,7 @@ }, { "cell_type": "markdown", - "id": "593a054c", + "id": "a7c5391c", "metadata": { "editable": true }, @@ -838,7 +838,7 @@ }, { "cell_type": "markdown", - "id": "59297c03", + "id": "4ffaa7be", "metadata": { "editable": true }, @@ -850,7 +850,7 @@ }, { "cell_type": "markdown", - "id": "057cdad6", + "id": "46f02a3c", "metadata": { "editable": true }, @@ -862,7 +862,7 @@ }, { "cell_type": "markdown", - "id": "6d33a864", + "id": "3d4a6274", "metadata": { "editable": true }, @@ -874,7 +874,7 @@ }, { "cell_type": "markdown", - "id": "34d0fd48", + "id": "3b840107", "metadata": { "editable": true }, @@ -885,7 +885,7 @@ }, { "cell_type": "markdown", - "id": "3d42eea4", + "id": "9b65c89b", "metadata": { "editable": true }, @@ -898,7 +898,7 @@ }, { "cell_type": "markdown", - "id": "c79756c9", + "id": "6245c0aa", "metadata": { "editable": true }, @@ -910,7 +910,7 @@ }, { "cell_type": "markdown", - "id": "5c86a7ce", + "id": "b832719f", "metadata": { "editable": true }, @@ -920,7 +920,7 @@ }, { "cell_type": "markdown", - "id": "286c2a94", + "id": "4d0a79c9", "metadata": { "editable": true }, @@ -932,7 +932,7 @@ }, { "cell_type": "markdown", - "id": "37d208ee", + "id": "3500bd92", "metadata": { "editable": true }, @@ -942,7 +942,7 @@ }, { "cell_type": "markdown", - "id": "3c8c11c4", + "id": "35e67596", "metadata": { "editable": true }, @@ -953,7 +953,7 @@ }, { "cell_type": "markdown", - "id": "d8c8761a", + "id": "d1d35ce3", "metadata": { "editable": true }, @@ -965,7 +965,7 @@ }, { "cell_type": "markdown", - "id": "23703339", + "id": "2231d370", "metadata": { "editable": true }, @@ -977,7 +977,7 @@ }, { "cell_type": "markdown", - "id": "73ef4ac9", + "id": "05b2c215", "metadata": { "editable": true }, @@ -989,7 +989,7 @@ }, { "cell_type": "markdown", - "id": "a110886a", + "id": "179fda60", "metadata": { "editable": true }, @@ -1000,7 +1000,7 @@ }, { "cell_type": "markdown", - "id": "da8567f7", + "id": "6083dc6b", "metadata": { "editable": true }, @@ -1011,7 +1011,7 @@ }, { "cell_type": "markdown", - "id": "a21844d3", + "id": "c8a2f343", "metadata": { "editable": true }, @@ -1023,7 +1023,7 @@ }, { "cell_type": "markdown", - "id": "443f94d6", + "id": "0f0c3a59", "metadata": { "editable": true }, @@ -1033,7 +1033,7 @@ }, { "cell_type": "markdown", - "id": "7c997294", + "id": "2392af5e", "metadata": { "editable": true }, @@ -1045,7 +1045,7 @@ }, { "cell_type": "markdown", - "id": "73e04632", + "id": "8d5f32b5", "metadata": { "editable": true }, @@ -1055,7 +1055,7 @@ }, { "cell_type": "markdown", - "id": "4b28339c", + "id": "242bfb88", "metadata": { "editable": true }, @@ -1067,7 +1067,7 @@ }, { "cell_type": "markdown", - "id": "c8281e7d", + "id": "3be0e98a", "metadata": { "editable": true }, @@ -1077,7 +1077,7 @@ }, { "cell_type": "markdown", - "id": "b4c113cb", + "id": "d26d8109", "metadata": { "editable": true }, @@ -1089,7 +1089,7 @@ }, { "cell_type": "markdown", - "id": "f4108458", + "id": "35dca0b8", "metadata": { "editable": true }, @@ -1099,7 +1099,7 @@ }, { "cell_type": "markdown", - "id": "037994ec", + "id": "eef08b9d", "metadata": { "editable": true }, @@ -1111,7 +1111,7 @@ }, { "cell_type": "markdown", - "id": "904823ee", + "id": "4c4615e3", "metadata": { "editable": true }, @@ -1122,7 +1122,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "a4b6ca0a", + "id": "bc2317d3", "metadata": { "collapsed": false, "editable": true @@ -1155,7 +1155,7 @@ }, { "cell_type": "markdown", - "id": "c2dbc9b5", + "id": "ff4817d3", "metadata": { "editable": true }, @@ -1166,7 +1166,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "c7fb6aa3", + "id": "1a8e357c", "metadata": { "collapsed": false, "editable": true @@ -1180,7 +1180,7 @@ }, { "cell_type": "markdown", - "id": "ee9a52f1", + "id": "ad6a7c02", "metadata": { "editable": true }, @@ -1191,7 +1191,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "0ef16313", + "id": "f631798e", "metadata": { "collapsed": false, "editable": true @@ -1204,7 +1204,7 @@ }, { "cell_type": "markdown", - "id": "1edac416", + "id": "5801b425", "metadata": { "editable": true }, @@ -1215,7 +1215,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "d9d151d3", + "id": "c17c4cff", "metadata": { "collapsed": false, "editable": true @@ -1233,7 +1233,7 @@ }, { "cell_type": "markdown", - "id": "754cba6b", + "id": "6a2cbdc0", "metadata": { "editable": true }, @@ -1244,7 +1244,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "cb686721", + "id": "bdf87f60", "metadata": { "collapsed": false, "editable": true @@ -1259,7 +1259,7 @@ }, { "cell_type": "markdown", - "id": "4c568d9a", + "id": "7877f2fd", "metadata": { "editable": true }, @@ -1269,7 +1269,7 @@ }, { "cell_type": "markdown", - "id": "9a20583c", + "id": "1db449ed", "metadata": { "editable": true }, @@ -1283,7 +1283,7 @@ }, { "cell_type": "markdown", - "id": "d456864b", + "id": "1995f4a9", "metadata": { "editable": true }, @@ -1295,7 +1295,7 @@ }, { "cell_type": "markdown", - "id": "c9447537", + "id": "15f63d97", "metadata": { "editable": true }, @@ -1306,7 +1306,7 @@ }, { "cell_type": "markdown", - "id": "2561b08f", + "id": "e84fa7eb", "metadata": { "editable": true }, @@ -1318,7 +1318,7 @@ }, { "cell_type": "markdown", - "id": "9acfa2b6", + "id": "638ca172", "metadata": { "editable": true }, @@ -1329,7 +1329,7 @@ }, { "cell_type": "markdown", - "id": "c4a63ef5", + "id": "0dc36f41", "metadata": { "editable": true }, @@ -1340,7 +1340,7 @@ }, { "cell_type": "markdown", - "id": "30998da6", + "id": "ccc4a308", "metadata": { "editable": true }, @@ -1352,7 +1352,7 @@ }, { "cell_type": "markdown", - "id": "66d87e8f", + "id": "05962b72", "metadata": { "editable": true }, @@ -1362,7 +1362,7 @@ }, { "cell_type": "markdown", - "id": "97ea59d5", + "id": "f4fd9df6", "metadata": { "editable": true }, @@ -1374,7 +1374,7 @@ }, { "cell_type": "markdown", - "id": "50b9225f", + "id": "31a5b2d4", "metadata": { "editable": true }, @@ -1386,7 +1386,7 @@ }, { "cell_type": "markdown", - "id": "ca17e40b", + "id": "f963d004", "metadata": { "editable": true }, @@ -1398,7 +1398,7 @@ }, { "cell_type": "markdown", - "id": "a3cb9d18", + "id": "03e5bacf", "metadata": { "editable": true }, @@ -1410,7 +1410,7 @@ }, { "cell_type": "markdown", - "id": "06371a0a", + "id": "bd673c40", "metadata": { "editable": true }, @@ -1421,7 +1421,7 @@ }, { "cell_type": "markdown", - "id": "481e895e", + "id": "631ed22c", "metadata": { "editable": true }, @@ -1433,7 +1433,7 @@ }, { "cell_type": "markdown", - "id": "50c84a7e", + "id": "b40cf902", "metadata": { "editable": true }, @@ -1443,7 +1443,7 @@ }, { "cell_type": "markdown", - "id": "b2ef7b62", + "id": "78f5d49a", "metadata": { "editable": true }, @@ -1455,7 +1455,7 @@ }, { "cell_type": "markdown", - "id": "c46e574a", + "id": "c97d1d7a", "metadata": { "editable": true }, @@ -1465,7 +1465,7 @@ }, { "cell_type": "markdown", - "id": "91e5770e", + "id": "2df5d3d0", "metadata": { "editable": true }, @@ -1477,7 +1477,7 @@ }, { "cell_type": "markdown", - "id": "05200d8c", + "id": "69153524", "metadata": { "editable": true }, @@ -1497,7 +1497,7 @@ }, { "cell_type": "markdown", - "id": "091e6c42", + "id": "b3c67854", "metadata": { "editable": true }, @@ -1509,7 +1509,7 @@ }, { "cell_type": "markdown", - "id": "005a6148", + "id": "c3db6272", "metadata": { "editable": true }, @@ -1519,7 +1519,7 @@ }, { "cell_type": "markdown", - "id": "55cf6c0d", + "id": "6d2cd34c", "metadata": { "editable": true }, @@ -1531,7 +1531,7 @@ }, { "cell_type": "markdown", - "id": "3375fea4", + "id": "ea43124f", "metadata": { "editable": true }, @@ -1541,7 +1541,7 @@ }, { "cell_type": "markdown", - "id": "366193e9", + "id": "58424f50", "metadata": { "editable": true }, @@ -1552,7 +1552,7 @@ }, { "cell_type": "markdown", - "id": "f698e8c7", + "id": "9252a76b", "metadata": { "editable": true }, @@ -1564,7 +1564,7 @@ }, { "cell_type": "markdown", - "id": "96ebe1cb", + "id": "6869545e", "metadata": { "editable": true }, @@ -1576,7 +1576,7 @@ }, { "cell_type": "markdown", - "id": "7b6bef19", + "id": "00428ab7", "metadata": { "editable": true }, @@ -1588,7 +1588,7 @@ }, { "cell_type": "markdown", - "id": "6c3d49d8", + "id": "a18922d0", "metadata": { "editable": true }, @@ -1601,7 +1601,7 @@ }, { "cell_type": "markdown", - "id": "eb6f4e59", + "id": "805d1ee0", "metadata": { "editable": true }, @@ -1612,7 +1612,7 @@ }, { "cell_type": "markdown", - "id": "73ee5acb", + "id": "b57dd5ea", "metadata": { "editable": true }, @@ -1624,7 +1624,7 @@ }, { "cell_type": "markdown", - "id": "ea2f0acc", + "id": "67976b7d", "metadata": { "editable": true }, @@ -1640,7 +1640,7 @@ }, { "cell_type": "markdown", - "id": "ea2c088e", + "id": "77e0497a", "metadata": { "editable": true }, @@ -1652,7 +1652,7 @@ }, { "cell_type": "markdown", - "id": "09d0f89a", + "id": "c4586aa6", "metadata": { "editable": true }, @@ -1663,7 +1663,7 @@ }, { "cell_type": "markdown", - "id": "1ea74654", + "id": "45e92248", "metadata": { "editable": true }, @@ -1675,7 +1675,7 @@ }, { "cell_type": "markdown", - "id": "d768070a", + "id": "b58df66a", "metadata": { "editable": true }, @@ -1685,7 +1685,7 @@ }, { "cell_type": "markdown", - "id": "fd0f164b", + "id": "98b1f06e", "metadata": { "editable": true }, @@ -1697,7 +1697,7 @@ }, { "cell_type": "markdown", - "id": "7725e6c6", + "id": "db4fa5bc", "metadata": { "editable": true }, @@ -1707,7 +1707,7 @@ }, { "cell_type": "markdown", - "id": "9d263e6f", + "id": "4c182d76", "metadata": { "editable": true }, @@ -1719,7 +1719,7 @@ }, { "cell_type": "markdown", - "id": "38950b5c", + "id": "6f3ac80c", "metadata": { "editable": true }, @@ -1729,7 +1729,7 @@ }, { "cell_type": "markdown", - "id": "2a1402d1", + "id": "880f9fbe", "metadata": { "editable": true }, @@ -1741,7 +1741,7 @@ }, { "cell_type": "markdown", - "id": "691bb2be", + "id": "c78b63ba", "metadata": { "editable": true }, @@ -1765,7 +1765,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "cf2962e8", + "id": "5fa1fd87", "metadata": { "collapsed": false, "editable": true @@ -1778,7 +1778,7 @@ }, { "cell_type": "markdown", - "id": "52392a05", + "id": "e0b10ac5", "metadata": { "editable": true }, @@ -1789,7 +1789,7 @@ }, { "cell_type": "markdown", - "id": "240b39a4", + "id": "0db173d9", "metadata": { "editable": true }, @@ -1801,7 +1801,7 @@ }, { "cell_type": "markdown", - "id": "6519bfc2", + "id": "cac2930c", "metadata": { "editable": true }, @@ -1811,7 +1811,7 @@ }, { "cell_type": "markdown", - "id": "1280663c", + "id": "90d618ab", "metadata": { "editable": true }, @@ -1823,7 +1823,7 @@ }, { "cell_type": "markdown", - "id": "1cc5d5b6", + "id": "a0ff7346", "metadata": { "editable": true }, @@ -1837,7 +1837,7 @@ }, { "cell_type": "markdown", - "id": "47ebfb08", + "id": "337cc827", "metadata": { "editable": true }, @@ -1853,7 +1853,7 @@ }, { "cell_type": "markdown", - "id": "c3df406e", + "id": "c0b1f537", "metadata": { "editable": true }, @@ -1863,7 +1863,7 @@ }, { "cell_type": "markdown", - "id": "8b58bdce", + "id": "4c828b26", "metadata": { "editable": true }, @@ -1875,7 +1875,7 @@ }, { "cell_type": "markdown", - "id": "f07f333f", + "id": "2668cc7e", "metadata": { "editable": true }, @@ -1885,7 +1885,7 @@ }, { "cell_type": "markdown", - "id": "65a80224", + "id": "18b9a82d", "metadata": { "editable": true }, @@ -1897,7 +1897,7 @@ }, { "cell_type": "markdown", - "id": "712eb572", + "id": "f1681651", "metadata": { "editable": true }, @@ -1911,7 +1911,7 @@ }, { "cell_type": "markdown", - "id": "702c4425", + "id": "009bc9a7", "metadata": { "editable": true }, @@ -1921,7 +1921,7 @@ }, { "cell_type": "markdown", - "id": "7b7bc005", + "id": "a70e82b2", "metadata": { "editable": true }, @@ -1932,7 +1932,7 @@ }, { "cell_type": "markdown", - "id": "17efa9c8", + "id": "1fb444bf", "metadata": { "editable": true }, @@ -1947,7 +1947,7 @@ }, { "cell_type": "markdown", - "id": "48539c00", + "id": "efb9792b", "metadata": { "editable": true }, @@ -1957,7 +1957,7 @@ }, { "cell_type": "markdown", - "id": "e309864e", + "id": "7ea2379d", "metadata": { "editable": true }, @@ -1969,7 +1969,7 @@ }, { "cell_type": "markdown", - "id": "ad6ac23b", + "id": "109b0551", "metadata": { "editable": true }, @@ -1981,7 +1981,7 @@ }, { "cell_type": "markdown", - "id": "62ec161c", + "id": "bbdc55df", "metadata": { "editable": true }, @@ -1996,7 +1996,7 @@ }, { "cell_type": "markdown", - "id": "3843c09e", + "id": "31da0822", "metadata": { "editable": true }, @@ -2009,7 +2009,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "69d9b859", + "id": "4ddd9983", "metadata": { "collapsed": false, "editable": true @@ -2066,7 +2066,7 @@ }, { "cell_type": "markdown", - "id": "a0b2fd91", + "id": "2dfe5753", "metadata": { "editable": true }, @@ -2077,7 +2077,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "6d71b4a2", + "id": "8e2ca2d9", "metadata": { "collapsed": false, "editable": true @@ -2104,7 +2104,7 @@ }, { "cell_type": "markdown", - "id": "1a6dd499", + "id": "f9e7b881", "metadata": { "editable": true }, @@ -2116,7 +2116,7 @@ }, { "cell_type": "markdown", - "id": "9a56cbf1", + "id": "36ba45da", "metadata": { "editable": true }, @@ -2128,7 +2128,7 @@ }, { "cell_type": "markdown", - "id": "716ca68c", + "id": "5e345cc0", "metadata": { "editable": true }, @@ -2138,7 +2138,7 @@ }, { "cell_type": "markdown", - "id": "eb6ec1d6", + "id": "14bc3bdb", "metadata": { "editable": true }, @@ -2152,7 +2152,7 @@ }, { "cell_type": "markdown", - "id": "7bd11dab", + "id": "99550715", "metadata": { "editable": true }, @@ -2162,7 +2162,7 @@ }, { "cell_type": "markdown", - "id": "14b44b82", + "id": "631985c4", "metadata": { "editable": true }, @@ -2174,7 +2174,7 @@ }, { "cell_type": "markdown", - "id": "f32b8bd3", + "id": "98c4126f", "metadata": { "editable": true }, @@ -2185,7 +2185,7 @@ }, { "cell_type": "markdown", - "id": "a80f3cf0", + "id": "c835d44a", "metadata": { "editable": true }, @@ -2200,7 +2200,7 @@ }, { "cell_type": "markdown", - "id": "f195a664", + "id": "143e72af", "metadata": { "editable": true }, @@ -2214,7 +2214,7 @@ }, { "cell_type": "markdown", - "id": "1d8f6e7a", + "id": "a2acdd56", "metadata": { "editable": true }, @@ -2225,7 +2225,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "94ef2984", + "id": "f025fe90", "metadata": { "collapsed": false, "editable": true @@ -2286,7 +2286,7 @@ }, { "cell_type": "markdown", - "id": "f8ca1f5d", + "id": "cca5b63a", "metadata": { "editable": true }, @@ -2308,7 +2308,7 @@ }, { "cell_type": "markdown", - "id": "213a069d", + "id": "8b538758", "metadata": { "editable": true }, @@ -2321,7 +2321,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "306b8448", + "id": "4cf7d8a8", "metadata": { "collapsed": false, "editable": true @@ -2387,7 +2387,7 @@ }, { "cell_type": "markdown", - "id": "4da5b553", + "id": "f578d3a4", "metadata": { "editable": true }, @@ -2398,7 +2398,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "dca96393", + "id": "a70f3a03", "metadata": { "collapsed": false, "editable": true @@ -2472,7 +2472,7 @@ }, { "cell_type": "markdown", - "id": "71836de5", + "id": "78be1eda", "metadata": { "editable": true }, @@ -2484,7 +2484,7 @@ }, { "cell_type": "markdown", - "id": "5f5900d3", + "id": "25a06db8", "metadata": { "editable": true }, @@ -2505,7 +2505,7 @@ }, { "cell_type": "markdown", - "id": "e1c69762", + "id": "cb111070", "metadata": { "editable": true }, @@ -2537,7 +2537,7 @@ }, { "cell_type": "markdown", - "id": "a24995d4", + "id": "d04f6b36", "metadata": { "editable": true }, @@ -2554,7 +2554,7 @@ }, { "cell_type": "markdown", - "id": "b4723ad5", + "id": "e0671fe5", "metadata": { "editable": true }, @@ -2567,7 +2567,7 @@ }, { "cell_type": "markdown", - "id": "091a9fc0", + "id": "3aa36538", "metadata": { "editable": true }, @@ -2580,7 +2580,7 @@ }, { "cell_type": "markdown", - "id": "d37822e0", + "id": "fa0a5f9c", "metadata": { "editable": true }, @@ -2593,7 +2593,7 @@ }, { "cell_type": "markdown", - "id": "6bea18e4", + "id": "302c406f", "metadata": { "editable": true }, @@ -2607,7 +2607,7 @@ }, { "cell_type": "markdown", - "id": "6d24cc6d", + "id": "8af6f0ab", "metadata": { "editable": true }, @@ -2629,7 +2629,7 @@ }, { "cell_type": "markdown", - "id": "d79d4ee2", + "id": "c4690e57", "metadata": { "editable": true }, @@ -2644,7 +2644,7 @@ }, { "cell_type": "markdown", - "id": "ffdefdaa", + "id": "4f9e2de7", "metadata": { "editable": true }, @@ -2656,7 +2656,7 @@ }, { "cell_type": "markdown", - "id": "5683612e", + "id": "327739e1", "metadata": { "editable": true }, @@ -2669,7 +2669,7 @@ }, { "cell_type": "markdown", - "id": "8a44e062", + "id": "5545ba16", "metadata": { "editable": true }, @@ -2683,7 +2683,7 @@ }, { "cell_type": "markdown", - "id": "ba4deae8", + "id": "aaaf4854", "metadata": { "editable": true }, @@ -2694,7 +2694,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "3875f457", + "id": "b51e78c3", "metadata": { "collapsed": false, "editable": true @@ -2719,7 +2719,7 @@ }, { "cell_type": "markdown", - "id": "b69098df", + "id": "1e743992", "metadata": { "editable": true }, @@ -2735,7 +2735,7 @@ }, { "cell_type": "markdown", - "id": "af41a865", + "id": "eb787ad1", "metadata": { "editable": true }, @@ -2756,7 +2756,7 @@ }, { "cell_type": "markdown", - "id": "2364163e", + "id": "4dafd0a4", "metadata": { "editable": true }, @@ -2776,7 +2776,7 @@ }, { "cell_type": "markdown", - "id": "c652788e", + "id": "c528825a", "metadata": { "editable": true }, @@ -2795,7 +2795,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "84ddc049", + "id": "132e16a2", "metadata": { "collapsed": false, "editable": true @@ -2830,7 +2830,7 @@ }, { "cell_type": "markdown", - "id": "0a5a2ab6", + "id": "8d03c83d", "metadata": { "editable": true }, @@ -2843,7 +2843,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "fabd4be2", + "id": "4b40b1f2", "metadata": { "collapsed": false, "editable": true @@ -2920,7 +2920,7 @@ }, { "cell_type": "markdown", - "id": "ebfd3188", + "id": "a5fa0d98", "metadata": { "editable": true }, @@ -2935,7 +2935,7 @@ }, { "cell_type": "markdown", - "id": "dfc82804", + "id": "c82ed12a", "metadata": { "editable": true }, @@ -2950,7 +2950,7 @@ }, { "cell_type": "markdown", - "id": "e744f5ce", + "id": "80a855b6", "metadata": { "editable": true }, @@ -2962,7 +2962,7 @@ }, { "cell_type": "markdown", - "id": "89eaa299", + "id": "d0ff292f", "metadata": { "editable": true }, @@ -2980,7 +2980,7 @@ }, { "cell_type": "markdown", - "id": "af3e12e1", + "id": "5bbeaa6d", "metadata": { "editable": true }, @@ -2999,7 +2999,7 @@ }, { "cell_type": "markdown", - "id": "43e1f77b", + "id": "8542f50f", "metadata": { "editable": true }, @@ -3011,7 +3011,7 @@ }, { "cell_type": "markdown", - "id": "984f3016", + "id": "a26483ac", "metadata": { "editable": true }, @@ -3021,7 +3021,7 @@ }, { "cell_type": "markdown", - "id": "09935c80", + "id": "062ba9d5", "metadata": { "editable": true }, @@ -3037,7 +3037,7 @@ }, { "cell_type": "markdown", - "id": "2befe8e7", + "id": "eae6e6e8", "metadata": { "editable": true }, @@ -3049,7 +3049,7 @@ }, { "cell_type": "markdown", - "id": "96fae894", + "id": "dbebbec2", "metadata": { "editable": true }, @@ -3059,7 +3059,7 @@ }, { "cell_type": "markdown", - "id": "481533a5", + "id": "210484ea", "metadata": { "editable": true }, @@ -3071,7 +3071,7 @@ }, { "cell_type": "markdown", - "id": "9b228f7a", + "id": "00f879a7", "metadata": { "editable": true }, @@ -3081,7 +3081,7 @@ }, { "cell_type": "markdown", - "id": "e75378ad", + "id": "d2aba931", "metadata": { "editable": true }, @@ -3093,7 +3093,7 @@ }, { "cell_type": "markdown", - "id": "f5d59945", + "id": "23954b3b", "metadata": { "editable": true }, @@ -3109,7 +3109,7 @@ }, { "cell_type": "markdown", - "id": "27a2fbd9", + "id": "8bee2209", "metadata": { "editable": true }, @@ -3121,7 +3121,7 @@ }, { "cell_type": "markdown", - "id": "96a8277f", + "id": "12820d4e", "metadata": { "editable": true }, @@ -3154,7 +3154,7 @@ }, { "cell_type": "markdown", - "id": "af3583c8", + "id": "a91af524", "metadata": { "editable": true }, @@ -3166,7 +3166,7 @@ }, { "cell_type": "markdown", - "id": "5b81c117", + "id": "824c37ae", "metadata": { "editable": true }, @@ -3184,7 +3184,7 @@ }, { "cell_type": "markdown", - "id": "7e12d71b", + "id": "7eccfb35", "metadata": { "editable": true }, @@ -3194,7 +3194,7 @@ }, { "cell_type": "markdown", - "id": "85e55cf5", + "id": "bb5edf8b", "metadata": { "editable": true }, @@ -3225,7 +3225,7 @@ }, { "cell_type": "markdown", - "id": "951e771a", + "id": "42ca26e3", "metadata": { "editable": true }, @@ -3240,7 +3240,7 @@ }, { "cell_type": "markdown", - "id": "b43fb0ea", + "id": "5756240b", "metadata": { "editable": true }, @@ -3258,7 +3258,7 @@ }, { "cell_type": "markdown", - "id": "65ab44a7", + "id": "c95373ac", "metadata": { "editable": true }, @@ -3270,7 +3270,7 @@ }, { "cell_type": "markdown", - "id": "fafcc41a", + "id": "897c8dd4", "metadata": { "editable": true }, @@ -3282,7 +3282,7 @@ }, { "cell_type": "markdown", - "id": "79b58e4e", + "id": "e4d0b41c", "metadata": { "editable": true }, @@ -3300,7 +3300,7 @@ }, { "cell_type": "markdown", - "id": "121b70e5", + "id": "33563225", "metadata": { "editable": true }, @@ -3329,7 +3329,7 @@ }, { "cell_type": "markdown", - "id": "a1fb8843", + "id": "9fef89b1", "metadata": { "editable": true }, @@ -3347,7 +3347,7 @@ }, { "cell_type": "markdown", - "id": "8d5c39b5", + "id": "e585d71d", "metadata": { "editable": true }, @@ -3359,7 +3359,7 @@ }, { "cell_type": "markdown", - "id": "f09683e8", + "id": "7562c9b1", "metadata": { "editable": true }, @@ -3371,7 +3371,7 @@ }, { "cell_type": "markdown", - "id": "ff9285bb", + "id": "81281189", "metadata": { "editable": true }, @@ -3383,7 +3383,7 @@ }, { "cell_type": "markdown", - "id": "3c82455d", + "id": "e5240c76", "metadata": { "editable": true }, @@ -3395,7 +3395,7 @@ }, { "cell_type": "markdown", - "id": "971cdc8f", + "id": "92f25846", "metadata": { "editable": true }, @@ -3407,7 +3407,7 @@ }, { "cell_type": "markdown", - "id": "8b5ce6d5", + "id": "ebd1c2d8", "metadata": { "editable": true }, @@ -3424,7 +3424,7 @@ }, { "cell_type": "markdown", - "id": "8936027f", + "id": "d20ee3a5", "metadata": { "editable": true }, @@ -3443,7 +3443,7 @@ }, { "cell_type": "markdown", - "id": "e117183b", + "id": "c331a269", "metadata": { "editable": true }, @@ -3455,7 +3455,7 @@ }, { "cell_type": "markdown", - "id": "717e5402", + "id": "4a71df9e", "metadata": { "editable": true }, @@ -3469,7 +3469,7 @@ }, { "cell_type": "markdown", - "id": "9ca2e669", + "id": "1a1bb9ae", "metadata": { "editable": true }, @@ -3489,7 +3489,7 @@ }, { "cell_type": "markdown", - "id": "5559377a", + "id": "e62b9f26", "metadata": { "editable": true }, @@ -3527,7 +3527,7 @@ }, { "cell_type": "markdown", - "id": "1890cb32", + "id": "f9d9adc2", "metadata": { "editable": true }, @@ -3539,7 +3539,7 @@ }, { "cell_type": "markdown", - "id": "7d40bd17", + "id": "a4d79d56", "metadata": { "editable": true }, @@ -3549,7 +3549,7 @@ }, { "cell_type": "markdown", - "id": "77e56134", + "id": "9e5b0a07", "metadata": { "editable": true }, @@ -3561,7 +3561,7 @@ }, { "cell_type": "markdown", - "id": "fdc7e85e", + "id": "fbb6af75", "metadata": { "editable": true }, @@ -3572,7 +3572,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "5c77027e", + "id": "4285b7b4", "metadata": { "collapsed": false, "editable": true @@ -3617,7 +3617,7 @@ }, { "cell_type": "markdown", - "id": "d400a43a", + "id": "8455bf36", "metadata": { "editable": true }, @@ -3634,7 +3634,7 @@ { "cell_type": "code", "execution_count": 16, - "id": "e142c575", + "id": "e1e93736", "metadata": { "collapsed": false, "editable": true @@ -3662,7 +3662,7 @@ }, { "cell_type": "markdown", - "id": "4a8eaab5", + "id": "c51dc7ec", "metadata": { "editable": true }, @@ -3677,7 +3677,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "b41b7db3", + "id": "d1db6a25", "metadata": { "collapsed": false, "editable": true @@ -3721,7 +3721,7 @@ }, { "cell_type": "markdown", - "id": "d548f0cb", + "id": "68b6c330", "metadata": { "editable": true }, @@ -3731,7 +3731,7 @@ }, { "cell_type": "markdown", - "id": "96676ad1", + "id": "af9ae977", "metadata": { "editable": true }, @@ -3742,7 +3742,7 @@ { "cell_type": "code", "execution_count": 18, - "id": "0aeeac8c", + "id": "9acb4eb4", "metadata": { "collapsed": false, "editable": true @@ -3770,7 +3770,7 @@ }, { "cell_type": "markdown", - "id": "d4523d9b", + "id": "6d6b42f7", "metadata": { "editable": true }, @@ -3785,7 +3785,7 @@ }, { "cell_type": "markdown", - "id": "65e8cd48", + "id": "343a76ad", "metadata": { "editable": true }, @@ -3796,7 +3796,7 @@ { "cell_type": "code", "execution_count": 19, - "id": "c45b4b03", + "id": "75535c38", "metadata": { "collapsed": false, "editable": true @@ -3824,7 +3824,7 @@ }, { "cell_type": "markdown", - "id": "47e146d0", + "id": "d54a7039", "metadata": { "editable": true }, @@ -3835,7 +3835,7 @@ { "cell_type": "code", "execution_count": 20, - "id": "7cf9ba71", + "id": "e81e3cef", "metadata": { "collapsed": false, "editable": true @@ -3860,7 +3860,7 @@ }, { "cell_type": "markdown", - "id": "304cb623", + "id": "f198f442", "metadata": { "editable": true }, @@ -3871,7 +3871,7 @@ { "cell_type": "code", "execution_count": 21, - "id": "aaf5c76c", + "id": "3fceaab1", "metadata": { "collapsed": false, "editable": true @@ -3907,7 +3907,7 @@ { "cell_type": "code", "execution_count": 22, - "id": "d8fac3e8", + "id": "90a85d63", "metadata": { "collapsed": false, "editable": true @@ -3927,7 +3927,7 @@ }, { "cell_type": "markdown", - "id": "af8a7233", + "id": "6962bfc0", "metadata": { "editable": true }, @@ -3938,7 +3938,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "35a7b10b", + "id": "23f3bd84", "metadata": { "collapsed": false, "editable": true @@ -3976,7 +3976,7 @@ }, { "cell_type": "markdown", - "id": "90d3920f", + "id": "aba409e2", "metadata": { "editable": true }, @@ -3986,7 +3986,7 @@ }, { "cell_type": "markdown", - "id": "128a60a9", + "id": "ae818693", "metadata": { "editable": true }, @@ -4000,7 +4000,7 @@ { "cell_type": "code", "execution_count": 24, - "id": "fa821d4c", + "id": "d743e45d", "metadata": { "collapsed": false, "editable": true @@ -4022,7 +4022,7 @@ }, { "cell_type": "markdown", - "id": "4601ca85", + "id": "46fb5331", "metadata": { "editable": true }, @@ -4032,7 +4032,7 @@ }, { "cell_type": "markdown", - "id": "646813ea", + "id": "5f5ddf2e", "metadata": { "editable": true }, @@ -4043,7 +4043,7 @@ { "cell_type": "code", "execution_count": 25, - "id": "a83e6d33", + "id": "a10c5598", "metadata": { "collapsed": false, "editable": true @@ -4065,7 +4065,7 @@ }, { "cell_type": "markdown", - "id": "66408265", + "id": "cea9e306", "metadata": { "editable": true }, @@ -4078,7 +4078,7 @@ { "cell_type": "code", "execution_count": 26, - "id": "5394dda5", + "id": "49e2d555", "metadata": { "collapsed": false, "editable": true @@ -4103,7 +4103,7 @@ }, { "cell_type": "markdown", - "id": "5668b5a9", + "id": "5eb98f5d", "metadata": { "editable": true }, @@ -4115,7 +4115,7 @@ { "cell_type": "code", "execution_count": 27, - "id": "ab83c77e", + "id": "f816abb3", "metadata": { "collapsed": false, "editable": true @@ -4130,7 +4130,7 @@ }, { "cell_type": "markdown", - "id": "971987ed", + "id": "21671b8e", "metadata": { "editable": true }, @@ -4145,7 +4145,7 @@ { "cell_type": "code", "execution_count": 28, - "id": "74c2088a", + "id": "b1a866a7", "metadata": { "collapsed": false, "editable": true @@ -4205,7 +4205,7 @@ }, { "cell_type": "markdown", - "id": "c42fc6e5", + "id": "2c79ef7d", "metadata": { "editable": true }, @@ -4216,7 +4216,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "dd515b08", + "id": "636cf11f", "metadata": { "collapsed": false, "editable": true @@ -4280,7 +4280,7 @@ }, { "cell_type": "markdown", - "id": "5e386451", + "id": "90dcdf32", "metadata": { "editable": true }, @@ -4291,7 +4291,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "027bf4c7", + "id": "9d954dd3", "metadata": { "collapsed": false, "editable": true @@ -4340,7 +4340,7 @@ }, { "cell_type": "markdown", - "id": "c1f45ccb", + "id": "919950b2", "metadata": { "editable": true }, @@ -4352,7 +4352,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "f2c2d0dd", + "id": "a3b4c6f7", "metadata": { "collapsed": false, "editable": true @@ -4436,7 +4436,7 @@ }, { "cell_type": "markdown", - "id": "27cfcd23", + "id": "84f737fa", "metadata": { "editable": true }, @@ -4447,7 +4447,7 @@ { "cell_type": "code", "execution_count": 32, - "id": "8d2b23d8", + "id": "865d644f", "metadata": { "collapsed": false, "editable": true @@ -4525,7 +4525,7 @@ }, { "cell_type": "markdown", - "id": "58a8f732", + "id": "1fd6c389", "metadata": { "editable": true }, @@ -4536,7 +4536,7 @@ { "cell_type": "code", "execution_count": 33, - "id": "6781ed86", + "id": "18eb26ab", "metadata": { "collapsed": false, "editable": true @@ -4600,7 +4600,7 @@ }, { "cell_type": "markdown", - "id": "358c2c6e", + "id": "119315be", "metadata": { "editable": true }, @@ -4610,7 +4610,7 @@ }, { "cell_type": "markdown", - "id": "ef808ab2", + "id": "069b4706", "metadata": { "editable": true }, @@ -4621,7 +4621,7 @@ { "cell_type": "code", "execution_count": 34, - "id": "4e115811", + "id": "e2b18b40", "metadata": { "collapsed": false, "editable": true @@ -4673,16 +4673,12 @@ " xi = X[random_index:random_index+M]\n", " yi = y[random_index:random_index+M]\n", " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", - "\t# Previous value for the outer product of gradients\n", - " Previous = Giter\n", - "\t# Accumulated gradient\n", - "# Giter +=gradients @ gradients.T\n", "\t# Scaling with rho the new and the previous results\n", " Giter = (rho*Giter+(1-rho)*gradients*gradients)\n", "\t# Taking the diagonal only and inverting\n", " Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]\n", "\t# Hadamard product\n", - " update = np.multiply(Ginverse,gradients)\n", + " update = Ginverse*gradients\n", " theta -= update\n", "print(\"theta from own RMSprop\")\n", "print(theta)" @@ -4690,7 +4686,88 @@ }, { "cell_type": "markdown", - "id": "33e4f0c7", + "id": "7858afb4", + "metadata": { + "editable": true + }, + "source": [ + "## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "d416cf5e", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n", + "# OLS example\n", + "from random import random, seed\n", + "import numpy as np\n", + "import autograd.numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from autograd import grad\n", + "\n", + "# Note change from previous example\n", + "def CostOLS(y,X,theta):\n", + " return np.sum((y-X @ theta)**2)\n", + "\n", + "n = 1000\n", + "x = np.random.rand(n,1)\n", + "y = 2.0+3*x +4*x*x# +np.random.randn(n,1)\n", + "\n", + "X = np.c_[np.ones((n,1)), x, x*x]\n", + "XT_X = X.T @ X\n", + "theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", + "print(\"Own inversion\")\n", + "print(theta_linreg)\n", + "\n", + "\n", + "# Note that we request the derivative wrt third argument (theta, 2 here)\n", + "training_gradient = grad(CostOLS,2)\n", + "# Define parameters for Stochastic Gradient Descent\n", + "n_epochs = 50\n", + "M = 5 #size of each minibatch\n", + "m = int(n/M) #number of minibatches\n", + "# Guess for unknown parameters theta\n", + "theta = np.random.randn(3,1)\n", + "\n", + "# Value for learning rate\n", + "eta = 0.01\n", + "# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980\n", + "beta1 = 0.9\n", + "beta2 = 0.999\n", + "# Including AdaGrad parameter to avoid possible division by zero\n", + "delta = 1e-7\n", + "iter = 0\n", + "for epoch in range(n_epochs):\n", + " first_moment = 0.0\n", + " second_moment = 0.0\n", + " iter += 1\n", + " for i in range(m):\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", + " gradients = (1.0/M)*training_gradient(yi, xi, theta)\n", + " # Computing moments first\n", + " first_moment = beta1*first_moment + (1-beta1)*gradients\n", + " second_moment = beta2*second_moment+(1-beta2)*gradients*gradients\n", + " first_term = first_moment/(1.0-beta1**iter)\n", + " second_term = second_moment/(1.0-beta2**iter)\n", + "\t# Scaling with rho the new and the previous results\n", + " update = eta*first_term/(np.sqrt(second_term)+delta)\n", + " theta -= update\n", + "print(\"theta from own ADAM\")\n", + "print(theta)" + ] + }, + { + "cell_type": "markdown", + "id": "fb2a3c56", "metadata": { "editable": true }, @@ -4700,8 +4777,8 @@ }, { "cell_type": "code", - "execution_count": 35, - "id": "5d85c167", + "execution_count": 36, + "id": "8edbb7b5", "metadata": { "collapsed": false, "editable": true @@ -4745,7 +4822,7 @@ }, { "cell_type": "markdown", - "id": "d9c27c54", + "id": "b081c299", "metadata": { "editable": true }, @@ -4763,8 +4840,8 @@ }, { "cell_type": "code", - "execution_count": 36, - "id": "0c5ee173", + "execution_count": 37, + "id": "49314f31", "metadata": { "collapsed": false, "editable": true @@ -4781,22 +4858,6 @@ "derivative_fn = grad(sum_logistic)\n", "print(derivative_fn(x_small))" ] - }, - { - "cell_type": "markdown", - "id": "650b565f", - "metadata": { - "editable": true - }, - "source": [ - "## Weekend challenge\n", - "\n", - "* Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.\n", - "\n", - "* Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods\n", - "\n", - "* Extend from linear regression to logistic regression." - ] } ], "metadata": {}, diff --git a/doc/src/week39/adam.py b/doc/src/week39/adam.py new file mode 100644 index 000000000..de3ab2dcf --- /dev/null +++ b/doc/src/week39/adam.py @@ -0,0 +1,59 @@ +# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent +# OLS example +from random import random, seed +import numpy as np +import autograd.numpy as np +import matplotlib.pyplot as plt +from autograd import grad + +# Note change from previous example +def CostOLS(y,X,theta): + return np.sum((y-X @ theta)**2) + +n = 1000 +x = np.random.rand(n,1) +y = 2.0+3*x +4*x*x# +np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x, x*x] +XT_X = X.T @ X +theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y) +print("Own inversion") +print(theta_linreg) + + +# Note that we request the derivative wrt third argument (theta, 2 here) +training_gradient = grad(CostOLS,2) +# Define parameters for Stochastic Gradient Descent +n_epochs = 50 +M = 5 #size of each minibatch +m = int(n/M) #number of minibatches +# Guess for unknown parameters theta +theta = np.random.randn(3,1) + +# Value for learning rate +eta = 0.01 +# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980 +beta1 = 0.9 +beta2 = 0.999 +# Including AdaGrad parameter to avoid possible division by zero +delta = 1e-7 +iter = 0 +for epoch in range(n_epochs): + first_moment = 0.0 + second_moment = 0.0 + iter += 1 + for i in range(m): + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] + gradients = (1.0/M)*training_gradient(yi, xi, theta) + # Computing moments first + first_moment = beta1*first_moment + (1-beta1)*gradients + second_moment = beta2*second_moment+(1-beta2)*gradients*gradients + first_term = first_moment/(1.0-beta1**iter) + second_term = second_moment/(1.0-beta2**iter) + # Scaling with rho the new and the previous results + update = eta*first_term/(np.sqrt(second_term)+delta) + theta -= update +print("theta from own ADAM") +print(theta) diff --git a/doc/src/week39/rmsprop.py b/doc/src/week39/rmsprop.py index 66e01e257..5239f58b3 100644 --- a/doc/src/week39/rmsprop.py +++ b/doc/src/week39/rmsprop.py @@ -43,16 +43,14 @@ for epoch in range(n_epochs): xi = X[random_index:random_index+M] yi = y[random_index:random_index+M] gradients = (1.0/M)*training_gradient(yi, xi, theta) - # Previous value for the outer product of gradients - Previous = Giter # Accumulated gradient -# Giter +=gradients @ gradients.T # Scaling with rho the new and the previous results Giter = (rho*Giter+(1-rho)*gradients*gradients) # Taking the diagonal only and inverting Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))] # Hadamard product - update = np.multiply(Ginverse,gradients) + update = Ginverse*gradients +# update = np.multiply(Ginverse,gradients) theta -= update print("theta from own RMSprop") print(theta) diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt index f53b4c92e..de109b210 100644 --- a/doc/src/week39/week39.do.txt +++ b/doc/src/week39/week39.do.txt @@ -2550,21 +2550,82 @@ for epoch in range(n_epochs): xi = X[random_index:random_index+M] yi = y[random_index:random_index+M] gradients = (1.0/M)*training_gradient(yi, xi, theta) - # Previous value for the outer product of gradients - Previous = Giter - # Accumulated gradient -# Giter +=gradients @ gradients.T # Scaling with rho the new and the previous results Giter = (rho*Giter+(1-rho)*gradients*gradients) # Taking the diagonal only and inverting Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))] # Hadamard product - update = np.multiply(Ginverse,gradients) + update = Ginverse*gradients theta -= update print("theta from own RMSprop") print(theta) + !ec +!split +===== And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf" ===== + +!bc pycod +# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent +# OLS example +from random import random, seed +import numpy as np +import autograd.numpy as np +import matplotlib.pyplot as plt +from autograd import grad + +# Note change from previous example +def CostOLS(y,X,theta): + return np.sum((y-X @ theta)**2) + +n = 1000 +x = np.random.rand(n,1) +y = 2.0+3*x +4*x*x# +np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x, x*x] +XT_X = X.T @ X +theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y) +print("Own inversion") +print(theta_linreg) + + +# Note that we request the derivative wrt third argument (theta, 2 here) +training_gradient = grad(CostOLS,2) +# Define parameters for Stochastic Gradient Descent +n_epochs = 50 +M = 5 #size of each minibatch +m = int(n/M) #number of minibatches +# Guess for unknown parameters theta +theta = np.random.randn(3,1) + +# Value for learning rate +eta = 0.01 +# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980 +beta1 = 0.9 +beta2 = 0.999 +# Including AdaGrad parameter to avoid possible division by zero +delta = 1e-7 +iter = 0 +for epoch in range(n_epochs): + first_moment = 0.0 + second_moment = 0.0 + iter += 1 + for i in range(m): + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] + gradients = (1.0/M)*training_gradient(yi, xi, theta) + # Computing moments first + first_moment = beta1*first_moment + (1-beta1)*gradients + second_moment = beta2*second_moment+(1-beta2)*gradients*gradients + first_term = first_moment/(1.0-beta1**iter) + second_term = second_moment/(1.0-beta2**iter) + # Scaling with rho the new and the previous results + update = eta*first_term/(np.sqrt(second_term)+delta) + theta -= update +print("theta from own ADAM") +print(theta) +!ec !split ===== And Logistic Regression ===== @@ -2630,11 +2691,4 @@ print(derivative_fn(x_small)) !ec -!split -===== Weekend challenge ===== - -* Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms. -* Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods -* Extend from linear regression to logistic regression. -