lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaning up codes for week 39

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-11-01 15:43:17 +01:00
parent 653d5a0a3e
commit d97092d9d2
9 changed files with 344 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-bs.html
+1 -1
View File
@@ -421,7 +421,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Oct 31, 2022</h4>
<h4>Nov 1, 2022</h4>
</center> <!-- date -->
<br>
doc/pub/week39/html/week39-reveal.html
+10 -15
View File
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Oct 31, 2022</h4>
<h4>Nov 1, 2022</h4>
</center> <!-- date -->
<br>
@@ -3526,9 +3526,9 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
@@ -3551,19 +3551,14 @@ eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># The outer product is calculated from scratch for each epoch</span>
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Calculate the outer product of the gradients</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Simpler algorithm with only diagonal elements</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># compute update</span>
update = np.multiply(Ginverse,gradients)
Giter += gradients*gradients
update = gradients*eta/(delta+np.sqrt(Giter))
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -3606,7 +3601,7 @@ delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
@@ -3633,18 +3628,18 @@ rho = <span style="color: #B452CD">0.99</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Accumulated gradient</span>
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = gradients*eta/(delta+np.sqrt(Giter))
<span style="color: #228B22"># Hadamard product</span>
update = Ginverse*gradients
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
doc/pub/week39/html/week39-solarized.html
+10 -15
View File
@@ -335,7 +335,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Oct 31, 2022</h4>
<h4>Nov 1, 2022</h4>
</center> <!-- date -->
<br>
@@ -3463,9 +3463,9 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
@@ -3488,19 +3488,14 @@ eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># The outer product is calculated from scratch for each epoch</span>
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Calculate the outer product of the gradients</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Simpler algorithm with only diagonal elements</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># compute update</span>
update = np.multiply(Ginverse,gradients)
Giter += gradients*gradients
update = gradients*eta/(delta+np.sqrt(Giter))
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -3542,7 +3537,7 @@ delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
@@ -3569,18 +3564,18 @@ rho = <span style="color: #B452CD">0.99</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
Giter = <span style="color: #B452CD">0.0</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Accumulated gradient</span>
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = gradients*eta/(delta+np.sqrt(Giter))
<span style="color: #228B22"># Hadamard product</span>
update = Ginverse*gradients
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
doc/pub/week39/html/week39.html
+10 -15
View File
@@ -412,7 +412,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Oct 31, 2022</h4>
<h4>Nov 1, 2022</h4>
</center> <!-- date -->
<br>
@@ -3540,9 +3540,9 @@ delta_momentum <span style="color: #666666">=</span> <span style="color: #666666
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(y,X,theta):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y<span style="color: #666666">-</span>X <span style="color: #666666">@</span> theta)<span style="color: #666666">**2</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x<span style="color: #408080; font-style: italic"># +np.random.randn(n,1)</span>
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x, x<span style="color: #666666">*</span>x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
@@ -3565,19 +3565,14 @@ eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</spa
<span style="color: #408080; font-style: italic"># Including AdaGrad parameter to avoid possible division by zero</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span>
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #408080; font-style: italic"># The outer product is calculated from scratch for each epoch</span>
Giter <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(shape<span style="color: #666666">=</span>(<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
Giter <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># Calculate the outer product of the gradients</span>
Giter <span style="color: #666666">+=</span>gradients <span style="color: #666666">@</span> gradients<span style="color: #666666">.</span>T
<span style="color: #408080; font-style: italic"># Simpler algorithm with only diagonal elements</span>
Ginverse <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>diagonal(Giter)))]
<span style="color: #408080; font-style: italic"># compute update</span>
update <span style="color: #666666">=</span> np<span style="color: #666666">.</span>multiply(Ginverse,gradients)
Giter <span style="color: #666666">+=</span> gradients<span style="color: #666666">*</span>gradients
update <span style="color: #666666">=</span> gradients<span style="color: #666666">*</span>eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(Giter))
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #008000">print</span>(theta)
@@ -3619,7 +3614,7 @@ delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(y,X,theta):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y<span style="color: #666666">-</span>X <span style="color: #666666">@</span> theta)<span style="color: #666666">**2</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x<span style="color: #408080; font-style: italic"># +np.random.randn(n,1)</span>
@@ -3646,18 +3641,18 @@ rho <span style="color: #666666">=</span> <span style="color: #666666">0.99</spa
<span style="color: #408080; font-style: italic"># Including AdaGrad parameter to avoid possible division by zero</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span>
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
Giter <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(shape<span style="color: #666666">=</span>(<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
Giter <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># Accumulated gradient</span>
<span style="color: #408080; font-style: italic"># Scaling with rho the new and the previous results</span>
Giter <span style="color: #666666">=</span> (rho<span style="color: #666666">*</span>Giter<span style="color: #666666">+</span>(<span style="color: #666666">1-</span>rho)<span style="color: #666666">*</span>gradients<span style="color: #666666">*</span>gradients)
<span style="color: #408080; font-style: italic"># Taking the diagonal only and inverting</span>
Ginverse <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>diagonal(Giter)))]
update <span style="color: #666666">=</span> gradients<span style="color: #666666">*</span>eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(Giter))
<span style="color: #408080; font-style: italic"># Hadamard product</span>
update <span style="color: #666666">=</span> Ginverse<span style="color: #666666">*</span>gradients
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #008000">print</span>(theta)
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+293 -298
View File
File diff suppressed because it is too large Load Diff
doc/src/week39/codes/adagradSGD.py
+6 -7
View File
@@ -10,9 +10,9 @@ from autograd import grad
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
y = 2.0+3*x +4*x*x
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
@@ -35,16 +35,15 @@ eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
Giter = 0.0
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Giter +=gradients @ gradients.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = np.multiply(Ginverse,gradients)
theta -= update
Giter += gradients*gradients
Ginverse = gradients*eta/(delta+np.sqrt(Giter))
theta -= Ginverse
print("theta from own AdaGrad")
print(theta)
doc/src/week39/codes/rmsprop.py
+3 -5
View File
@@ -10,7 +10,7 @@ from autograd import grad
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
@@ -37,7 +37,7 @@ rho = 0.99
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
Giter = 0.0
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
@@ -47,10 +47,8 @@ for epoch in range(n_epochs):
# Scaling with rho the new and the previous results
Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = gradients*eta/(delta+np.sqrt(Giter))
# Hadamard product
update = Ginverse*gradients
# update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own RMSprop")
print(theta)
doc/src/week39/week39.do.txt
+11 -15
View File
@@ -2457,9 +2457,9 @@ from autograd import grad
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
y = 2.0+3*x +4*x*x
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
@@ -2482,22 +2482,19 @@ eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
# The outer product is calculated from scratch for each epoch
Giter = np.zeros(shape=(3,3))
Giter = 0.0
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Calculate the outer product of the gradients
Giter +=gradients @ gradients.T
# Simpler algorithm with only diagonal elements
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# compute update
update = np.multiply(Ginverse,gradients)
Giter += gradients*gradients
update = gradients*eta/(delta+np.sqrt(Giter))
theta -= update
print("theta from own AdaGrad")
print(theta)
!ec
Running this code we note an almost perfect agreement with the results from matrix inversion.
@@ -2517,7 +2514,7 @@ from autograd import grad
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
@@ -2544,22 +2541,21 @@ rho = 0.99
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
Giter = 0.0
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Accumulated gradient
# Scaling with rho the new and the previous results
Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = gradients*eta/(delta+np.sqrt(Giter))
# Hadamard product
update = Ginverse*gradients
theta -= update
print("theta from own RMSprop")
print(theta)
!ec
!split