lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

small update

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-09-30 09:08:09 +02:00
parent d5b60602de
commit 11eccda344
7 changed files with 301 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-reveal.html
+4 -4
View File
@@ -3495,7 +3495,7 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #228B22"># OLS example with AdaGrad</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
@@ -3539,9 +3539,9 @@ M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size o
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Including AdaGrad</span>
delta = <span style="color: #B452CD">0.000001</span>
r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">for</span> _ <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(gradients.shape[<span style="color: #B452CD">0</span>])]
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
@@ -3551,8 +3551,8 @@ delta = <span style="color: #B452CD">0.000001</span>
<span style="color: #228B22"># calculate squared gradient by Hadamard multiplication</span>
r -= gradients*gradients
<span style="color: #228B22"># compute update</span>
update = (<span style="color: #B452CD">1.0</span>/delta+np.sqrt(r))*gradients
theta = eta*update
update = <span style="color: #B452CD">1.0</span>/(delta+np.sqrt(r))*gradients
theta -= eta*update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
doc/pub/week39/html/week39-solarized.html
+4 -4
View File
@@ -3422,7 +3422,7 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #228B22"># OLS example with AdaGrad</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
@@ -3466,9 +3466,9 @@ M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size o
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Including AdaGrad</span>
delta = <span style="color: #B452CD">0.000001</span>
r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">for</span> _ <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(gradients.shape[<span style="color: #B452CD">0</span>])]
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
@@ -3478,8 +3478,8 @@ delta = <span style="color: #B452CD">0.000001</span>
<span style="color: #228B22"># calculate squared gradient by Hadamard multiplication</span>
r -= gradients*gradients
<span style="color: #228B22"># compute update</span>
update = (<span style="color: #B452CD">1.0</span>/delta+np.sqrt(r))*gradients
theta = eta*update
update = <span style="color: #B452CD">1.0</span>/(delta+np.sqrt(r))*gradients
theta -= eta*update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
doc/pub/week39/html/week39.html
+4 -4
View File
@@ -3499,7 +3499,7 @@ delta_momentum <span style="color: #666666">=</span> <span style="color: #666666
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #408080; font-style: italic"># OLS example</span>
<span style="color: #408080; font-style: italic"># OLS example with AdaGrad</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
@@ -3543,9 +3543,9 @@ M <span style="color: #666666">=</span> <span style="color: #666666">5</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic"># Including AdaGrad</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">0.000001</span>
r <span style="color: #666666">=</span> [<span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">for</span> _ <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(gradients<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>])]
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
@@ -3555,8 +3555,8 @@ delta <span style="color: #666666">=</span> <span style="color: #666666">0.0000
<span style="color: #408080; font-style: italic"># calculate squared gradient by Hadamard multiplication</span>
r <span style="color: #666666">-=</span> gradients<span style="color: #666666">*</span>gradients
<span style="color: #408080; font-style: italic"># compute update</span>
update <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(r))<span style="color: #666666">*</span>gradients
theta <span style="color: #666666">=</span> eta<span style="color: #666666">*</span>update
update <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(r))<span style="color: #666666">*</span>gradients
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #008000">print</span>(theta)
</pre>
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+282 -282
View File
File diff suppressed because it is too large Load Diff
doc/src/week39/adagrad.py
+3 -3
View File
@@ -43,9 +43,9 @@ M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
theta = np.random.randn(2,1)
# Including AdaGrad
delta = 0.000001
r = [0.0 for _ in range(gradients.shape[0])]
for epoch in range(n_epochs):
for i in range(m):
random_index = M*np.random.randint(m)
@@ -55,8 +55,8 @@ for epoch in range(n_epochs):
# calculate squared gradient by Hadamard multiplication
r -= gradients*gradients
# compute update
update = (1.0/delta+np.sqrt(r))*gradients
theta = eta*update
update = 1.0/(delta+np.sqrt(r))*gradients
theta -= eta*update
print("theta from own AdaGrad")
print(theta)
doc/src/week39/week39.do.txt
+4 -5
View File
@@ -2435,7 +2435,7 @@ print(theta)
===== Same problem but now with AdaGrad =====
!bc pycod
# Using Autograd to calculate gradients using SGD
# OLS example
# OLS example with AdaGrad
from random import random, seed
import numpy as np
import autograd.numpy as np
@@ -2479,9 +2479,9 @@ M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
theta = np.random.randn(2,1)
# Including AdaGrad
delta = 0.000001
r = [0.0 for _ in range(gradients.shape[0])]
for epoch in range(n_epochs):
for i in range(m):
random_index = M*np.random.randint(m)
@@ -2491,11 +2491,10 @@ for epoch in range(n_epochs):
# calculate squared gradient by Hadamard multiplication
r -= gradients*gradients
# compute update
update = (1.0/delta+np.sqrt(r))*gradients
theta = eta*update
update = 1.0/(delta+np.sqrt(r))*gradients
theta -= eta*update
print("theta from own AdaGrad")
print(theta)
!ec